Catch sqlite timeout & forward queries to CalCat

This should cope better if there are GPFS issues which mean the database stays locked for longer than it should (as we think caused https://redmine.xfel.eu/issues/163349 ).

Of course, if GPFS is very slow, that's likely to be a problem for calibration anyway, because the raw data, the output and the constants are all on GPFS. But this might help if there's a very temporary issue.

calparrot/proxy.py

@@ -7,6 +7,7 @@ constants.
 import asyncio
 import logging
 import os
+import sqlite3
 from dataclasses import dataclass
 from secrets import token_hex
 from urllib.parse import parse_qs, urlencode
@@ -140,7 +141,20 @@ class ProxyHandler(RequestHandler):
             )
 
     async def _get_with_cache(self, req_path) -> ProxyResponse:
-        if self.response_store.insert_pending(req_path, self.request.body):
+        store_response = True
+        try:
+            new_query = self.response_store.insert_pending(req_path, self.request.body)
+        except sqlite3.OperationalError:
+            log.warning(
+                "Could not check for query (%s) in CalParrot database. "
+                "Will treat this as new & forward to CalCat.",
+                exc_info=True,
+            )
+            new_query = True
+            # Don't try to write the response if there are issues accessing the DB
+            store_response = False
+
+        if new_query:
             if os.environ.get('CALPARROT_NEW_QUERY', '').lower() == 'warn':
                 log.warning(
                     "CalParrot received a new query (for %s) when it was told to "
@@ -150,9 +164,11 @@ class ProxyHandler(RequestHandler):
                     "New CalCat queries made while reproducing correction"
                 )
 
-            # We put a pending response in the database: now query upstream
+            # Send query upstream
             resp = await self._get_upstream(req_path)
-            self._record_response(req_path, resp)
+            if store_response:
+                # We put a pending record in the database; add the response
+                self._record_response(req_path, resp)
         else:
             # This URL is already in the database
             try:
@@ -169,7 +185,8 @@ class ProxyHandler(RequestHandler):
                         "forwarding request for %s", req_path
                 )
                 resp = await self._get_upstream(req_path)
-                self._record_response(req_path, resp)
+                if store_response:
+                    self._record_response(req_path, resp)
 
         return resp