Merge pull request #187 from spc-group/run_browser

Asynchronous run browser loading

environment.yml

@@ -59,6 +59,7 @@ dependencies:
   # - pyqt >=5.12, <5.13  
   - qt >=5.12
   - qtawesome
+  - qasync
 
   # --- general support packages
   - bitshuffle

pytest.ini

@@ -1,3 +1,3 @@
 [pytest]
 mongodb_fixture_dir =
-  tests/fixtures
+  src/haven/tests/fixtures

src/conftest.py

@@ -1,14 +1,18 @@
+import gc
+import asyncio
 import os
 import subprocess
 from pathlib import Path
 from unittest import mock
 
 import psutil
+from qasync import QEventLoop, DefaultQEventLoopPolicy
 
 # from pydm.data_plugins import plugin_modules, add_plugin
 import pydm
 import pytest
-from bluesky import RunEngine
+import numpy as np
+import pandas as pd
 from ophyd import DynamicDeviceComponent as DDC
 from ophyd import Kind
 from ophyd.sim import (
@@ -18,6 +22,12 @@
     make_fake_device,
 )
 from pytestqt.qt_compat import qt_api
+from tiled.adapters.mapping import MapAdapter
+from tiled.adapters.xarray import DatasetAdapter
+from tiled.adapters.table import TableAdapter
+from tiled.client import Context, from_context
+from tiled.server.app import build_app
+
 
 import haven
 from firefly.application import FireflyApplication
@@ -35,6 +45,7 @@
 from haven.instrument.slits import ApertureSlits, BladeSlits
 from haven.instrument.xspress import Xspress3Detector
 from haven.instrument.xspress import add_mcas as add_xspress_mcas
+from haven.catalog import Catalog
 
 top_dir = Path(__file__).parent.resolve()
 haven_dir = top_dir / "haven"
@@ -67,53 +78,6 @@ def beamline_connected():
         yield
 
 
-class RunEngineStub(RunEngine):
-    def __repr__(self):
-        return "<run_engine.RunEngineStub>"
-
-
-@pytest.fixture()
-def RE(event_loop):
-    return RunEngineStub(call_returns_result=True)
-
-
-@pytest.fixture(scope="session")
-def qapp_cls():
-    return FireflyApplication
-
-
-# def pytest_configure(config):
-#     app = QtWidgets.QApplication.instance()
-#     assert app is None
-#     app = FireflyApplication()
-#     app = QtWidgets.QApplication.instance()
-#     assert isinstance(app, FireflyApplication)
-#     # # Create event loop for asyncio stuff
-#     # loop = asyncio.new_event_loop()
-#     # asyncio.set_event_loop(loop)
-
-
-def tiled_is_running(port, match_command=True):
-    lsof = subprocess.run(["lsof", "-i", f":{port}", "-F"], capture_output=True)
-    assert lsof.stderr.decode() == ""
-    stdout = lsof.stdout.decode().split("\n")
-    is_running = len(stdout) >= 3
-    if match_command:
-        is_running = is_running and stdout[3] == "ctiled"
-    return is_running
-
-
-def kill_process(process_name):
-    processes = []
-    for proc in psutil.process_iter():
-        # check whether the process name matches
-        if proc.name() == process_name:
-            proc.kill()
-            processes.append(proc)
-    # Wait for them all the terminate
-    [proc.wait(timeout=5) for proc in processes]
-
-
 @pytest.fixture()
 def sim_registry(monkeypatch):
     # mock out Ophyd connections so devices can be created
@@ -307,68 +271,124 @@ def shutters(sim_registry):
     yield shutters
 
 
-@pytest.fixture(scope="session")
-def pydm_ophyd_plugin():
-    return pydm.data_plugins.plugin_for_address("haven://")
-
-
-qs_status = {
-    "msg": "RE Manager v0.0.18",
-    "items_in_queue": 0,
-    "items_in_history": 0,
-    "running_item_uid": None,
-    "manager_state": "idle",
-    "queue_stop_pending": False,
-    "worker_environment_exists": False,
-    "worker_environment_state": "closed",
-    "worker_background_tasks": 0,
-    "re_state": None,
-    "pause_pending": False,
-    "run_list_uid": "4f2d48cc-980d-4472-b62b-6686caeb3833",
-    "plan_queue_uid": "2b99ccd8-f69b-4a44-82d0-947d32c5d0a2",
-    "plan_history_uid": "9af8e898-0f00-4e7a-8d97-0964c8d43f47",
-    "devices_existing_uid": "51d8b88d-7457-42c4-b67f-097b168be96d",
-    "plans_existing_uid": "65f11f60-0049-46f5-9eb3-9f1589c4a6dd",
-    "devices_allowed_uid": "a5ddff29-917c-462e-ba66-399777d2442a",
-    "plans_allowed_uid": "d1e907cd-cb92-4d68-baab-fe195754827e",
-    "plan_queue_mode": {"loop": False},
-    "task_results_uid": "159e1820-32be-4e01-ab03-e3478d12d288",
-    "lock_info_uid": "c7fe6f73-91fc-457d-8db0-dfcecb2f2aba",
-    "lock": {"environment": False, "queue": False},
+# holds a global QApplication instance created in the qapp fixture; keeping
+# this reference alive avoids it being garbage collected too early
+_ffapp_instance = None
+
+
+# Tiled data to use for testing
+# Some mocked test data
+run1 = pd.DataFrame(
+    {
+        "energy_energy": np.linspace(8300, 8400, num=100),
+        "It_net_counts": np.abs(np.sin(np.linspace(0, 4 * np.pi, num=100))),
+        "I0_net_counts": np.linspace(1, 2, num=100),
+    }
+).to_xarray()
+
+grid_scan = pd.DataFrame(
+    {
+        'CdnIPreKb': np.linspace(0, 104, num=105),
+        "It_net_counts": np.linspace(0, 104, num=105),
+        "aerotech_horiz": np.linspace(0, 104, num=105),
+        "aerotech_vert": np.linspace(0, 104, num=105),
+    }
+).to_xarray()
+
+hints = {
+    "energy": {"fields": ["energy_energy", "energy_id_energy_readback"]},
+}
+
+bluesky_mapping = {
+    "7d1daf1d-60c7-4aa7-a668-d1cd97e5335f": MapAdapter(
+        {
+            "primary": MapAdapter(
+                {
+                    "data": DatasetAdapter.from_dataset(run1),
+                },
+                metadata={"descriptors": [{"hints": hints}]},
+            ),
+        },
+        metadata={
+            "plan_name": "xafs_scan",
+            "start": {
+                "plan_name": "xafs_scan",
+                "uid": "7d1daf1d-60c7-4aa7-a668-d1cd97e5335f",
+                "hints": {"dimensions": [[["energy_energy"], "primary"]]},
+            },
+        },
+    ),
+    "9d33bf66-9701-4ee3-90f4-3be730bc226c": MapAdapter(
+        {
+            "primary": MapAdapter(
+                {
+                    "data": DatasetAdapter.from_dataset(run1),
+                },
+                metadata={"descriptors": [{"hints": hints}]},
+            ),
+        },
+        metadata={
+            "start": {
+                "plan_name": "rel_scan",
+                "uid": "9d33bf66-9701-4ee3-90f4-3be730bc226c",
+                "hints": {"dimensions": [[["pitch2"], "primary"]]},
+            }
+        },
+    ),
+    # 2D grid scan map data
+    "85573831-f4b4-4f64-b613-a6007bf03a8d": MapAdapter(
+        {
+            "primary": MapAdapter(
+                {
+                    "data": DatasetAdapter.from_dataset(grid_scan),
+                }, metadata={
+                    "descriptors": [{"hints": {'Ipreslit': {'fields': ['Ipreslit_net_counts']},
+                                               'CdnIPreKb': {'fields': ['CdnIPreKb_net_counts']},
+                                               'I0': {'fields': ['I0_net_counts']},
+                                               'CdnIt': {'fields': ['CdnIt_net_counts']},
+                                               'aerotech_vert': {'fields': ['aerotech_vert']},
+                                               'aerotech_horiz': {'fields': ['aerotech_horiz']},
+                                               'Ipre_KB': {'fields': ['Ipre_KB_net_counts']},
+                                               'CdnI0': {'fields': ['CdnI0_net_counts']},
+                                               'It': {'fields': ['It_net_counts']}}}]
+                }),
+        },
+        metadata={
+            "start": {
+                "plan_name": "grid_scan",
+                "uid": "85573831-f4b4-4f64-b613-a6007bf03a8d",
+                "hints": {
+                    'dimensions': [[['aerotech_vert'], 'primary'],
+                                   [['aerotech_horiz'], 'primary']],
+                    'gridding': 'rectilinear'
+                },
+                "shape": [5, 21],
+                "extents": [[-80, 80], [-100, 100]],
+            },
+        },
+    ),
 }
 
 
+mapping = {
+    "255id_testing": MapAdapter(bluesky_mapping),
+}
+
+tree = MapAdapter(mapping)
+
+
 @pytest.fixture(scope="session")
-def ffapp(pydm_ophyd_plugin, qapp_cls, qapp_args, pytestconfig):
-    # Get an instance of the application
-    app = qt_api.QtWidgets.QApplication.instance()
-    if app is None:
-        # New Application
-        global _ffapp_instance
-        _ffapp_instance = qapp_cls(qapp_args)
-        app = _ffapp_instance
-        name = pytestconfig.getini("qt_qapp_name")
-        app.setApplicationName(name)
-    # Make sure there's at least one Window, otherwise things get weird
-    if getattr(app, "_dummy_main_window", None) is None:
-        # Set up the actions and other boildplate stuff
-        app.setup_window_actions()
-        app.setup_runengine_actions()
-        app._dummy_main_window = FireflyMainWindow()
-    # Sanity check to make sure a QApplication was not created by mistake
-    assert isinstance(app, FireflyApplication)
-    # Yield the finalized application object
-    try:
-        yield app
-    finally:
-        if hasattr(app, "_queue_thread"):
-            app._queue_thread.quit()
-            app._queue_thread.wait(msecs=5000)
+def tiled_client():
+    app = build_app(tree)
+    with Context.from_app(app) as context:
+        client = from_context(context)
+        yield client["255id_testing"]
 
 
-# holds a global QApplication instance created in the qapp fixture; keeping
-# this reference alive avoids it being garbage collected too early
-_ffapp_instance = None
+@pytest.fixture(scope="session")
+def catalog(tiled_client):
+    return Catalog(client=tiled_client)
+
 
 
 # -----------------------------------------------------------------------------

src/firefly/application.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import subprocess
 from collections import OrderedDict
@@ -133,7 +134,7 @@ def load_instrument(self):
         # Actions for controlling the bluesky run engine
         self.setup_runengine_actions()
         # Prepare the client for interacting with the queue server
-        self.prepare_queue_client()
+        # self.prepare_queue_client()
         # Launch the default display
         show_default_window = getattr(self, f"show_{self.default_display}_window")
         default_window = show_default_window()
@@ -216,7 +217,7 @@ def setup_window_actions(self):
         self._setup_window_action(
             action_name="show_run_browser_action",
             text="Browse Runs",
-            slot=self.show_run_browser,
+            slot=self.show_run_browser_window,
         )
         # Action for launch queue-monitor
         self._setup_window_action(
@@ -529,6 +530,7 @@ def show_window(self, WindowClass, ui_file, name=None, macros={}):
         if (w := self.windows.get(name)) is None:
             # Window is not yet created, so create one
             w = self.create_window(WindowClass, ui_dir / ui_file, macros=macros)
+            # return
             self.windows[name] = w
             # Connect signals to remove the window when it closes
             w.destroyed.connect(partial(self.forget_window, name=name))
@@ -591,7 +593,7 @@ def show_status_window(self, stylesheet_path=None):
         )
 
     @QtCore.Slot()
-    def show_run_browser(self):
+    def show_run_browser_window(self):
         return self.show_window(
             PlanMainWindow, ui_dir / "run_browser.py", name="run_browser"
         )