Merge pull request #287 from scipp/detector-data

Add "detector data" service

.gitignore

@@ -41,6 +41,7 @@ docs/generated/
 *.ort
 
 # Test outputs
+*.png
 .coverage
 coverate_html
 .benchmarks/

docker-compose-beamlime.yml

@@ -65,11 +65,11 @@ services:
       - ./scripts:/scripts
     command: ["sh", "/scripts/setup-kafka-topics.sh", "${BEAMLIME_INSTRUMENT:-dummy}"]
 
-  fake-producer:
+  fake-monitors:
     profiles:
-      - dev  # Only run fake producer with `docker compose --profile dev up`
+      - monitor  # Only run fake producer with `docker compose --profile monitor up`
     image: python:3.12
-    container_name: fake-producer
+    container_name: fake-monitors
     depends_on:
       init-kafka:
         condition: service_completed_successfully
@@ -86,9 +86,34 @@ services:
       KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-SCRAM-SHA-256}
       KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-DUMMY}
       KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-DUMMY}
-    command: sh -c "python -m pip install -e . && python -m beamlime.services.fake_producer --mode ev44 --instrument $BEAMLIME_INSTRUMENT"
+    command: sh -c "python -m pip install -e . && python -m beamlime.services.fake_monitors --mode ev44 --instrument $BEAMLIME_INSTRUMENT"
+
+  fake-detectors:
+    profiles:
+      - detector
+    image: python:3.12
+    container_name: fake-detectors
+    depends_on:
+      init-kafka:
+        condition: service_completed_successfully
+      kafka:
+        condition: service_healthy
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      BEAMLIME_ENV: docker
+      BEAMLIME_INSTRUMENT: ${BEAMLIME_INSTRUMENT:-dummy}
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:29092}
+      KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+      KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-SCRAM-SHA-256}
+      KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-DUMMY}
+      KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-DUMMY}
+    command: sh -c "python -m pip install -e . && python -m beamlime.services.fake_detectors --instrument $BEAMLIME_INSTRUMENT"
 
   monitor-data:
+    profiles:
+      - monitor
     image: python:3.12
     container_name: monitor-data
     depends_on:
@@ -114,7 +139,37 @@ services:
       KAFKA2_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-DUMMY}
     command: sh -c "python -m pip install -e . && python -m beamlime.services.monitor_data --instrument $BEAMLIME_INSTRUMENT"
 
+  detector-data:
+    profiles:
+      - detector
+    image: python:3.12
+    container_name: detector-data
+    depends_on:
+      init-kafka:
+        condition: service_completed_successfully
+      kafka:
+        condition: service_healthy
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      BEAMLIME_ENV: docker
+      BEAMLIME_INSTRUMENT: ${BEAMLIME_INSTRUMENT:-dummy}
+      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:29092}
+      KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+      KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-SCRAM-SHA-256}
+      KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-DUMMY}
+      KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-DUMMY}
+      KAFKA2_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:29092}
+      KAFKA2_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
+      KAFKA2_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-SCRAM-SHA-256}
+      KAFKA2_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-DUMMY}
+      KAFKA2_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-DUMMY}
+    command: sh -c "python -m pip install -e . && python -m beamlime.services.detector_data --instrument $BEAMLIME_INSTRUMENT"
+
   dashboard:
+    profiles:
+      - monitor
     image: python:3.12
     container_name: dashboard
     depends_on:

docs/user-guide/services.md

@@ -7,10 +7,10 @@ Configuration is in `beamlime.config.defaults`.
 By default the files with the `dev` suffix are used.
 Set `BEAMLIME_ENV` to, e.g., `staging` to use the `staging` configuration.
 
-For a local demo, run the fake producer:
+For a local demo, run the fake monitor data producer:
 
 ```sh
-python -m beamlime.services.fake_producer --mode ev44 --instrument dummy
+python -m beamlime.services.fake_monitors --mode ev44 --instrument dummy
 ```
 
 Run the monitor data histogramming and accumulation service:
@@ -27,27 +27,43 @@ BEAMLIME_INSTRUMENT=dummy gunicorn beamlime.services.wsgi:application
 
 Navigate to `http://localhost:8000` to see the dashboard.
 
+
+For a local demo, run the fake detector data producer:
+
+```sh
+python -m beamlime.services.fake_detectors --instrument dummy
+```
+
+Run the detector data histogramming and accumulation service:
+
+```sh
+python -m beamlime.services.detector_data --instrument dummy --sink png
+```
+
+Note the `--sink png` argument, which will save the detector data histograms as PNG files in the `data` directory.
+The Dashboard does not support detector data yet.
+
 ## Running the services using Docker
 
 You can also run all the services using Docker.
-Use the provided `docker-compose-beamlime.yml` file to start the services:
+Use the provided `docker-compose-beamlime.yml` file to start Kafka:
 
 ```sh
 BEAMLIME_INSTRUMENT=dummy docker-compose -f docker-compose-beamlime.yml up
 ```
 
-This will start the Zookeeper, Kafka broker, and all the necessary services for the monitor data dashboard.
-It will take a minute or two for the services to start fully.
-Navigate to `http://localhost:8000` to see the dashboard.
-
-### Development Profile
+This will start the Zookeeper, Kafka broker.
+This can be then used with the services run manually as described above.
 
-To run the fake monitor data producer for local development, use the `dev` profile:
+Alternatively, the `monitor` or `detector` profile can be used to start the respective services in Docker:
 
 ```sh
-BEAMLIME_INSTRUMENT=dummy docker-compose --profile dev -f docker-compose-beamlime.yml up
+BEAMLIME_INSTRUMENT=dummy docker-compose --profile monitor -f docker-compose-beamlime.yml up
 ```
 
+It will take a minute or two for the services to start fully.
+Navigate to `http://localhost:8000` to see the dashboard.
+
 ### Kafka Configuration
 
 The services can be configured to connect to different Kafka brokers using environment variables. There may be two distinct Kafka brokers: one upstream with raw data and one downstream for processed data and Beamlime control.

scripts/setup-kafka-topics.sh

@@ -11,6 +11,16 @@ BEAMLIME_INSTRUMENT=$1
 kafka-topics --bootstrap-server kafka:29092 --delete --topic '.*' || true
 kafka-leader-election --bootstrap-server kafka:29092 --election-type PREFERRED --all-topic-partitions
 
+kafka-topics --create --bootstrap-server kafka:29092 \
+  --topic ${BEAMLIME_INSTRUMENT}_detector \
+  --config cleanup.policy=delete \
+  --config delete.retention.ms=60000 \
+  --config max.message.bytes=104857600 \
+  --config retention.bytes=10737418240 \
+  --config retention.ms=30000 \
+  --config segment.bytes=104857600 \
+  --config segment.ms=60000
+
 kafka-topics --create --bootstrap-server kafka:29092 \
   --topic ${BEAMLIME_INSTRUMENT}_beam_monitor \
   --config cleanup.policy=delete \
@@ -22,7 +32,7 @@ kafka-topics --create --bootstrap-server kafka:29092 \
   --config segment.ms=60000
 
 kafka-topics --create --bootstrap-server kafka:29092 \
-  --topic ${BEAMLIME_INSTRUMENT}_beamlime_monitor_data_control \
+  --topic ${BEAMLIME_INSTRUMENT}_beamlime_control \
   --config cleanup.policy=compact \
   --config retention.ms=-1 \
   --config min.compaction.lag.ms=86400000 \

src/beamlime/__init__.py

@@ -12,12 +12,12 @@
 del importlib
 
 from .core import (
+    CommonHandlerFactory,
     ConfigSubscriber,
     Service,
     ServiceBase,
     Processor,
     Handler,
-    HandlerRegistry,
     StreamProcessor,
     MessageKey,
     MessageSource,
@@ -29,9 +29,9 @@
 
 __all__ = [
     'compact_messages',
+    'CommonHandlerFactory',
     'ConfigSubscriber',
     'Handler',
-    'HandlerRegistry',
     'LiveWorkflow',
     'Message',
     'MessageKey',

src/beamlime/config/config_names.py

@@ -5,6 +5,7 @@
 """
 
 control_consumer = 'control_consumer'
+detector_data = 'detector_data'
 kafka_downstream = 'kafka_downstream'
 kafka_upstream = 'kafka_upstream'
 monitor_data = 'monitor_data'

src/beamlime/config/defaults/detector_data.yaml

@@ -0,0 +1,2 @@
+topics:
+  - detector

src/beamlime/config/raw_detectors/__init__.py

@@ -7,8 +7,14 @@
 they can be moved to a separate file format in the future.
 """
 
+from .dummy import dummy_detectors_config
 from .dream import dream_detectors_config
 from .loki import loki_detectors_config
 from .nmx import nmx_detectors_config
 
-__all__ = [dream_detectors_config, loki_detectors_config, nmx_detectors_config]
+__all__ = [
+    'dummy_detectors_config',
+    'dream_detectors_config',
+    'loki_detectors_config',
+    'nmx_detectors_config',
+]

src/beamlime/config/raw_detectors/dream.py

@@ -5,6 +5,7 @@
 # The other DREAM detectors have non-consecutive detector numbers. This is not
 # supported currently
 
+import scipp as sc
 from ess.reduce.live import raw
 
 _res_scale = 8
@@ -36,6 +37,7 @@
         # a projection.
         'mantle_front_layer': {
             'detector_name': 'mantle_detector',
+            'detector_number': sc.arange('dummy', 229377, 720897, unit=None),
             'projection': raw.LogicalView(
                 fold={
                     'wire': 32,

src/beamlime/config/raw_detectors/dummy.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
+"""
+Detector configuration for a dummy instrument used for development and testing.
+"""
+
+import scipp as sc
+from ess.reduce.live import raw
+
+dummy_detectors_config = {
+    'dashboard': {'nrow': 1, 'ncol': 3},
+    'detectors': {
+        'Panel 0': {
+            'detector_name': 'panel_0',
+            'detector_number': sc.arange('yx', 0, 128**2, unit=None).fold(
+                dim='yx', sizes={'y': -1, 'x': 128}
+            ),
+            'gridspec': (0, 0),
+            'projection': raw.LogicalView(),
+        },
+    },
+}

src/beamlime/core/__init__.py

@@ -2,15 +2,15 @@
 # Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
 from .service import Service, ServiceBase
 from .processor import Processor, StreamProcessor
-from .handler import Handler, HandlerRegistry
+from .handler import CommonHandlerFactory, Handler
 from .message import Message, compact_messages, MessageKey, MessageSource, MessageSink
 from .config_subscriber import ConfigSubscriber
 
 __all__ = [
     'compact_messages',
+    'CommonHandlerFactory',
     'ConfigSubscriber',
     'Handler',
-    'HandlerRegistry',
     'Message',
     'MessageKey',
     'MessageSink',