Basic simulation for Circle Sight

field_friend/automations/automation_watcher.py

@@ -53,8 +53,9 @@ def pause(self, reason: str) -> None:
             if self.path_recorder.state == 'recording':
                 return
             else:
-                self.log.info(f'pausing automation because {reason}')
-                self.automator.pause(because=f'{reason})')
+                if self.automator.is_running:
+                    self.log.info(f'pausing automation because {reason}')
+                    self.automator.pause(because=f'{reason})')
                 return
         if reason.startswith('GNSS') and not self.gnss_watch_active:
             return

field_friend/automations/coin_collecting.py

@@ -61,12 +61,12 @@ async def _weeding(self) -> None:
         already_explored = False
         try:
             while True:
-                self.system.plant_locator.pause()
+                self.system.plant_locator.camera = None
                 self.system.plant_provider.clear()
                 self.log.info(f'cleared crops at start {self.system.plant_provider.crops}')
                 if not self.system.is_real:
                     self.create_simulated_plants()
-                self.system.plant_locator.resume()
+                self.system.plant_locator.camera = self.system.bottom_cam
                 await rosys.sleep(2)
                 while upcoming_crop_positions := self.get_upcoming_crops():
                     try:
@@ -140,7 +140,7 @@ async def _use_implement(self, center: Point) -> None:
         if self.system.field_friend.y_axis.min_position > center.y > self.system.field_friend.y_axis.max_position:
             self.log.info('crop is not reachable with y axis')
             return
-        self.system.plant_locator.pause()
+        self.system.plant_locator.camera = None
         self.log.info('Using implement')
         await self.system.puncher.punch(center.y, angle=TORNADO_ANGLE)
-        self.system.plant_locator.resume()
+        self.system.plant_locator.camera = self.system.bottom_cam

field_friend/automations/plant_locator.py

@@ -5,8 +5,8 @@
 
 from .plant_provider import Plant, PlantProvider
 
-WEED_CATEGORY_NAME = ['coin', 'weed']
-CROP_CATEGORY_NAME = ['coin_with_hole', 'crop']
+WEED_CATEGORY_NAME = ['coin', 'weed', 'big_weed', 'thistle', 'orache', 'weedy_area', ]
+CROP_CATEGORY_NAME = ['coin_with_hole', 'crop', 'sugar_beet', 'onion', 'garlic', ]
 MINIMUM_WEED_CONFIDENCE = 0.8
 MINIMUM_CROP_CONFIDENCE = 0.75
 
@@ -18,34 +18,41 @@ class DetectorError(Exception):
 class PlantLocator:
 
     def __init__(self,
-                 camera_provider: rosys.vision.CameraProvider,
                  detector: rosys.vision.Detector,
                  plant_provider: PlantProvider,
                  odometer: rosys.driving.Odometer,
                  ) -> None:
+        """The PlantLocator manages the whole localization pipeline for plants.
+
+        It pulls images from the configured camera, 
+        triggers the plant detection and
+        adds the detected plants to the PlantProvider for future use.
+        The odometer (eg. current robot position) is used to transform the detected plants from camera coordinates to world coordinates.
+
+        You can change the used camera by setting the camera property.
+        If the camera property is None there is no localization being performed.
+        """
         self.log = logging.getLogger('field_friend.plant_detection')
-        self.camera_provider = camera_provider
+        self.camera: rosys.vision.CalibratableCamera | None = None
         self.detector = detector
         self.plant_provider = plant_provider
         self.odometer = odometer
-        self.is_paused = True
         self.weed_category_names: list[str] = WEED_CATEGORY_NAME
         self.crop_category_names: list[str] = CROP_CATEGORY_NAME
         self.minimum_weed_confidence: float = MINIMUM_WEED_CONFIDENCE
         self.minimum_crop_confidence: float = MINIMUM_CROP_CONFIDENCE
         rosys.on_repeat(self._detect_plants, 0.01)  # as fast as possible, function will sleep if necessary
 
     async def _detect_plants(self) -> None:
-        if self.is_paused:
+        if self.camera is None:
             await asyncio.sleep(0.01)
             return
         t = rosys.time()
-        camera = next((camera for camera in self.camera_provider.cameras.values() if camera.is_connected), None)
-        if not camera:
+        if not self.camera:
             raise DetectorError()
-        if camera.calibration is None:
+        if self.camera.calibration is None:
             raise DetectorError()
-        new_image = camera.latest_captured_image
+        new_image = self.camera.latest_captured_image
         if new_image is None or new_image.detections:
             await asyncio.sleep(0.01)
             return
@@ -61,7 +68,7 @@ async def _detect_plants(self) -> None:
             if d.category_name in self.weed_category_names and d.confidence >= self.minimum_weed_confidence:
                 # self.log.info('weed found')
                 image_point = rosys.geometry.Point(x=d.cx, y=d.cy)
-                floor_point = camera.calibration.project_from_image(image_point)
+                floor_point = self.camera.calibration.project_from_image(image_point)
                 if floor_point is None:
                     self.log.error('could not generate floor point of detection, calibration error')
                     continue
@@ -71,7 +78,7 @@ async def _detect_plants(self) -> None:
             elif d.category_name in self.crop_category_names and d.confidence >= self.minimum_crop_confidence:
                 # self.log.info('crop found')
                 image_point = rosys.geometry.Point(x=d.cx, y=d.cy)
-                floor_point = camera.calibration.project_from_image(image_point)
+                floor_point = self.camera.calibration.project_from_image(image_point)
                 if floor_point is None:
                     self.log.error('could not generate floor point of detection, calibration error')
                     continue
@@ -83,11 +90,3 @@ async def _detect_plants(self) -> None:
                 self.log.info(f'{d.category_name} not in categories')
             # else:
             #     self.log.info(f'confidence of {d.category_name} to low: {d.confidence}')
-
-    def pause(self) -> None:
-        self.log.info('pausing plant detection')
-        self.is_paused = True
-
-    def resume(self) -> None:
-        self.log.info('resuming plant detection')
-        self.is_paused = False

field_friend/automations/weeding.py

@@ -344,7 +344,7 @@ async def _weeding(self):
         finally:
             self.kpi_provider.increment_weeding_kpi('weeding_completed')
             await self.system.field_friend.stop()
-            self.system.plant_locator.pause()
+            self.system.plant_locator.camera = None
             self.system.automation_watcher.stop_field_watch()
             self.system.automation_watcher.gnss_watch_active = False
 
@@ -373,13 +373,13 @@ async def _weed_with_plan(self):
             self.system.driver.parameters.can_drive_backwards = False
             self.system.driver.parameters.minimum_turning_radius = 0.02
             self.current_row = self.sorted_weeding_rows[i]
-            self.system.plant_locator.pause()
+            self.system.plant_locator.camera = None
             self.system.plant_provider.clear()
             if self.system.field_friend.tool != 'none':
                 await self.system.puncher.clear_view()
             await self.system.field_friend.flashlight.turn_on()
             await rosys.sleep(3)
-            self.system.plant_locator.resume()
+            self.system.plant_locator.camera = self.system.bottom_cam
             await rosys.sleep(3)
             for j, segment in enumerate(path):
                 if self.continue_canceled_weeding and self.current_segment != segment:
@@ -409,7 +409,7 @@ async def _weed_with_plan(self):
                             self.row_segment_completed = True
 
             await self.system.field_friend.flashlight.turn_off()
-            self.system.plant_locator.pause()
+            self.system.plant_locator.camera = None
             if i < len(self.weeding_plan) - 1:
                 self.system.driver.parameters.can_drive_backwards = True
                 self.log.info('Driving to next row...')
@@ -429,7 +429,7 @@ async def _weed_with_plan(self):
     async def _weed_planless(self):
         already_explored_count = 0
         while True:
-            self.system.plant_locator.pause()
+            self.system.plant_locator.camera = None
             self.system.plant_provider.clear()
             if not self.system.is_real:
                 self.system.detector.simulated_objects = []
@@ -438,7 +438,7 @@ async def _weed_planless(self):
                 await self.system.puncher.clear_view()
             await self.system.field_friend.flashlight.turn_on()
             await rosys.sleep(2)
-            self.system.plant_locator.resume()
+            self.system.plant_locator.camera = self.system.bottom_cam
             await rosys.sleep(0.5)
             await self._get_upcoming_plants()
             while self.crops_to_handle or self.weeds_to_handle:
@@ -576,7 +576,7 @@ async def _monitor_workflow(self) -> None:
                 # do not steer while advancing on a crop
                 target = self.system.odometer.prediction.transform(Point(x=closest_crop_position.x, y=0))
                 await self.system.driver.drive_to(target)
-                self.system.plant_locator.resume()
+                self.system.plant_locator.camera = self.system.bottom_cam
             else:
                 if self.crops_to_handle:
                     await self._follow_line_of_crops()

field_friend/interface/components/field_friend_object.py

@@ -9,7 +9,9 @@
 
 class field_friend_object(robot_object):
 
-    def __init__(self, odometer: Odometer, camera_provider: CameraProvider,
+    def __init__(self, odometer: Odometer,
+                 internal_camera_provider: CameraProvider,
+                 circle_sight_provider: CameraProvider,
                  field_friend: FieldFriend) -> None:
         super().__init__(Prism(outline=[], height=0), odometer)
 
@@ -18,7 +20,9 @@ def __init__(self, odometer: Odometer, camera_provider: CameraProvider,
 
         self.with_stl('assets/field_friend.stl', x=-0.365, y=-0.3, z=0.06, scale=0.001, color='#6E93D6', opacity=0.7)
         with self:
-            camera_objects(camera_provider, CameraProjector(camera_provider))
+            camera_objects(internal_camera_provider, CameraProjector(internal_camera_provider))
+            if False:  # TODO enable when mjpeg cameras are calibratable
+                camera_objects(circle_sight_provider, CameraProjector(circle_sight_provider))
             if isinstance(self.robot.y_axis, YAxis):
                 with Group() as self.tool:
                     Box(0.015, 0.015, 0.35).material('#4488ff').move(z=0.4)

field_friend/interface/components/monitoring.py

@@ -31,7 +31,8 @@ def __init__(self,
                  field_friend: FieldFriend,
                  system: 'System',
                  *,
-                 shrink_factor: int = 1) -> None:
+                 shrink_factor: int = 1,
+                 ) -> None:
         self.log = logging.getLogger('field_friend.monitoring')
         self.usb_camera_provider = usb_camera_provider
         self.mjpg_camera_provider = mjpeg_camera_provider
@@ -62,17 +63,21 @@ def __init__(self,
                 ui.label('Right').classes('text-2xl text-bold')
                 ui.image('assets/field_friend.webp').classes('w-full')
         with ui.row().classes('w-full items-stretch gap-0'):
-            with ui.card().style('background-color: #6E93D6; color: white;').classes('w-full'):
-                with ui.row().classes('w-full items-stretch'):
-                    ui.label('Person count:').classes('text-2xl text-bold').bind_text_from(self,
-                                                                                           'person_count', backward=lambda x: f'Person count: {x}')
-                    ui.label('Animal count:').classes('text-2xl text-bold').bind_text_from(self,
-                                                                                           'animal_count', backward=lambda x: f'Animal count: {x}')
+            with ui.card().classes('w-full'):
+                with ui.row().classes('w-full items-center'):
+                    ui.label('Person count:').classes('text-2xl text-bold') \
+                        .bind_text_from(self, 'person_count', backward=lambda x: f'Person count: {x}')
+                    ui.label('Animal count:').classes('text-2xl text-bold') \
+                        .bind_text_from(self, 'animal_count', backward=lambda x: f'Animal count: {x}')
                     ui.space()
-                    ui.switch('Person detection').bind_value(
-                        self, 'monitoring_active').bind_enabled_from(self.automator, 'is_running', backward=lambda x: not x)
-                    ui.switch('Plant detection').bind_value(self.plant_locator, 'is_paused', forward=lambda x: not x,
-                                                            backward=lambda x: not x).bind_enabled_from(self.automator, 'is_running', backward=lambda x: not x)
+                    ui.switch('Environment detection') \
+                        .bind_value(self, 'monitoring_active') \
+                        .bind_enabled_from(self.automator, 'is_running', backward=lambda x: not x)
+                    ui.label('Plant detection:').classes('ml-6')
+                    detection = ui.select(['bottom', 'front'], clearable=True, on_change=self._update_plant_detection) \
+                        .classes('w-36').props('dense') \
+                        .bind_enabled_from(self.automator, 'is_running', backward=lambda x: not x)
+                    detection.value = 'front'
 
         with ui.row().classes('w-full items-stretch gap-0'):
             column_classes = 'w-1/3 items-center mt-[50px]'
@@ -151,12 +156,11 @@ async def update_monitor_content(self):
         self.animal_count = animal_count
 
     async def update_bottom_view(self):
-        cameras = list(self.usb_camera_provider.cameras.values())
-        camera = next((camera for camera in cameras if camera.is_connected), None)
+        camera = self.system.bottom_cam
         if not camera:
             self.bottom_view.set_source('assets/field_friend.webp')
             return
-        image = camera.latest_captured_image if self.plant_locator and self.plant_locator.is_paused \
+        image = camera.latest_captured_image if self.plant_locator and self.plant_locator.camera is None \
             else camera.latest_detected_image
         if not image:
             return
@@ -245,3 +249,15 @@ def handle_key(self, e: events.KeyEventArguments) -> None:
                                    add='w-1/4').style(remove='border: 5px solid #6E93D6; border-radius: 5px; background-color: #6E93D6; color: white')
             self.right_view.classes(remove='hidden w-1/2',
                                     add='w-1/4').style(remove='border: 5px solid #6E93D6; border-radius: 5px; background-color: #6E93D6; color: white')
+
+    def _update_plant_detection(self, e: events.ValueChangeEventArguments) -> None:
+        match e.value:
+            case None:
+                self.plant_locator.camera = None
+            case 'bottom':
+                self.plant_locator.camera = self.system.bottom_cam
+            case 'front':
+                key = next((k for k in self.mjpg_camera_provider.cameras.keys() if CameraPosition.FRONT in k))
+                self.plant_locator.camera = self.mjpg_camera_provider.cameras[key]
+            case _:
+                self.log.warning(f'unknown camera for plant detection {e.value}')

field_friend/interface/components/robot_scene.py

@@ -1,4 +1,6 @@
 import logging
+from typing import TYPE_CHECKING
+
 import rosys
 from nicegui import events, ui
 
@@ -7,7 +9,6 @@
 from .plant_object import plant_objects
 from .visualizer_object import visualizer_object
 
-from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from field_friend.system import System
 
@@ -19,23 +20,27 @@ def __init__(self, system: 'System'):
         self.system = system
         self.scene_card = ui.card()
         self.scene_look = False
-        self.locked_view = True
+        self.locked_view = False
 
         with self.scene_card.tight().classes('w-full place-items-center').style('max-width: 100%; overflow: hidden;'):
             def toggle_lock():
                 self.locked_view = not self.locked_view
-                self.lock_view_button.props(
-                    f'flat color={"primary" if self.locked_view else "grey"}')
-            self.lock_view_button = ui.button(icon='sym_o_visibility_lock', on_click=toggle_lock).props('flat color=primary').style(
-                'position: absolute; left: 1px; top: 1px; z-index: 500;').tooltip('Lock view to robot')
+                self.lock_view_button.props(f'flat color={"primary" if self.locked_view else "grey"}')
+            self.lock_view_button = ui.button(icon='sym_o_visibility_lock', on_click=toggle_lock).props('flat color=primary') \
+                .style('position: absolute; left: 1px; top: 1px; z-index: 500;').tooltip('Lock view to robot')
 
-            with ui.scene(200, 200, on_click=self.handle_click, grid=False).classes('w-full') as self.scene:
-                field_friend_object(self.system.odometer, self.system.usb_camera_provider, self.system.field_friend)
+            with ui.scene(200, 200, on_click=self.handle_click, grid=True).classes('w-full') as self.scene:
+                field_friend_object(self.system.odometer,
+                                    self.system.usb_camera_provider,
+                                    self.system.circle_sight_provider,
+                                    self.system.field_friend)
                 rosys.driving.driver_object(self.system.driver)
-                plant_objects(self.system.plant_provider, self.system.big_weed_category_names +
-                              self.system.small_weed_category_names)
-                visualizer_object(self.system.automator, self.system.path_provider,
-                                  self.system.mowing, self.system.weeding)
+                plant_objects(self.system.plant_provider,
+                              self.system.big_weed_category_names + self.system.small_weed_category_names)
+                visualizer_object(self.system.automator,
+                                  self.system.path_provider,
+                                  self.system.mowing,
+                                  self.system.weeding)
                 field_object(self.system.field_provider)
                 self.scene.move_camera(-0.5, -1, 2)
 

field_friend/interface/pages/monitor_page.py

@@ -20,9 +20,13 @@ def page() -> None:
             self.content()
 
     def content(self) -> None:
-        # ui.query('body').classes('bg-black text-white')
-        # ui.query('.nicegui-content').classes('p-0 h-screen')
         ui.colors(primary='#6E93D6', secondary='#53B689', accent='#111B1E', positive='#53B689')
-        if self.system.is_real:
-            monitoring(self.system.usb_camera_provider, self.system.mjpeg_camera_provider,
-                       self.system.detector, self.system.monitoring_detector, self.system.plant_locator, self.system.automator, self.system.field_friend, self.system)
+        monitoring(self.system.usb_camera_provider,
+                   self.system.circle_sight_provider,
+                   self.system.detector,
+                   self.system.monitoring_detector,
+                   self.system.plant_locator,
+                   self.system.automator,
+                   self.system.field_friend,
+                   self.system,
+                   )