Humble dev fixes

.devcontainer/Dockerfile

@@ -26,7 +26,7 @@ RUN sudo apt install -y git wget python3-pip
 # Rosdep update
 RUN rosdep update
 
-RUN pip3 install -U --user pip && pip3 install --user -U numpy pillow tensorflow keras opencv-python rospkg
+RUN pip3 install -U --user pip && pip3 install --user -U numpy pillow tensorflow==2.15.0 keras keras_applications opencv-python rospkg
 
 # Source the ROS setup file
 RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> ~/.bashrc

.devcontainer/devcontainer.json

@@ -1,11 +1,26 @@
 {
   "name": "humble desktop",
   "dockerFile": "Dockerfile",
+  "features": {
+    "ghcr.io/devcontainers/features/desktop-lite:1": {}
+  },
+  "forwardPorts": [
+    6080
+  ],
+  "portsAttributes": {
+    "6080": {
+      "label": "desktop"
+    }
+  },
   //"image": "lcas.lincoln.ac.uk/lcas/devcontainer/engagement_detector:humble-dev",
   "runArgs": [
     // "--privileged",
     // "--network=host"
+    "--privileged",
+    "--shm-size=1g"
+    // "--gpus=all"
   ],
+  "postCreateCommand": "bash .devcontainer/run.sh",
   "workspaceMount": "source=${localWorkspaceFolder},target=/${localWorkspaceFolderBasename},type=bind",
   "workspaceFolder": "/${localWorkspaceFolderBasename}",
   "mounts": [

.devcontainer/run.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+
+sudo mkdir -p /ws/src && sudo chown -R vscode /ws
+sudo ln -s /engagement_detector/models /models && sudo chown -R vscode /models
+
+ln -s /engagement_detector /ws/src/
+
+source /opt/ros/${ROS_DISTRO}/setup.bash
+# # sudo apt update
+
+rosdep --rosdistro=${ROS_DISTRO} update 
+
+cd /ws
+rosdep install --from-paths src --ignore-src -r -y 
+colcon build 
+
+echo "source /ws/install/setup.bash" >> ~/.bashrc
+
+cd /engagement_detector
+mkdir -p models
+
+if [ -f models/lstm_10_50_runsigm_runsigm.h5 ]; then
+    echo "File exists"
+else
+    echo "Downloading model"
+    wget -O models/lstm_10_50_runsigm_runsigm.h5 'http://lcas.lincoln.ac.uk/owncloud/index.php/s/UHWU1g5qXU9RiAp/download'
+    sudo ln -s /models /engagement_detector/models
+fi

README.md

@@ -1,28 +1,36 @@
 # Engagement Detector
 
-ROS package to detect overall users engagement from a robot's ego camera in human-robot interactions.
+ROS 2 package to detect overall users engagement from a robot's ego camera in human-robot interactions. 
 
 The approach is detailed in the journal paper: 
 [Del Duchetto F, Baxter P and Hanheide M (2020) Are You Still With Me? Continuous Engagement Assessment From a Robot's Point of View. Front. Robot. AI 7:116. doi: 10.3389/frobt.2020.00116](https://doi.org/10.3389/frobt.2020.00116)
 
 
-## Install
+## Setup Devcontainer
 
 ### Install package with dependencies
+1. Make sure you have VSCode installed: https://code.visualstudio.com/download
+2. Make sure you have git installed: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git
+3. Make sure you have the `Docker` and the `Dev Containers` extension in VSCode installed and working: https://code.visualstudio.com/docs/containers/overview and https://code.visualstudio.com/docs/devcontainers/containers
+    * ensure docker is working, i.e. try `docker run --rm hello-world` and check it succeeds for your user
 
-1. Install python catkin util: `pip install catkin_pkg`
+### Environment
 
-2. In a terminal go into the root folder of the package: `cd engagement_detector/`
+**If you are on a Windows PC the following two additional steps are required:**
 
-3. and install: `pip install .`
+   - Open a terminal(e.g., window's powershell), type `git config --global core.autocrlf false` and press Enter
+   - Make sure docker is running by launching the docker desktop application
 
-4. Then download the keras model of the newtork: `./download_model.sh`
+Then:
+
+1. `git clone https://github.com/LCAS/engagement_detector.git`
+2. `cd engagement_detector && git checkout humble-dev`
+3. Open the folder `engagement_detector` with VSCode in the devcontainer.
 
-5. Now, you can build the package in your catkin workspace (i.e. [http://wiki.ros.org/catkin/Tutorials/create_a_workspace](http://wiki.ros.org/catkin/Tutorials/create_a_workspace)). 
 
 ## Launch
 
-`roslaunch engagement_detector engagement_detector.launch`  
+`ros2 run engagement_detector engagement_detector_node`  
 
 ### Engagement value
 
@@ -34,9 +42,20 @@ The predicted engagment value is published on the topic `/engagement_detector/va
 - `debug_image`: (default: `true`) whether to publish the out debug image
 - `out_image`: (default: `/engagement_detector/out_image`) the debug image topic
 
-## Real-time visualization of engagement
+## Webcam test in devcontainer
+
+The following command will take images from your PC's camera and publish it to the default image topic:
+```
+ros2 run engagement_detector webcam_node
+```
+
+If you want to visualise the `out_image` topic you can do so by:
+
+1. launching the devcontainer VNC window (PORTS tab > right-click on desktop (6080) > Open in Browser > insert password: `vscode`)
+2. open a new terminal and launch `rqt`
+3. select Visualization > Image View. 
 
-`rosrun image_view image_view image:=/engagement_detector/out_image` will show the camera image with the engagement serie data plotted on it. Something like this:
+## Example visualization of engagement
 
 
 Single user example            |  Multi users example 

engagement_detector/engagement_detector.py

@@ -4,19 +4,21 @@
 import numpy as np
 from PIL import Image
 
-import keras
+import tensorflow.compat.v1.keras as keras
 from keras.models import load_model
 from keras.preprocessing import image
 from keras_applications.resnext import ResNeXt50, preprocess_input
 from keras.applications.imagenet_utils import decode_predictions
-import tensorflow as tf
+# import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 
 # disable most logs
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 
 # ROS-independent class for the detector
 class EngagementDetector:
-    def __init__(self):
+    def __init__(self, test=False):
         self._this_dir_path = os.path.dirname(os.path.realpath(__file__))
 
         self.resNet = None
@@ -32,27 +34,28 @@ def __init__(self):
         keras.backend.set_session(sess)
 
         # load the networks in memory
-        self._load_networks()
+        self._load_networks(test=test)
 
 
-    def _load_networks(self):
+    def _load_networks(self, test=False):
         # load resnet
-        self.resNet = ResNeXt50(include_top=False, input_shape=(224, 224, 3),  pooling='max', weights='imagenet', backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils)
+        self.resNet = ResNeXt50(include_top=test, input_shape=(224, 224, 3),  pooling='max', weights='imagenet', backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils)
         self.resNet._make_predict_function()
         self.resNet_graph = tf.get_default_graph()
 
         ##### test resnet in detecting a dog in img
         ##### NOTE: set `include_top=True` above to test this
-        if False:
-            img_path = os.path.join(self._this_dir_path, "../../imgs/dog.png")
+        if test:
+            img_path = os.path.join(self._this_dir_path, "../imgs/dog.png")
             img = image.load_img(img_path, target_size=(224, 224))
             img = image.img_to_array(img)
             x = preprocess_input(np.expand_dims(img.copy(), axis=0), backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils)
             preds = self.resNet.predict(x)
-            print(">> TEST: dog prediction\n", decode_predictions(preds, top=5))
-
+            print("\n\n\n>> TEST: dog prediction\n", decode_predictions(preds, top=5))
+            print("\n\n\n")
+
         # load lstm
-        model_path = os.path.join(self._this_dir_path, "../../models/lstm_10_50_runsigm_runsigm.h5")
+        model_path = os.path.join(self._this_dir_path, "/models/lstm_10_50_runsigm_runsigm.h5")
         self.lstm = load_model(model_path)
         self.lstm._make_predict_function()
         self.lstm_graph = tf.get_default_graph()
@@ -81,4 +84,4 @@ def predict(self, frames_seq):
         return prediction
 
 if __name__ == "__main__":
-    det = EngagementDetector()
+    det = EngagementDetector(test=True)

engagement_detector/ros_engagement_detector.py

@@ -7,26 +7,26 @@
 from cv_bridge import CvBridge, CvBridgeError
 import cv2
 import numpy as np
-from engagement_detector import EngagementDetector
-from time_serie_in_image import TimeSerieInImage
+from engagement_detector.engagement_detector import EngagementDetector
+from engagement_detector.time_serie_in_image import TimeSerieInImage
 from threading import Lock
 
 
 class ROSEngagementDetector(Node):
 
     def __init__(self):
         super().__init__('engagement_detector')
-        image_topic = node.declare_parameter("image", "/camera/color/image_raw").value
-        plot_in_image = node.declare_parameter("debug_image", True).value
-        out_image_topic = node.declare_parameter("out_image", "engagement_detector/out_image").value
+        image_topic = self.declare_parameter("image", "/camera/color/image_raw").value
+        plot_in_image = self.declare_parameter("debug_image", True).value
+        out_image_topic = self.declare_parameter("out_image", "engagement_detector/out_image").value
 
         self.plot_in_image = plot_in_image
         self.bridge = CvBridge()
         self.detector = EngagementDetector()
 
         self.subscription = self.create_subscription(
             Image,
-            image_topic,
+            "/camera/color/image_raw",
             self._img_cb,
             10
         )
@@ -68,6 +68,10 @@ def _img_cb(self, msg):
             self.outImg_pub.publish(out_imgmsg)
 
     def timed_cb(self):
+        if len(self.image_seq) < 10:
+            self.get_logger().warn("Waiting to receive images...")
+            return
+
         self.sequence_lock.acquire()
         tmp_image_seq = self.image_seq
         self.sequence_lock.release()
@@ -76,16 +80,13 @@ def timed_cb(self):
             self.get_logger().warn("Could not make a prediction, probably the frame sequence length is not 10.")
             return
 
-        value = np.squeeze(prediction)
-        self.eng_pub.publish(value)
+        value = np.squeeze(prediction)[()].item()
+        fmsg = Float32()
+        fmsg.data = value
+        self.eng_pub.publish(fmsg)
         self.last_value = value
 
     def spin(self, hz=10):
-        self.get_logger().info("Waiting to receive images...")
-        while len(self.image_seq) < 10 and rclpy.ok():
-            rclpy.sleep(0.2)
-        self.get_logger().info("DONE")
-
         timer = self.create_timer(1.0 / hz, self.timed_cb)
 
         rclpy.spin(self)

engagement_detector/time_serie_in_image.py

@@ -19,6 +19,8 @@ def __init__(self):
         self.pre_color = (0, 0, 255, 255)
         self.post_color = (0, 0, 100, 255)
 
+        self.font = ImageFont.load_default()
+
 
     def step(self, frame, value):
         img = Image.fromarray(frame)
@@ -39,8 +41,8 @@ def step(self, frame, value):
         grid_draw.line([(0, plot_offset), (width, plot_offset)], fill=(255, 255, 255), width=2)
         grid_draw.line([(0, plot_offset + plot_max_height), (width, plot_offset + plot_max_height)], fill=(255, 255, 255), width=2)
         grid_draw.line([(width/2., plot_offset - 10), (width/2., plot_offset + plot_max_height + 1)], fill=(200, 200, 200), width=2)
-        grid_draw.text((0, plot_offset - 11), "1.0", font=ImageFont.truetype("arial", 10), fill=(255, 255, 255))
-        grid_draw.text((0, plot_offset + plot_max_height - 11), "0.0", font=ImageFont.truetype("arial", 10), fill=(255, 255, 255))
+        grid_draw.text((0, plot_offset - 11), "1.0", font=self.font, fill=(255, 255, 255))
+        grid_draw.text((0, plot_offset + plot_max_height - 11), "0.0", font=self.font, fill=(255, 255, 255))
 
         # plot the series
         y_pre  = self.data_serie[0:min(self.num_points_to_plot, self.data_serie.size)]
@@ -61,6 +63,6 @@ def step(self, frame, value):
         plot_draw = ImageDraw.Draw(img)
         plot_draw.line(xy_pre, fill=self.pre_color, width=self.line_width, joint="curve")
         plot_draw.line(xy_post, fill=self.post_color, width=self.line_width, joint="curve")
-        plot_draw.text((width/2., plot_offset - 20  ), "%.2f" % value, font=ImageFont.truetype("arial", 16), fill=self.pre_color)
+        plot_draw.text((width/2., plot_offset - 20  ), "%.2f" % value, font=self.font, fill=self.pre_color)
 
         return np.array(img)

engagement_detector/webcam.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+import rclpy
+from rclpy.node import Node
+import cv2
+from sensor_msgs.msg import Image
+from cv_bridge import CvBridge, CvBridgeError
+
+class ImagePublisher(Node):
+    def __init__(self):
+        super().__init__("image_publisher")
+        self.bridge = CvBridge()
+        self.cap = cv2.VideoCapture(0)
+        self.bgr8pub = self.create_publisher(Image, "/camera/color/image_raw", 10)
+
+    def run(self):
+        while True:
+            try:
+                r, frame = self.cap.read()
+                if not r:
+                    return
+                # BGR8
+                self.bgr8pub.publish(self.bridge.cv2_to_imgmsg(frame, "bgr8"))
+
+            except CvBridgeError as e:
+                print(e)
+        self.cap.release()
+
+def main(args=None):
+    rclpy.init(args=args)
+
+    ip = ImagePublisher()
+    print("Publishing...")
+    ip.run()
+
+    ip.destroy_node()
+    rclpy.shutdown()
+
+if __name__ == '__main__':
+    main()

package.xml

@@ -15,13 +15,11 @@
   <test_depend>ament_flake8</test_depend>
   <test_depend>ament_pep257</test_depend>
   <test_depend>python3-pytest</test_depend>
-  <test_depend>launch_pytest</test_depend>
 
-  <depend>python-numpy</depend>
-  <depend>python-imaging</depend>
-  <depend>python-tensorflow-gpu-pip</depend>
+  <depend>python3-numpy</depend>
+  <depend>python-tensorflow-pip</depend>
   <depend>python-keras-pip</depend>
-  <depend>python-opencv</depend>
+  <depend>python3-opencv</depend>
   <depend>std_msgs</depend>
   <depend>sensor_msgs</depend>
   <depend>rclpy</depend>

setup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-
+from os import path
+from glob import glob
 from setuptools import setup
 from setuptools import find_packages
 
@@ -13,6 +14,8 @@
         ('share/ament_index/resource_index/packages',
             ['resource/' + package_name]),
         ('share/' + package_name, ['package.xml']),
+        (path.join('share', package_name), glob(path.join('engagement_detector', '*.py'))),
+        # (path.join('share', "models"), glob(path.join('models', '*.h5'))),
     ],
     install_requires=['setuptools'],
     zip_safe=True,
@@ -28,7 +31,8 @@
     tests_require=['pytest'],
     entry_points={
         'console_scripts': [
-            'engagement_detector_node = engagement_detector.ros_engagement_detector:main'
+            'engagement_detector_node = engagement_detector.ros_engagement_detector:main',
+            'webcam_node = engagement_detector.webcam:main',
         ],
     },
 )