diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b1036ae..e3e6bc8 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -26,7 +26,7 @@ RUN sudo apt install -y git wget python3-pip # Rosdep update RUN rosdep update -RUN pip3 install -U --user pip && pip3 install --user -U numpy pillow tensorflow keras opencv-python rospkg +RUN pip3 install -U --user pip && pip3 install --user -U numpy pillow tensorflow==2.15.0 keras keras_applications opencv-python rospkg # Source the ROS setup file RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> ~/.bashrc diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7f466ae..c3fca73 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,11 +1,26 @@ { "name": "humble desktop", "dockerFile": "Dockerfile", + "features": { + "ghcr.io/devcontainers/features/desktop-lite:1": {} + }, + "forwardPorts": [ + 6080 + ], + "portsAttributes": { + "6080": { + "label": "desktop" + } + }, //"image": "lcas.lincoln.ac.uk/lcas/devcontainer/engagement_detector:humble-dev", "runArgs": [ // "--privileged", // "--network=host" + "--privileged", + "--shm-size=1g" + // "--gpus=all" ], + "postCreateCommand": "bash .devcontainer/run.sh", "workspaceMount": "source=${localWorkspaceFolder},target=/${localWorkspaceFolderBasename},type=bind", "workspaceFolder": "/${localWorkspaceFolderBasename}", "mounts": [ diff --git a/.devcontainer/run.sh b/.devcontainer/run.sh new file mode 100644 index 0000000..f3a1d60 --- /dev/null +++ b/.devcontainer/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + + +sudo mkdir -p /ws/src && sudo chown -R vscode /ws +sudo ln -s /engagement_detector/models /models && sudo chown -R vscode /models + +ln -s /engagement_detector /ws/src/ + +source /opt/ros/${ROS_DISTRO}/setup.bash +# # sudo apt update + +rosdep --rosdistro=${ROS_DISTRO} update + +cd /ws +rosdep install --from-paths src --ignore-src -r -y +colcon build + +echo "source /ws/install/setup.bash" >> ~/.bashrc + +cd /engagement_detector +mkdir -p models + +if [ -f models/lstm_10_50_runsigm_runsigm.h5 ]; then + echo "File exists" +else + echo "Downloading model" + wget -O models/lstm_10_50_runsigm_runsigm.h5 'http://lcas.lincoln.ac.uk/owncloud/index.php/s/UHWU1g5qXU9RiAp/download' + sudo ln -s /models /engagement_detector/models +fi diff --git a/README.md b/README.md index a8009bc..39add2a 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,36 @@ # Engagement Detector -ROS package to detect overall users engagement from a robot's ego camera in human-robot interactions. +ROS 2 package to detect overall users engagement from a robot's ego camera in human-robot interactions. The approach is detailed in the journal paper: [Del Duchetto F, Baxter P and Hanheide M (2020) Are You Still With Me? Continuous Engagement Assessment From a Robot's Point of View. Front. Robot. AI 7:116. doi: 10.3389/frobt.2020.00116](https://doi.org/10.3389/frobt.2020.00116) -## Install +## Setup Devcontainer ### Install package with dependencies +1. Make sure you have VSCode installed: https://code.visualstudio.com/download +2. Make sure you have git installed: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git +3. Make sure you have the `Docker` and the `Dev Containers` extension in VSCode installed and working: https://code.visualstudio.com/docs/containers/overview and https://code.visualstudio.com/docs/devcontainers/containers + * ensure docker is working, i.e. try `docker run --rm hello-world` and check it succeeds for your user -1. Install python catkin util: `pip install catkin_pkg` +### Environment -2. In a terminal go into the root folder of the package: `cd engagement_detector/` +**If you are on a Windows PC the following two additional steps are required:** -3. and install: `pip install .` + - Open a terminal(e.g., window's powershell), type `git config --global core.autocrlf false` and press Enter + - Make sure docker is running by launching the docker desktop application -4. Then download the keras model of the newtork: `./download_model.sh` +Then: + +1. `git clone https://github.com/LCAS/engagement_detector.git` +2. `cd engagement_detector && git checkout humble-dev` +3. Open the folder `engagement_detector` with VSCode in the devcontainer. -5. Now, you can build the package in your catkin workspace (i.e. [http://wiki.ros.org/catkin/Tutorials/create_a_workspace](http://wiki.ros.org/catkin/Tutorials/create_a_workspace)). ## Launch -`roslaunch engagement_detector engagement_detector.launch` +`ros2 run engagement_detector engagement_detector_node` ### Engagement value @@ -34,9 +42,20 @@ The predicted engagment value is published on the topic `/engagement_detector/va - `debug_image`: (default: `true`) whether to publish the out debug image - `out_image`: (default: `/engagement_detector/out_image`) the debug image topic -## Real-time visualization of engagement +## Webcam test in devcontainer + +The following command will take images from your PC's camera and publish it to the default image topic: +``` +ros2 run engagement_detector webcam_node +``` + +If you want to visualise the `out_image` topic you can do so by: + +1. launching the devcontainer VNC window (PORTS tab > right-click on desktop (6080) > Open in Browser > insert password: `vscode`) +2. open a new terminal and launch `rqt` +3. select Visualization > Image View. -`rosrun image_view image_view image:=/engagement_detector/out_image` will show the camera image with the engagement serie data plotted on it. Something like this: +## Example visualization of engagement Single user example | Multi users example diff --git a/engagement_detector/engagement_detector.py b/engagement_detector/engagement_detector.py index 1e4614b..4610140 100755 --- a/engagement_detector/engagement_detector.py +++ b/engagement_detector/engagement_detector.py @@ -4,19 +4,21 @@ import numpy as np from PIL import Image -import keras +import tensorflow.compat.v1.keras as keras from keras.models import load_model from keras.preprocessing import image from keras_applications.resnext import ResNeXt50, preprocess_input from keras.applications.imagenet_utils import decode_predictions -import tensorflow as tf +# import tensorflow as tf +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() # disable most logs os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # ROS-independent class for the detector class EngagementDetector: - def __init__(self): + def __init__(self, test=False): self._this_dir_path = os.path.dirname(os.path.realpath(__file__)) self.resNet = None @@ -32,27 +34,28 @@ def __init__(self): keras.backend.set_session(sess) # load the networks in memory - self._load_networks() + self._load_networks(test=test) - def _load_networks(self): + def _load_networks(self, test=False): # load resnet - self.resNet = ResNeXt50(include_top=False, input_shape=(224, 224, 3), pooling='max', weights='imagenet', backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils) + self.resNet = ResNeXt50(include_top=test, input_shape=(224, 224, 3), pooling='max', weights='imagenet', backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils) self.resNet._make_predict_function() self.resNet_graph = tf.get_default_graph() ##### test resnet in detecting a dog in img ##### NOTE: set `include_top=True` above to test this - if False: - img_path = os.path.join(self._this_dir_path, "../../imgs/dog.png") + if test: + img_path = os.path.join(self._this_dir_path, "../imgs/dog.png") img = image.load_img(img_path, target_size=(224, 224)) img = image.img_to_array(img) x = preprocess_input(np.expand_dims(img.copy(), axis=0), backend = keras.backend, layers = keras.layers, models = keras.models, utils = keras.utils) preds = self.resNet.predict(x) - print(">> TEST: dog prediction\n", decode_predictions(preds, top=5)) - + print("\n\n\n>> TEST: dog prediction\n", decode_predictions(preds, top=5)) + print("\n\n\n") + # load lstm - model_path = os.path.join(self._this_dir_path, "../../models/lstm_10_50_runsigm_runsigm.h5") + model_path = os.path.join(self._this_dir_path, "/models/lstm_10_50_runsigm_runsigm.h5") self.lstm = load_model(model_path) self.lstm._make_predict_function() self.lstm_graph = tf.get_default_graph() @@ -81,4 +84,4 @@ def predict(self, frames_seq): return prediction if __name__ == "__main__": - det = EngagementDetector() \ No newline at end of file + det = EngagementDetector(test=True) \ No newline at end of file diff --git a/engagement_detector/ros_engagement_detector.py b/engagement_detector/ros_engagement_detector.py index 2d72787..8dfa8f3 100755 --- a/engagement_detector/ros_engagement_detector.py +++ b/engagement_detector/ros_engagement_detector.py @@ -7,8 +7,8 @@ from cv_bridge import CvBridge, CvBridgeError import cv2 import numpy as np -from engagement_detector import EngagementDetector -from time_serie_in_image import TimeSerieInImage +from engagement_detector.engagement_detector import EngagementDetector +from engagement_detector.time_serie_in_image import TimeSerieInImage from threading import Lock @@ -16,9 +16,9 @@ class ROSEngagementDetector(Node): def __init__(self): super().__init__('engagement_detector') - image_topic = node.declare_parameter("image", "/camera/color/image_raw").value - plot_in_image = node.declare_parameter("debug_image", True).value - out_image_topic = node.declare_parameter("out_image", "engagement_detector/out_image").value + image_topic = self.declare_parameter("image", "/camera/color/image_raw").value + plot_in_image = self.declare_parameter("debug_image", True).value + out_image_topic = self.declare_parameter("out_image", "engagement_detector/out_image").value self.plot_in_image = plot_in_image self.bridge = CvBridge() @@ -26,7 +26,7 @@ def __init__(self): self.subscription = self.create_subscription( Image, - image_topic, + "/camera/color/image_raw", self._img_cb, 10 ) @@ -68,6 +68,10 @@ def _img_cb(self, msg): self.outImg_pub.publish(out_imgmsg) def timed_cb(self): + if len(self.image_seq) < 10: + self.get_logger().warn("Waiting to receive images...") + return + self.sequence_lock.acquire() tmp_image_seq = self.image_seq self.sequence_lock.release() @@ -76,16 +80,13 @@ def timed_cb(self): self.get_logger().warn("Could not make a prediction, probably the frame sequence length is not 10.") return - value = np.squeeze(prediction) - self.eng_pub.publish(value) + value = np.squeeze(prediction)[()].item() + fmsg = Float32() + fmsg.data = value + self.eng_pub.publish(fmsg) self.last_value = value def spin(self, hz=10): - self.get_logger().info("Waiting to receive images...") - while len(self.image_seq) < 10 and rclpy.ok(): - rclpy.sleep(0.2) - self.get_logger().info("DONE") - timer = self.create_timer(1.0 / hz, self.timed_cb) rclpy.spin(self) diff --git a/engagement_detector/time_serie_in_image.py b/engagement_detector/time_serie_in_image.py index 893d426..3248c99 100755 --- a/engagement_detector/time_serie_in_image.py +++ b/engagement_detector/time_serie_in_image.py @@ -19,6 +19,8 @@ def __init__(self): self.pre_color = (0, 0, 255, 255) self.post_color = (0, 0, 100, 255) + self.font = ImageFont.load_default() + def step(self, frame, value): img = Image.fromarray(frame) @@ -39,8 +41,8 @@ def step(self, frame, value): grid_draw.line([(0, plot_offset), (width, plot_offset)], fill=(255, 255, 255), width=2) grid_draw.line([(0, plot_offset + plot_max_height), (width, plot_offset + plot_max_height)], fill=(255, 255, 255), width=2) grid_draw.line([(width/2., plot_offset - 10), (width/2., plot_offset + plot_max_height + 1)], fill=(200, 200, 200), width=2) - grid_draw.text((0, plot_offset - 11), "1.0", font=ImageFont.truetype("arial", 10), fill=(255, 255, 255)) - grid_draw.text((0, plot_offset + plot_max_height - 11), "0.0", font=ImageFont.truetype("arial", 10), fill=(255, 255, 255)) + grid_draw.text((0, plot_offset - 11), "1.0", font=self.font, fill=(255, 255, 255)) + grid_draw.text((0, plot_offset + plot_max_height - 11), "0.0", font=self.font, fill=(255, 255, 255)) # plot the series y_pre = self.data_serie[0:min(self.num_points_to_plot, self.data_serie.size)] @@ -61,6 +63,6 @@ def step(self, frame, value): plot_draw = ImageDraw.Draw(img) plot_draw.line(xy_pre, fill=self.pre_color, width=self.line_width, joint="curve") plot_draw.line(xy_post, fill=self.post_color, width=self.line_width, joint="curve") - plot_draw.text((width/2., plot_offset - 20 ), "%.2f" % value, font=ImageFont.truetype("arial", 16), fill=self.pre_color) + plot_draw.text((width/2., plot_offset - 20 ), "%.2f" % value, font=self.font, fill=self.pre_color) return np.array(img) diff --git a/engagement_detector/webcam.py b/engagement_detector/webcam.py new file mode 100644 index 0000000..0f29633 --- /dev/null +++ b/engagement_detector/webcam.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +import rclpy +from rclpy.node import Node +import cv2 +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + +class ImagePublisher(Node): + def __init__(self): + super().__init__("image_publisher") + self.bridge = CvBridge() + self.cap = cv2.VideoCapture(0) + self.bgr8pub = self.create_publisher(Image, "/camera/color/image_raw", 10) + + def run(self): + while True: + try: + r, frame = self.cap.read() + if not r: + return + # BGR8 + self.bgr8pub.publish(self.bridge.cv2_to_imgmsg(frame, "bgr8")) + + except CvBridgeError as e: + print(e) + self.cap.release() + +def main(args=None): + rclpy.init(args=args) + + ip = ImagePublisher() + print("Publishing...") + ip.run() + + ip.destroy_node() + rclpy.shutdown() + +if __name__ == '__main__': + main() diff --git a/imgs/dog.png b/imgs/dog.png new file mode 100644 index 0000000..6417a4e Binary files /dev/null and b/imgs/dog.png differ diff --git a/package.xml b/package.xml index 8f13e11..fce65b0 100644 --- a/package.xml +++ b/package.xml @@ -15,13 +15,11 @@ ament_flake8 ament_pep257 python3-pytest - launch_pytest - python-numpy - python-imaging - python-tensorflow-gpu-pip + python3-numpy + python-tensorflow-pip python-keras-pip - python-opencv + python3-opencv std_msgs sensor_msgs rclpy diff --git a/setup.py b/setup.py index f77810f..0dbb47e 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python - +from os import path +from glob import glob from setuptools import setup from setuptools import find_packages @@ -13,6 +14,8 @@ ('share/ament_index/resource_index/packages', ['resource/' + package_name]), ('share/' + package_name, ['package.xml']), + (path.join('share', package_name), glob(path.join('engagement_detector', '*.py'))), + # (path.join('share', "models"), glob(path.join('models', '*.h5'))), ], install_requires=['setuptools'], zip_safe=True, @@ -28,7 +31,8 @@ tests_require=['pytest'], entry_points={ 'console_scripts': [ - 'engagement_detector_node = engagement_detector.ros_engagement_detector:main' + 'engagement_detector_node = engagement_detector.ros_engagement_detector:main', + 'webcam_node = engagement_detector.webcam:main', ], }, )