Implement person detection

.github/workflows/docker-build.yml

@@ -25,7 +25,7 @@ jobs:
           docker build . --file backend/smart_sec_cam/server/Dockerfile --tag ghcr.io/${{ github.repository }}/sec-cam-server:latest
           docker push ghcr.io/${{ github.repository }}/sec-cam-server:latest
 
-  build-motion-detection:
+  build-detector:
     runs-on: ubuntu-latest
 
     steps:
@@ -40,9 +40,9 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Build and push the Smart Sec Cam motion detection docker image
+      - name: Build and push the Smart Sec Cam detection docker image
         run: | 
-          docker build backend/ --file backend/smart_sec_cam/motion/Dockerfile --tag ghcr.io/${{ github.repository }}/sec-cam-motion:latest
+          docker build backend/ --file backend/smart_sec_cam/motion/Dockerfile --tag ghcr.io/${{ github.repository }}/sec-cam-detection:latest
           docker push ghcr.io/${{ github.repository }}/sec-cam-motion:latest
 
   build-redis:

README.md

@@ -4,6 +4,7 @@ A privacy-focused, intelligent security camera system.
 
 ## Features:
 - Multi-camera support w/ minimal configuration. Supports USB cameras and the Raspberry Pi camera module.
+- Face detection and person (full body) detection.
 - Motion detection that automatically saves videos and lets you view them in the web app.
 - Encrypted in transit, both from the cameras to the server and the server to your browser.
 - Integrated authentication 
@@ -100,4 +101,6 @@ module, run `cd backend && python3 -m pip install .[streamer,picam]`.
 
 ## Contributors
 
-- @khlam for his help with Github Actions and building docker images
+Thank you to the contributors that help keep this project moving forward!
+
+- [Khlam](https://github.com/khlam) for his help with Github Actions and building docker images

backend/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = smart-sec-cam
-version = 0.3.8
+version = 0.4.0
 author = Scott Barnes
 author_email = [email protected]
 description = A privacy-focused intelligent security camera system

backend/smart_sec_cam/detectors/detect_faces.py

@@ -0,0 +1,67 @@
+import time
+
+from smart_sec_cam.detectors.face import FaceDetector
+from smart_sec_cam.redis import RedisImageReceiver
+
+
+CHANNEL_LIST_INTERVAL = 10
+SLEEP_TIME = 0.01
+
+
+def main(redis_url: str, redis_port: int, video_dir: str):
+    # Fetch list of channels
+    # Subscribe to each channel to get frames
+    image_receiver = RedisImageReceiver(redis_url, redis_port)
+    active_channels = image_receiver.get_all_channels()
+    last_channel_check_time = time.monotonic()
+    image_receiver.set_channels(active_channels)
+    image_receiver.start_listener_thread()
+    # Create and start MotionDetection instance for each channel
+    face_detectors = {channel: FaceDetector(channel, video_dir=video_dir)
+                        for channel in active_channels}
+    for detector in face_detectors.values():
+        detector.run_in_background()
+    while True:
+        # Check for new frames from each channel and push to the corresponding MotionDetection instance
+        if image_receiver.has_message():
+            message = image_receiver.get_message()
+            frame = message.get("data")
+            channel = message.get("channel").decode("utf-8")
+            face_detectors.get(channel).add_frame(frame)
+        else:
+            time.sleep(SLEEP_TIME)
+        # Periodically check for updated channel list in background thread
+        if time.monotonic() - last_channel_check_time > CHANNEL_LIST_INTERVAL:
+            active_channels = image_receiver.get_all_channels()
+            # Check for new channels
+            new_channels = []
+            for channel in active_channels:
+                if channel not in face_detectors.keys():
+                    print(f"Detected new channel: {channel}")
+                    new_channels.append(channel)
+                    face_detectors[channel] = FaceDetector(channel, video_dir=video_dir)
+            # Check for removed channels
+            removed_channels = []
+            for channel in face_detectors.keys():
+                if channel not in active_channels:
+                    print(f"Removing channel: {channel}")
+                    removed_channels.append(channel)
+                    face_detectors.get(channel).stop()
+                    del face_detectors[channel]
+            # If the active channel list changed, update the redis subscription list
+            if new_channels or removed_channels:
+                image_receiver.set_channels(active_channels)
+            last_channel_check_time = time.monotonic()
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--redis-url', help='Server address to stream images to', type=str, default='localhost')
+    parser.add_argument('--redis-port', help='Server port to stream images to', type=int, default=6379)
+    parser.add_argument('--video-dir', help='Directory in which video files are stored', type=str,
+                        default="data/videos")
+    args = parser.parse_args()
+
+    main(args.redis_url, args.redis_port, args.video_dir)

backend/smart_sec_cam/motion/main.py → backend/smart_sec_cam/detectors/detect_motion.py

@@ -1,7 +1,7 @@
 import os
 import time
 
-from smart_sec_cam.motion.detection import MotionDetector
+from smart_sec_cam.detectors.motion import MotionDetector
 from smart_sec_cam.redis import RedisImageReceiver
 
 
@@ -69,4 +69,4 @@ def main(redis_url: str, redis_port: int, video_dir: str, motion_threshold: int)
 
     motion_threshold = int(os.environ.get("MOTION_THRESHOLD"))
 
-    main(args.redis_url, args.redis_port, args.video_dir, motion_threshold)
+    main(args.redis_url, args.redis_port, args.video_dir, motion_threshold)

backend/smart_sec_cam/detectors/detect_people.py

@@ -0,0 +1,67 @@
+import time
+
+from smart_sec_cam.detectors.person import PersonDetector
+from smart_sec_cam.redis import RedisImageReceiver
+
+
+CHANNEL_LIST_INTERVAL = 10
+SLEEP_TIME = 0.01
+
+
+def main(redis_url: str, redis_port: int, video_dir: str):
+    # Fetch list of channels
+    # Subscribe to each channel to get frames
+    image_receiver = RedisImageReceiver(redis_url, redis_port)
+    active_channels = image_receiver.get_all_channels()
+    last_channel_check_time = time.monotonic()
+    image_receiver.set_channels(active_channels)
+    image_receiver.start_listener_thread()
+    # Create and start MotionDetection instance for each channel
+    person_detectors = {channel: PersonDetector(channel, video_dir=video_dir)
+                        for channel in active_channels}
+    for detector in person_detectors.values():
+        detector.run_in_background()
+    while True:
+        # Check for new frames from each channel and push to the corresponding MotionDetection instance
+        if image_receiver.has_message():
+            message = image_receiver.get_message()
+            frame = message.get("data")
+            channel = message.get("channel").decode("utf-8")
+            person_detectors.get(channel).add_frame(frame)
+        else:
+            time.sleep(SLEEP_TIME)
+        # Periodically check for updated channel list in background thread
+        if time.monotonic() - last_channel_check_time > CHANNEL_LIST_INTERVAL:
+            active_channels = image_receiver.get_all_channels()
+            # Check for new channels
+            new_channels = []
+            for channel in active_channels:
+                if channel not in person_detectors.keys():
+                    print(f"Detected new channel: {channel}")
+                    new_channels.append(channel)
+                    person_detectors[channel] = PersonDetector(channel, video_dir=video_dir)
+            # Check for removed channels
+            removed_channels = []
+            for channel in person_detectors.keys():
+                if channel not in active_channels:
+                    print(f"Removing channel: {channel}")
+                    removed_channels.append(channel)
+                    person_detectors.get(channel).stop()
+                    del person_detectors[channel]
+            # If the active channel list changed, update the redis subscription list
+            if new_channels or removed_channels:
+                image_receiver.set_channels(active_channels)
+            last_channel_check_time = time.monotonic()
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--redis-url', help='Server address to stream images to', type=str, default='localhost')
+    parser.add_argument('--redis-port', help='Server port to stream images to', type=int, default=6379)
+    parser.add_argument('--video-dir', help='Directory in which video files are stored', type=str,
+                        default="data/videos")
+    args = parser.parse_args()
+
+    main(args.redis_url, args.redis_port, args.video_dir)

backend/smart_sec_cam/detectors/face.py

@@ -0,0 +1,91 @@
+import queue
+import threading
+import time
+
+import cv2
+import numpy as np
+
+from smart_sec_cam.video.writer import VideoWriter
+
+
+class FaceDetector:
+    scale_factor = 1.1
+    min_neighbors = 16
+
+    def __init__(self, channel_name: str, video_duration_seconds: int = 10, video_dir: str = "data/videos"):
+        self.channel_name = channel_name
+        self.video_duration = video_duration_seconds
+        self.video_dir = video_dir
+        self.video_writer = VideoWriter(self.channel_name, path=self.video_dir)
+        self.frame_queue = queue.Queue()
+        self.face_detector = cv2.CascadeClassifier('smart_sec_cam/detectors/haarcascade_frontalface_default.xml')
+        self.detection_thread = threading.Thread(target=self.run, daemon=True)
+        self.shutdown = False
+
+    def add_frame(self, frame: bytes):
+        self.frame_queue.put(frame)
+
+    def run(self):
+        while not self.shutdown:
+            # Get latest frame
+            decoded_frame_greyscale = self._get_decoded_frame(greyscale=True)
+            decoded_frame = self._get_decoded_frame()
+            # Check for faces
+            faces = self.face_detector.detectMultiScale(decoded_frame_greyscale, self.scale_factor, self.min_neighbors)
+            if len(faces) != 0:
+                print(f"Detected face for {self.channel_name}")
+                self._record_video(decoded_frame, faces)
+
+    def _get_decoded_frame(self, greyscale=False):
+        new_frame = self.frame_queue.get()
+        if greyscale:
+            return self._decode_frame_greyscale(new_frame)
+        else:
+            return self._decode_frame(new_frame)
+
+    @staticmethod
+    def _decode_frame(frame: bytes):
+        return cv2.imdecode(np.frombuffer(frame, dtype=np.uint8), cv2.IMREAD_COLOR)
+
+    @staticmethod
+    def _decode_frame_greyscale(frame: bytes):
+        # Convert frame to greyscale and blur it
+        greyscale_frame = cv2.imdecode(np.frombuffer(frame, dtype=np.uint8), cv2.IMREAD_GRAYSCALE)
+        return cv2.GaussianBlur(greyscale_frame, (21, 21), 0)
+
+    def _record_video(self, first_frame, first_frame_faces):
+        start_time = time.monotonic()
+        self.video_writer.reset()
+        # Add first frame to video writer
+        first_frame_with_face = self._draw_face_on_frame(first_frame, first_frame_faces)
+        self.video_writer.add_frame(first_frame_with_face)
+        while not self._done_recording_video(start_time):
+            if self._has_decoded_frame():
+                new_frame = self._get_decoded_frame()
+                new_frame_greyscale = self._get_decoded_frame(greyscale=True)
+                faces = self.face_detector.detectMultiScale(new_frame_greyscale, self.scale_factor, self.min_neighbors)
+                new_frame_with_face = self._draw_face_on_frame(new_frame, faces)
+                self.video_writer.add_frame(new_frame_with_face)
+            else:
+                time.sleep(0.01)
+        self.video_writer.write()
+
+    def _done_recording_video(self, start_time: float) -> bool:
+        return time.monotonic() - start_time > self.video_duration
+
+    def _has_decoded_frame(self) -> bool:
+        return not self.frame_queue.empty()
+
+    @staticmethod
+    def _draw_face_on_frame(frame, faces):
+        modified_frame = frame.copy()
+        for (x, y, w, h) in faces:
+            cv2.rectangle(modified_frame, (x, y), (x + w, y + h), (255, 0, 0), 2)
+        return modified_frame
+
+    def run_in_background(self):
+        self.detection_thread.start()
+
+
+
+