Merge branch 'main' of ssh://gitlab.fdmci.hva.nl/technische-informatica-sm3/ti-projectten/rooziinuubii79

opencv camera logic rewrite

See merge request technische-informatica-sm3/ti-projectten/rooziinuubii79!5

README.md

@@ -1,8 +1,70 @@
-# TI-project - Kobuki
+# TI-project - exploration robot Kobuki
 
 ## Description
-This project is a kobuki that drives around in dangerous areas and detects objects in its path. It uses a camera to detect objects. The kobuki is able to drive around in a room and detect objects.
+This project is a kobuki that drives around in dangerous areas and detects objects in its path. It uses a camera to detect objects. The purpose of this project is to explore dangerous areas without risking human lives. You are able to control the robot using controller on the website. 
 
 ## Photos
 ![Kobuki](/docs/assets/KobukiPhoto.jpg)
 
+## Installation
+
+### Requirements
+
+- Kobuki robot
+- Raspberry Pi (minimum 3B)
+- Camera
+- power supply for Raspberry Pi
+- laptop or computer
+
+### Steps
+
+1. **Install Python and Pip**
+   - Ensure you have Python installed on your system. You can download it from [python.org](https://www.python.org/).
+   - Pip is the package installer for Python. It usually comes with Python, but you can install it separately if needed.
+
+2. **Clone Our Repository**
+    - Clone our repository to your local machine doing the following :
+        - Open your terminal
+        - Change the current working directory to the location where you want the cloned directory.
+        - Type `git clone https://gitlab.fdmci.hva.nl/technische-informatica-sm3/ti-projectten/rooziinuubii79.git
+  
+3. **Install the required packages**
+    - Install the following packages on the server: "docker docker-buildx mosquitto nginx"
+    - Install the following packages on the Raspberry Pi: "g++ make cmake", https://github.com/eclipse-paho/paho.mqtt.c, https://github.com/eclipse-paho/paho.mqtt.cpp
+
+
+
+4. **Run the project**
+
+    #### Server side
+    - Run the following commands in the terminal to start the website:
+        - `cd src/Python/flask`
+        - `sudo docker buildx build -t flaskapp:latest .`
+        - `sudo docker run --network="host" --restart=always flaskapp:latest`
+    - Run the following commands in the terminal to start the MQTT broker:
+        - `cd src/config/server/`
+        - `mosquitto -c mosquitto.conf`
+    - Run the following commands in the terminal to start the Nginx server:
+        - `cd src/config/server/`
+        -  `cp nginx.conf /etc/nginx/nginx.conf`
+        -  `cp nginx-sites.conf /etc/nginx/sites-enable/nginx-sites.conf`
+
+
+    #### Raspberry Pi side
+    - Run the following commands to build and start the driver:
+        - `cd src/C++/Driver`
+        - `cmake ..`
+        - `make`
+        - `./kobuki_driver`
+    - Run the following commands to autostart the driver on startup of the Raspberry Pi:
+        - `cd src/config/rpi/`
+        - `cp kobukiDriver.service /etc/systemd/system/kobukiDriver.service`
+        - `systemctl enable kobukiDriver.service`
+        - `systemctl start kobukiDriver.service`
+
+## Extra notes
+Dont forget to change the IP address in the `src/C++/Driver/src/main.cpp` file to the IP address of the server.
+
+
+
+

src/C++/Driver/src/main.cpp

@@ -1,5 +1,6 @@
 #include <iostream>
 #include <thread>
+#include <fstream>
 #include "MQTT/MqttClient.h"
 #include "KobukiDriver/CKobuki.h"
 #include <opencv4/opencv2/opencv.hpp>
@@ -8,10 +9,12 @@
 using namespace std;
 using namespace cv;
 CKobuki robot;
+std::atomic<bool> kobuki_connected(false);
 
 std::string readMQTT();
 void parseMQTT(std::string message);
 void CapnSend();
+void checkKobukiConnection();
 // ip, clientID, username, password
 MqttClient client("ws://145.92.224.21/ws/", "KobukiRPI", "rpi", "rpiwachtwoordofzo"); // create a client object
 std::string message = "stop";
@@ -23,7 +26,6 @@ void setup()
   unsigned char *null_ptr(0);
   robot.startCommunication("/dev/ttyUSB0", true, null_ptr);
   // connect mqtt server and sub to commands
-
   client.connect();
   client.subscribe("home/commands");
 }
@@ -34,13 +36,17 @@ int main()
   std::thread image(CapnSend);
   std::thread safety([&](){ robot.robotSafety(&message); });
   std::thread sendMqtt([&](){ sendKobukiData(robot.parser.data); });
+  std::thread connectionChecker(checkKobukiConnection);
+  connectionChecker.detach(); // Laat deze thread onafhankelijk draaien
  
-    while(true){
+
+  while (true)
+  {
     std::string message = readMQTT();
-        if (!message.empty()){
+    if (!message.empty())
+    {
       parseMQTT(message);
     }
-        
   }
 
   sendMqtt.join();
@@ -48,6 +54,30 @@ int main()
   image.join();
 }
 
+std::mutex connectionMutex;
+
+void checkKobukiConnection()
+{
+    while (true)
+    {
+        std::lock_guard<std::mutex> lock(connectionMutex);
+        bool connected = robot.isConnected();
+        if (!connected && kobuki_connected)
+        {
+            cout << "Kobuki is disconnected" << endl;
+            kobuki_connected = false;
+        }
+        else if (connected && !kobuki_connected)
+        {
+            cout << "Kobuki is reconnecting..." << endl;
+            robot.startCommunication("/dev/ttyUSB0", true, nullptr);
+            kobuki_connected = true;
+        }
+        std::this_thread::sleep_for(std::chrono::seconds(5)); // Controleer elke 5 seconden
+    }
+}
+
+
 std::string readMQTT()
 {
   static std::string lastMessage;
@@ -64,32 +94,49 @@ std::string readMQTT()
   return lastMessage;
 }
 
-void parseMQTT(std::string message) {
+void parseMQTT(std::string message)
-  if (message == "up") {
+{
+  if (message == "up")
+  {
     robot.forward(350);
-  } else if (message == "left") {
+  }
+  else if (message == "left")
+  {
     robot.setRotationSpeed(4);
-  } else if (message == "right") {
+  }
+  else if (message == "right")
+  {
     robot.setRotationSpeed(-4);
-  } else if (message == "down") {
+  }
+  else if (message == "down")
+  {
     robot.forward(-350);
-  } else if (message == "stop") {
+  }
+  else if (message == "stop")
+  {
     robot.sendNullMessage();
     robot.sendNullMessage();
-  } else if (message == "estop") {
+  }
+  else if (message == "estop")
+  {
     robot.forward(-400);
-  } else {
+  }
+  else
+  {
     std::cout << "Invalid command" << std::endl;
   }
 }
 
-void logToFile() {
+void logToFile()
-  while (true) {
+{
+  while (true)
+  {
     TKobukiData robotData = robot.parser.data;
     std::ofstream outputFile("log",
                              std::ios_base::app); // Open file in append mode to
                                                   // not overwrite own content
-    if (outputFile.is_open()) { // check if the file was opened successfully
+    if (outputFile.is_open())
+    { // check if the file was opened successfully
       // Get current time
       std::time_t now = std::time(nullptr);
       outputFile << "Timestamp: " << std::ctime(&now);
@@ -147,7 +194,9 @@ void logToFile() {
       outputFile << "UDID1: " << robotData.extraInfo.UDID1 << "\n";
       outputFile << "UDID2: " << robotData.extraInfo.UDID2 << "\n";
       outputFile.close();
-    } else {
+    }
+    else
+    {
       std::cerr << "Error opening file\n";
     }
 
@@ -155,8 +204,10 @@ void logToFile() {
   }
 }
 
-void sendIndividualKobukiData(const TKobukiData &data) {
+void sendIndividualKobukiData(const TKobukiData &data)
-  while (true) {
+{
+  while (true)
+  {
     std::cout << "Kobuki Data wordt gepubliceerd naar kobuki/data/timestamp: "
               << data.timestamp << std::endl;
     client.publishMessage("kobuki/data/timestamp",
@@ -244,7 +295,8 @@ void sendIndividualKobukiData(const TKobukiData &data) {
     client.publishMessage("kobuki/data/extraInfo/UDID2",
                           std::to_string(data.extraInfo.UDID2));
 
-    if (!data.gyroData.empty()) {
+    if (!data.gyroData.empty())
+    {
       const auto &latestGyro = data.gyroData.back();
       client.publishMessage("kobuki/data/gyroData/x",
                             std::to_string(latestGyro.x));
@@ -258,7 +310,8 @@ void sendIndividualKobukiData(const TKobukiData &data) {
   }
 }
 
-std::string serializeKobukiData(const TKobukiData &data) {
+std::string serializeKobukiData(const TKobukiData &data)
+{
   std::string json =
       "{\"timestamp\":" + std::to_string(data.timestamp) +
       ",\"BumperCenter\":" + std::to_string(data.BumperCenter) +
@@ -311,7 +364,8 @@ std::string serializeKobukiData(const TKobukiData &data) {
       ",\"UDID1\":" + std::to_string(data.extraInfo.UDID1) +
       ",\"UDID2\":" + std::to_string(data.extraInfo.UDID2) + "},\"gyroData\":[";
 
-  if (!data.gyroData.empty()) {
+  if (!data.gyroData.empty())
+  {
     const auto &latestGyro = data.gyroData.back();
     json += "{\"x\":" + std::to_string(latestGyro.x) +
             ",\"y\":" + std::to_string(latestGyro.y) +
@@ -323,8 +377,10 @@ std::string serializeKobukiData(const TKobukiData &data) {
 }
 // create extra function to send the message every 100ms
 // needed it so it can be threaded
-void sendKobukiData(TKobukiData &data) {
+void sendKobukiData(TKobukiData &data)
-    while (true) {
+{
+  while (true)
+  {
     client.publishMessage("kobuki/data", serializeKobukiData(data));
     std::cout << "Sent data" << std::endl;
     std::this_thread::sleep_for(std::chrono::milliseconds(1000));
@@ -340,10 +396,20 @@ void CapnSend() {
 
     Mat frame;
     while (true) {
-        cap >> frame; // Capture a new image frame
+        if (!cap.read(frame)) { 
-        if (frame.empty()) {
+            cout << "Reconnecting camera" << endl;
-            cerr << "Error: Could not capture image" << endl;
+            cap.release();
+            std::this_thread::sleep_for(std::chrono::seconds(1));
+            // Attempt to reconnect to the camera
+            cap.open(0);
+            if (!cap.isOpened()) {
+                cerr << "Error: Could not reconnect to camera" << endl;
+                std::this_thread::sleep_for(std::chrono::seconds(1)); // Wait before retrying
                 continue;
+            } else {
+                cout << "Reconnected to camera" << endl;
+                continue;
+            }
         }
 
         // Convert the image to a byte array

src/Python/flask/Dockerfile

@@ -14,5 +14,5 @@ EXPOSE 5000
 CMD ["python", "web/app.py"]
 
 #build instruction: sudo docker buildx build -t flaskapp:latest .   
-#run instruction: sudo docker run --network="host" flaskapp:latest
+#run instruction: sudo docker run --network="host" --restart=always flaskapp:latest
 # need to use network host to connect to the host's mqtt server

src/Python/flask/requirements.txt

@@ -3,3 +3,4 @@ paho-mqtt==1.6.1
 ultralytics==8.3.58 
 opencv-python-headless==4.6.0.66
 numpy==1.23.4
+mysql-connector-python==9.1.0

src/Python/flask/web/app.py

@@ -1,4 +1,4 @@
-from flask import Flask, Response, request, render_template, jsonify
+from flask import Flask, Response, request, render_template, jsonify, g
 import paho.mqtt.client as mqtt
 from ultralytics import YOLO
 import cv2
@@ -49,18 +49,6 @@ def on_message(client, userdata, message):
                     cv2.rectangle(processed_image, (x1, y1), (x2, y2), (0, 255, 0), 2)
                     cv2.putText(processed_image, f"{class_name} {box.conf.item():.2f}", (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0), 2)
 
-
-# Globale MQTT setup
-def on_message(client,userdata, message):
-    global kobuki_message, latest_image
-    if message.topic == "kobuki/data":
-        kobuki_message = str(message.payload.decode("utf-8"))
-        with app.app_context():
-            sensor_data(kobuki_message)  # Sla de data op in de database
-    elif message.topic == "kobuki/cam":
-        latest_image = message.payload
-
-
 # Create an MQTT client instance
 mqtt_client = mqtt.Client()
 mqtt_client.username_pw_set("server", "serverwachtwoordofzo")
@@ -71,9 +59,6 @@ mqtt_client.subscribe("kobuki/cam")
 
 mqtt_client.on_message = on_message  # this line needs to be under the function definition otherwise it can't find which function it needs to use
 
-mqtt_client.on_message = on_message  # this line needs to be under the function definition otherwise it can't find which function it needs to use
-
-
 # Database connectie-functie
 def get_db():
     if 'db' not in g:  # 'g' is specifiek voor een request en leeft zolang een request duurt
@@ -123,16 +108,9 @@ def move():
     db_connection.close()
     return jsonify({"status": "success", "direction": direction})
     
-@app.route("/database")
-def database():
-    db = get_db()
-    cursor = db.cursor()
-    cursor.execute("SELECT * FROM kobuki_data")
-    rows = cursor.fetchall()
-    cursor.close()
-    return str(rows)
 
-def sensor_data(kobuki_message):
+@app.route('/data', methods=['GET'])
+def data():
     try:
         # Parse de JSON-string naar een Python-dictionary
         data = json.loads(kobuki_message)
@@ -152,7 +130,7 @@ def sensor_data(kobuki_message):
 
         # Database-insert
         db = get_db()
-        cursor = db.cursor()
+        with db.cursor() as cursor:
 
             # Zorg dat je tabel `kobuki_data` kolommen heeft: `name` en `value`
             sql_sensor = "INSERT INTO kobuki_data (name, value) VALUES (%s, %s)"
@@ -165,12 +143,10 @@ def sensor_data(kobuki_message):
         print(f"JSON decode error: {e}")
     except mysql.connector.Error as err:
         print(f"Database error: {err}")
-
-@app.route('/data', methods=['GET'])
-def data():
     return kobuki_message
     
 
+
 @app.route('/image')
 def image():
     global processed_image
@@ -186,6 +162,14 @@ def image():
 def yolo_results_endpoint():
     global yolo_results
     with lock:
+        db = get_db()
+        with db.cursor() as cursor:
+            sql_yolo = "INSERT INTO image (class, confidence) VALUES (%s, %s)"
+            yolo_tuples = [(result["class"], result["confidence"]) for result in yolo_results]
+            print(f"YOLO Tuples: {yolo_tuples}")  # Debug statement
+            cursor.executemany(sql_yolo, yolo_tuples)
+            db.commit()
+            cursor.close()
         return jsonify(yolo_results)
 
 

src/Python/flask/web/static/script.js

@@ -35,6 +35,7 @@ document.addEventListener("DOMContentLoaded", function() {
   }
 
   // Parse the data and show it on the website
+  async function parseData() {
     const data = await fetchData();
     const sensorDataContainer = document.getElementById("sensor-data");
     sensorDataContainer.innerHTML = ""; // Clear previous data
@@ -42,20 +43,21 @@ document.addEventListener("DOMContentLoaded", function() {
     for (const [key, value] of Object.entries(data)) {
       const dataElement = document.createElement("p");
       dataElement.textContent = `${key}: ${value}`;
-      sensorDataContainer.appendChild(dataElement); // Voeg het element toe aan de container
+      sensorDataContainer.appendChild(dataElement); // Add the element to the container
     }
   }
 
   // Update the image
-  function updateImage() {
+  async function updateImage() {
-    var img = document.getElementById("robot-image");
+    let img = document.getElementById("robot-image");
     img.src = "/image?" + new Date().getTime(); // Add timestamp to avoid caching
+    // Wait for 200 milliseconds before fetching the next image
+    setTimeout(updateImage, 200);
   }
 
   // Fetch and display sensor data every 1 second
   setInterval(parseData, 1000);
 
-  // Update the image every 200 milliseconds
+  // Start updating the image
-  setInterval(updateImage, 100);
+  updateImage();
 });
-