opencv camera logic rewrite

Resolve "Als gebruiker wil ik dat mijn data word opgeslagen in een database om data terug te zien"

Closes #35

See merge request technische-informatica-sm3/ti-projectten/rooziinuubii79!4

README.md

@@ -1,8 +1,70 @@
-# TI-project - Kobuki
+# TI-project - exploration robot Kobuki
 
 ## Description
-This project is a kobuki that drives around in dangerous areas and detects objects in its path. It uses a camera to detect objects. The kobuki is able to drive around in a room and detect objects.
+This project is a kobuki that drives around in dangerous areas and detects objects in its path. It uses a camera to detect objects. The purpose of this project is to explore dangerous areas without risking human lives. You are able to control the robot using controller on the website. 
 
 ## Photos
 ![Kobuki](/docs/assets/KobukiPhoto.jpg)
 
+## Installation
+
+### Requirements
+
+- Kobuki robot
+- Raspberry Pi (minimum 3B)
+- Camera
+- power supply for Raspberry Pi
+- laptop or computer
+
+### Steps
+
+1. **Install Python and Pip**
+   - Ensure you have Python installed on your system. You can download it from [python.org](https://www.python.org/).
+   - Pip is the package installer for Python. It usually comes with Python, but you can install it separately if needed.
+
+2. **Clone Our Repository**
+    - Clone our repository to your local machine doing the following :
+        - Open your terminal
+        - Change the current working directory to the location where you want the cloned directory.
+        - Type `git clone https://gitlab.fdmci.hva.nl/technische-informatica-sm3/ti-projectten/rooziinuubii79.git
+  
+3. **Install the required packages**
+    - Install the following packages on the server: "docker docker-buildx mosquitto nginx"
+    - Install the following packages on the Raspberry Pi: "g++ make cmake", https://github.com/eclipse-paho/paho.mqtt.c, https://github.com/eclipse-paho/paho.mqtt.cpp
+
+
+
+4. **Run the project**
+
+    #### Server side
+    - Run the following commands in the terminal to start the website:
+        - `cd src/Python/flask`
+        - `sudo docker buildx build -t flaskapp:latest .`
+        - `sudo docker run --network="host" --restart=always flaskapp:latest`
+    - Run the following commands in the terminal to start the MQTT broker:
+        - `cd src/config/server/`
+        - `mosquitto -c mosquitto.conf`
+    - Run the following commands in the terminal to start the Nginx server:
+        - `cd src/config/server/`
+        -  `cp nginx.conf /etc/nginx/nginx.conf`
+        -  `cp nginx-sites.conf /etc/nginx/sites-enable/nginx-sites.conf`
+
+
+    #### Raspberry Pi side
+    - Run the following commands to build and start the driver:
+        - `cd src/C++/Driver`
+        - `cmake ..`
+        - `make`
+        - `./kobuki_driver`
+    - Run the following commands to autostart the driver on startup of the Raspberry Pi:
+        - `cd src/config/rpi/`
+        - `cp kobukiDriver.service /etc/systemd/system/kobukiDriver.service`
+        - `systemctl enable kobukiDriver.service`
+        - `systemctl start kobukiDriver.service`
+
+## Extra notes
+Dont forget to change the IP address in the `src/C++/Driver/src/main.cpp` file to the IP address of the server.
+
+
+
+

src/C++/Driver/src/main.cpp

@@ -8,11 +8,12 @@
 using namespace std;
 using namespace cv;
 CKobuki robot;
+std::atomic<bool> kobuki_connected(false);
 
 std::string readMQTT();
 void parseMQTT(std::string message);
 void CapnSend();
-//ip, clientID, username, password
+// ip, clientID, username, password
 MqttClient client("ws://145.92.224.21/ws/", "KobukiRPI", "rpi", "rpiwachtwoordofzo"); // create a client object
 std::string message = "stop";
 std::string serializeKobukiData(const TKobukiData &data);
@@ -20,76 +21,114 @@ void sendKobukiData(TKobukiData &data);
 
 void setup()
 {
-    unsigned char *null_ptr(0);
-    robot.startCommunication("/dev/ttyUSB0", true, null_ptr);
-    //connect mqtt server and sub to commands
+  unsigned char *null_ptr(0);
+  robot.startCommunication("/dev/ttyUSB0", true, null_ptr);
+  // connect mqtt server and sub to commands
 
-    client.connect();
-    client.subscribe("home/commands");
+  client.connect();
+  client.subscribe("home/commands");
+}
+
+void checkKobukiConnection()
+{
+  while (true)
+  {
+    bool connected = robot.isConnected();
+    if (!connected && kobuki_connected)
+    {
+      cout << "Kobuki is disconnected" << endl;
+      kobuki_connected = false;
+    }
+    else if (connected && !kobuki_connected)
+    {
+      cout << "Kobuki is connecting..." << endl;
+      // Start de Kobuki automatisch
+      robot.startCommunication("/dev/ttyUSB0", true, nullptr);
+    }
+    std::this_thread::sleep_for(std::chrono::seconds(5)); // Controleer elke 5 seconden
+  }
 }
 
 int main()
 {
-    setup();
-    std::thread image (CapnSend);
-    std::thread safety([&]() { robot.robotSafety(&message); });
-    std::thread sendMqtt([&]() { sendKobukiData(robot.parser.data); });
-    
-    while(true){
-        std::string message = readMQTT();
-        if (!message.empty()){
-            parseMQTT(message);
-        }
-        
-    }
+  setup();
+  std::thread image(CapnSend);
+  std::thread safety([&](){ robot.robotSafety(&message); });
+  std::thread sendMqtt([&](){ sendKobukiData(robot.parser.data); });
 
-    sendMqtt.join();
-    safety.join();
-    image.join();
+  while (true)
+  {
+    std::string message = readMQTT();
+    if (!message.empty())
+    {
+      parseMQTT(message);
+    }
+  }
+
+  sendMqtt.join();
+  safety.join();
+  image.join();
 }
 
 std::string readMQTT()
 {
-    static std::string lastMessage;
+  static std::string lastMessage;
 
-    std::string message = client.getLastMessage();
-    if (!message.empty() && message != lastMessage)
-    {
-        std::cout << "MQTT Message: " << message << std::endl;
-        lastMessage = message;
-    }
+  std::string message = client.getLastMessage();
+  if (!message.empty() && message != lastMessage)
+  {
+    std::cout << "MQTT Message: " << message << std::endl;
+    lastMessage = message;
+  }
 
   // Add a small delay to avoid busy-waiting
   std::this_thread::sleep_for(std::chrono::milliseconds(100));
   return lastMessage;
 }
 
-void parseMQTT(std::string message) {
-  if (message == "up") {
+void parseMQTT(std::string message)
+{
+  if (message == "up")
+  {
     robot.forward(350);
-  } else if (message == "left") {
+  }
+  else if (message == "left")
+  {
     robot.setRotationSpeed(4);
-  } else if (message == "right") {
+  }
+  else if (message == "right")
+  {
     robot.setRotationSpeed(-4);
-  } else if (message == "down") {
+  }
+  else if (message == "down")
+  {
     robot.forward(-350);
-  } else if (message == "stop") {
+  }
+  else if (message == "stop")
+  {
     robot.sendNullMessage();
     robot.sendNullMessage();
-  } else if (message == "estop") {
+  }
+  else if (message == "estop")
+  {
     robot.forward(-400);
-  } else {
+  }
+  else
+  {
     std::cout << "Invalid command" << std::endl;
   }
 }
 
-void logToFile() {
-  while (true) {
+void logToFile()
+{
+  while (true)
+  {
     TKobukiData robotData = robot.parser.data;
     std::ofstream outputFile("log",
                              std::ios_base::app); // Open file in append mode to
                                                   // not overwrite own content
-    if (outputFile.is_open()) { // check if the file was opened successfully
+    if (outputFile.is_open())
+    { // check if the file was opened successfully
       // Get current time
       std::time_t now = std::time(nullptr);
       outputFile << "Timestamp: " << std::ctime(&now);
@@ -147,7 +186,9 @@ void logToFile() {
       outputFile << "UDID1: " << robotData.extraInfo.UDID1 << "\n";
       outputFile << "UDID2: " << robotData.extraInfo.UDID2 << "\n";
       outputFile.close();
-    } else {
+    }
+    else
+    {
       std::cerr << "Error opening file\n";
     }
 
@@ -155,8 +196,10 @@ void logToFile() {
   }
 }
 
-void sendIndividualKobukiData(const TKobukiData &data) {
-  while (true) {
+void sendIndividualKobukiData(const TKobukiData &data)
+{
+  while (true)
+  {
     std::cout << "Kobuki Data wordt gepubliceerd naar kobuki/data/timestamp: "
               << data.timestamp << std::endl;
     client.publishMessage("kobuki/data/timestamp",
@@ -244,7 +287,8 @@ void sendIndividualKobukiData(const TKobukiData &data) {
     client.publishMessage("kobuki/data/extraInfo/UDID2",
                           std::to_string(data.extraInfo.UDID2));
 
-    if (!data.gyroData.empty()) {
+    if (!data.gyroData.empty())
+    {
       const auto &latestGyro = data.gyroData.back();
       client.publishMessage("kobuki/data/gyroData/x",
                             std::to_string(latestGyro.x));
@@ -258,7 +302,8 @@ void sendIndividualKobukiData(const TKobukiData &data) {
   }
 }
 
-std::string serializeKobukiData(const TKobukiData &data) {
+std::string serializeKobukiData(const TKobukiData &data)
+{
   std::string json =
       "{\"timestamp\":" + std::to_string(data.timestamp) +
       ",\"BumperCenter\":" + std::to_string(data.BumperCenter) +
@@ -311,7 +356,8 @@ std::string serializeKobukiData(const TKobukiData &data) {
       ",\"UDID1\":" + std::to_string(data.extraInfo.UDID1) +
       ",\"UDID2\":" + std::to_string(data.extraInfo.UDID2) + "},\"gyroData\":[";
 
-  if (!data.gyroData.empty()) {
+  if (!data.gyroData.empty())
+  {
     const auto &latestGyro = data.gyroData.back();
     json += "{\"x\":" + std::to_string(latestGyro.x) +
             ",\"y\":" + std::to_string(latestGyro.y) +
@@ -323,12 +369,14 @@ std::string serializeKobukiData(const TKobukiData &data) {
 }
 // create extra function to send the message every 100ms
 // needed it so it can be threaded
-void sendKobukiData(TKobukiData &data) {
-    while (true) {
-        client.publishMessage("kobuki/data", serializeKobukiData(data));
-        std::cout << "Sent data" << std::endl;
-        std::this_thread::sleep_for(std::chrono::milliseconds(1000));
-    }
+void sendKobukiData(TKobukiData &data)
+{
+  while (true)
+  {
+    client.publishMessage("kobuki/data", serializeKobukiData(data));
+    std::cout << "Sent data" << std::endl;
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+  }
 }
 
 void CapnSend() {
@@ -340,16 +388,26 @@ void CapnSend() {
 
     Mat frame;
     while (true) {
-        cap >> frame; // Capture a new image frame
-        if (frame.empty()) {
-            cerr << "Error: Could not capture image" << endl;
-            continue;
+        if (!cap.read(frame)) { 
+            cout << "Reconnecting camera" << endl;
+            cap.release();
+            std::this_thread::sleep_for(std::chrono::seconds(1));
+            // Attempt to reconnect to the camera
+            cap.open(0);
+            if (!cap.isOpened()) {
+                cerr << "Error: Could not reconnect to camera" << endl;
+                std::this_thread::sleep_for(std::chrono::seconds(1)); // Wait before retrying
+                continue;
+            } else {
+                cout << "Reconnected to camera" << endl;
+                continue;
+            }
         }
 
         // Convert the image to a byte array
         vector<uchar> buf;
         imencode(".jpg", frame, buf);
-        auto* enc_msg = reinterpret_cast<unsigned char*>(buf.data());
+        auto *enc_msg = reinterpret_cast<unsigned char *>(buf.data());
 
         // Publish the image data
         client.publishMessage("kobuki/cam", string(enc_msg, enc_msg + buf.size()));

src/Python/flask/Dockerfile

@@ -14,5 +14,5 @@ EXPOSE 5000
 CMD ["python", "web/app.py"]
 
 #build instruction: sudo docker buildx build -t flaskapp:latest .   
-#run instruction: sudo docker run --network="host" flaskapp:latest
+#run instruction: sudo docker run --network="host" --restart=always flaskapp:latest
 # need to use network host to connect to the host's mqtt server

src/Python/flask/requirements.txt

@@ -3,3 +3,4 @@ paho-mqtt==1.6.1
 ultralytics==8.3.58 
 opencv-python-headless==4.6.0.66
 numpy==1.23.4
+mysql-connector-python==9.1.0

src/Python/flask/web/app.py

@@ -1,4 +1,4 @@
-from flask import Flask, Response, request, render_template, jsonify
+from flask import Flask, Response, request, render_template, jsonify, g
 import paho.mqtt.client as mqtt
 from ultralytics import YOLO
 import cv2
@@ -48,18 +48,7 @@ def on_message(client, userdata, message):
                     x1, y1, x2, y2 = map(int, box.xyxy[0])
                     cv2.rectangle(processed_image, (x1, y1), (x2, y2), (0, 255, 0), 2)
                     cv2.putText(processed_image, f"{class_name} {box.conf.item():.2f}", (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0), 2)
-
-
-# Globale MQTT setup
-def on_message(client,userdata, message):
-    global kobuki_message, latest_image
-    if message.topic == "kobuki/data":
-        kobuki_message = str(message.payload.decode("utf-8"))
-        with app.app_context():
-            sensor_data(kobuki_message)  # Sla de data op in de database
-    elif message.topic == "kobuki/cam":
-        latest_image = message.payload
-
+            # yolo_results_db()
 
 # Create an MQTT client instance
 mqtt_client = mqtt.Client()
@@ -71,9 +60,6 @@ mqtt_client.subscribe("kobuki/cam")
 
 mqtt_client.on_message = on_message  # this line needs to be under the function definition otherwise it can't find which function it needs to use
 
-mqtt_client.on_message = on_message  # this line needs to be under the function definition otherwise it can't find which function it needs to use
-
-
 # Database connectie-functie
 def get_db():
     if 'db' not in g:  # 'g' is specifiek voor een request en leeft zolang een request duurt
@@ -122,17 +108,10 @@ def move():
     cursor.close()
     db_connection.close()
     return jsonify({"status": "success", "direction": direction})
+    
 
-@app.route("/database")
-def database():
-    db = get_db()
-    cursor = db.cursor()
-    cursor.execute("SELECT * FROM kobuki_data")
-    rows = cursor.fetchall()
-    cursor.close()
-    return str(rows)
-
-def sensor_data(kobuki_message):
+@app.route('/data', methods=['GET'])
+def data():
     try:
         # Parse de JSON-string naar een Python-dictionary
         data = json.loads(kobuki_message)
@@ -152,23 +131,21 @@ def sensor_data(kobuki_message):
 
         # Database-insert
         db = get_db()
-        cursor = db.cursor()
+        with db.cursor() as cursor:
 
-        # Zorg dat je tabel `kobuki_data` kolommen heeft: `name` en `value`
-        sql_sensor = "INSERT INTO kobuki_data (name, value) VALUES (%s, %s)"
-        cursor.executemany(sql_sensor, sensor_data_tuples)
-
-        # Commit en sluit de cursor
-        db.commit()
-        cursor.close()
+            # Zorg dat je tabel `kobuki_data` kolommen heeft: `name` en `value`
+            sql_sensor = "INSERT INTO kobuki_data (name, value) VALUES (%s, %s)"
+            cursor.executemany(sql_sensor, sensor_data_tuples)
+    
+            # Commit en sluit de cursor
+            db.commit()
+            cursor.close()
     except json.JSONDecodeError as e:
         print(f"JSON decode error: {e}")
     except mysql.connector.Error as err:
         print(f"Database error: {err}")
-
-@app.route('/data', methods=['GET'])
-def data():
     return kobuki_message
+    
 
 
 @app.route('/image')
@@ -186,8 +163,28 @@ def image():
 def yolo_results_endpoint():
     global yolo_results
     with lock:
+        print(f"YOLO Results: {yolo_results}")  # Debug statement
+        db = get_db()
+        with db.cursor() as cursor:
+            sql_yolo = "INSERT INTO image (class, confidence) VALUES (%s, %s)"
+            yolo_tuples = [(result["class"], result["confidence"]) for result in yolo_results]
+            print(f"YOLO Tuples: {yolo_tuples}")  # Debug statement
+            cursor.executemany(sql_yolo, yolo_tuples)
+            db.commit()
+            cursor.close()
         return jsonify(yolo_results)
 
+# def yolo_results_db():
+#     global yolo_results
+#     with lock:
+#         db = get_db()
+#         with db.cursor() as cursor:
+#             sql_yolo = "INSERT INTO image (object, confidence) VALUES (%s, %s)"
+#             yolo_tuples = [(result["class"], result["confidence"]) for result in yolo_results]
+#             cursor.executemany(sql_yolo, yolo_tuples)
+#             db.commit()
+#             cursor.close()
+
 
 if __name__ == '__main__':
     app.run(debug=True, port=5000)

src/Python/flask/web/static/script.js

@@ -34,7 +34,8 @@ document.addEventListener("DOMContentLoaded", function() {
     }
   }
 
-    // Parse the data and show it on the website
+  // Parse the data and show it on the website
+  async function parseData() {
     const data = await fetchData();
     const sensorDataContainer = document.getElementById("sensor-data");
     sensorDataContainer.innerHTML = ""; // Clear previous data
@@ -42,20 +43,21 @@ document.addEventListener("DOMContentLoaded", function() {
     for (const [key, value] of Object.entries(data)) {
       const dataElement = document.createElement("p");
       dataElement.textContent = `${key}: ${value}`;
-      sensorDataContainer.appendChild(dataElement); // Voeg het element toe aan de container
+      sensorDataContainer.appendChild(dataElement); // Add the element to the container
     }
   }
 
   // Update the image
-  function updateImage() {
-    var img = document.getElementById("robot-image");
+  async function updateImage() {
+    let img = document.getElementById("robot-image");
     img.src = "/image?" + new Date().getTime(); // Add timestamp to avoid caching
+    // Wait for 200 milliseconds before fetching the next image
+    setTimeout(updateImage, 200);
   }
 
   // Fetch and display sensor data every 1 second
   setInterval(parseData, 1000);
 
-  // Update the image every 200 milliseconds
-  setInterval(updateImage, 100);
-});
-
+  // Start updating the image
+  updateImage();
+});