updated set up docs

opencv camera logic rewrite

See merge request technische-informatica-sm3/ti-projectten/rooziinuubii79!5

README.md

@@ -29,11 +29,42 @@ This project is a kobuki that drives around in dangerous areas and detects objec
         - Type `git clone https://gitlab.fdmci.hva.nl/technische-informatica-sm3/ti-projectten/rooziinuubii79.git
   
 3. **Install the required packages**
-    - Open the terminal and navigate to the project - scr - Python - flask.
+    - Install the following packages on the server: "docker docker-buildx mosquitto nginx"
-    - Run the following command to install the required packages:
+    - Install the following packages on the Raspberry Pi: "g++ make cmake libopencv-dev libssl-dev", https://github.com/eclipse-paho/paho.mqtt.c, https://github.com/eclipse-paho/paho.mqtt.cpp 
-        - `pip install -r requirements.txt`
+
-        -  This will install all the python packages required to run the project.
+
-        -  for C++, you will need to install the following packages:
+
-           - OpenCV
+4. **Run the project**
-           - mqtt-client
+
-4.   
+    #### Server side
+    - Run the following commands in the terminal to start the website:
+        - `cd src/Python/flask`
+        - `sudo docker buildx build -t flaskapp:latest .`
+        - `sudo docker run --network="host" --restart=always flaskapp:latest`
+    - Run the following commands in the terminal to start the MQTT broker:
+        - `cd src/config/server/`
+        - `mosquitto -c mosquitto.conf`
+    - Run the following commands in the terminal to start the Nginx server:
+        - `cd src/config/server/`
+        -  `cp nginx.conf /etc/nginx/nginx.conf`
+        -  `cp nginx-sites.conf /etc/nginx/sites-enable/nginx-sites.conf`
+
+
+    #### Raspberry Pi side
+    - Run the following commands to build and start the driver:
+        - `cd src/C++/Driver`
+        - `cmake ..`
+        - `make`
+        - `./kobuki_driver`
+    - Run the following commands to autostart the driver on startup of the Raspberry Pi:
+        - `cd src/config/rpi/`
+        - `cp kobukiDriver.service /etc/systemd/system/kobukiDriver.service`
+        - `systemctl enable kobukiDriver.service`
+        - `systemctl start kobukiDriver.service`
+
+## Extra notes
+Dont forget to change the IP address in the `src/C++/Driver/src/main.cpp` file to the IP address of the server.
+
+
+
+

docs/code/kobuki-reconnect.md

@@ -0,0 +1,78 @@
+# Kobuki automatische reconnect
+
+Mijn taak was om de kobuki automatisch te reconnecten als de verbinding verbroken werd met de pi. nu moet je telkens handmatig de pi laten connecten met de kobuki, dit is niet handig als iemand niet weet hoe dit moet. 
+
+De connectie word gemaakt met ttyUSB0. Dat is de eerste poort die wij kunnen gebruiken voor de kobuki. 
+
+```cpp
+  robot.startCommunication("/dev/ttyUSB0", true, null_ptr);
+```
+
+ik heb een functie gemaakt die kijkt of de pi nog is verbonden aan de kobuki. in de if statement kijkt hij elke 5 seconden of de ttyusb0 poort beschikbaar is. 
+
+```cpp
+void checkKobukiConnection()
+{
+    while (true)
+    {
+        std::lock_guard<std::mutex> lock(connectionMutex);
+
+        // Controleer of het apparaat beschikbaar is
+        if (!std::ifstream("/dev/ttyUSB0")){
+            if (kobuki_connected){
+                cout << "Kobuki disconnected: USB device not found." << endl;
+                kobuki_connected = false;
+            }
+            std::this_thread::sleep_for(std::chrono::seconds(5));
+            continue; // Probeer later opnieuw
+        }
+```	
+
+Hier kijk ik dan of de kobuki is geconnect, zoniet dan moet ie weer connecten met ttyUSB0
+
+```cpp
+ // Controleer of de Kobuki verbonden is
+        if (!robot.isConnected()){
+            if (kobuki_connected){
+                cout << "Kobuki disconnected." << endl;
+                kobuki_connected = false;
+            }
+
+            cout << "Attempting to reconnect Kobuki..." << endl;
+            robot.startCommunication("/dev/ttyUSB0", true, nullptr);
+
+            if (robot.isConnected()){
+                cout << "Kobuki reconnected successfully!" << endl;
+                kobuki_connected = true;
+            }
+            else{
+                cout << "Failed to reconnect Kobuki, retrying in 5 seconds..." << endl;
+            }
+        }
+```        
+
+Nu heb ik het probleem dat als ik de kabel eruit steek en terug in de kobuki stop dat de pi niet meer wil connecten. Dit komt omdat het systeem denkt dat de poort "ttyUSB0" nog steeds gebruikt word. Als ik de kabel dan terug stop word de poort "ttyUSB1" gebruikt, omdat ttyusb0 niet word vrijgegeven.
+
+```bash
+ishak@raspberrypi:~ $ dmesg | tail -n 20
+[10516.084132] usb 1-1.3: Product: iClebo Kobuki
+[10516.084144] usb 1-1.3: Manufacturer: Yujin Robot
+[10516.084155] usb 1-1.3: SerialNumber: kobuki_AI02MQMK
+[10516.091210] ftdi_sio 1-1.3:1.0: FTDI USB Serial Device converter detected
+[10516.091414] usb 1-1.3: Detected FT232R
+[10516.099169] usb 1-1.3: FTDI USB Serial Device converter now attached to ttyUSB1
+[10574.100491] usb 1-1.3: USB disconnect, device number 34
+[10574.101596] ftdi_sio ttyUSB1: FTDI USB Serial Device converter now disconnected from ttyUSB1
+[10574.101735] ftdi_sio 1-1.3:1.0: device disconnected
+[10579.697816] usb 1-1.3: new full-speed USB device number 35 using dwc_otg
+[10579.829776] usb 1-1.3: New USB device found, idVendor=0403, idProduct=6001, bcdDevice= 6.00
+[10579.829821] usb 1-1.3: New USB device strings: Mfr=1, Product=2, SerialNumber=3
+[10579.829836] usb 1-1.3: Product: iClebo Kobuki
+[10579.829848] usb 1-1.3: Manufacturer: Yujin Robot
+[10579.829860] usb 1-1.3: SerialNumber: kobuki_AI02MQMK
+[10579.840148] ftdi_sio 1-1.3:1.0: FTDI USB Serial Device converter detected
+[10579.840351] usb 1-1.3: Detected FT232R
+[10579.842208] usb 1-1.3: FTDI USB Serial Device converter now attached to ttyUSB1
+[10612.745819] hwmon hwmon1: Voltage normalised
+[10614.761829] hwmon hwmon1: Undervoltage detected!
+```

src/C++/Driver/src/main.cpp

@@ -1,5 +1,7 @@
 #include <iostream>
 #include <thread>
+#include <fstream>
+#include <filesystem>
 #include "MQTT/MqttClient.h"
 #include "KobukiDriver/CKobuki.h"
 #include <opencv4/opencv2/opencv.hpp>
@@ -10,105 +12,163 @@ using namespace cv;
 CKobuki robot;
 std::atomic<bool> kobuki_connected(false);
 
-
 std::string readMQTT();
 void parseMQTT(std::string message);
 void CapnSend();
-//ip, clientID, username, password
+void checkKobukiConnection();
+// ip, clientID, username, password
 MqttClient client("ws://145.92.224.21/ws/", "KobukiRPI", "rpi", "rpiwachtwoordofzo"); // create a client object
 std::string message = "stop";
 std::string serializeKobukiData(const TKobukiData &data);
 void sendKobukiData(TKobukiData &data);
 
-void setup()
+std::string findKobukiPort()
 {
-    unsigned char *null_ptr(0);
+    for (const auto& entry : std::filesystem::directory_iterator("/dev"))
-    robot.startCommunication("/dev/ttyUSB0", true, null_ptr);
+    {
-    //connect mqtt server and sub to commands
+        std::string device = entry.path().string();
-
+        if (device.find("ttyUSB") != std::string::npos)
-    client.connect();
+        {
-    client.subscribe("home/commands");
+            return device; // Returneer de eerste gevonden poort
+        }
+    }
+    return ""; // Geen poort gevonden
 }
 
-void checkKobukiConnection() {
+void setup()
-    while (true) {
+{
-        bool connected = robot.isConnected();
+  std::string port = findKobukiPort();
-        if (!connected && kobuki_connected) {
+  unsigned char *null_ptr(0);
-            cout << "Kobuki is disconnected" << endl;
+  robot.startCommunication(const_cast<char*>(port.c_str()), true, null_ptr);
-            kobuki_connected = false;
+  // connect mqtt server and sub to commands
-        } else if (connected && !kobuki_connected) {
+  client.connect();
-            cout << "Kobuki is connecting..." << endl;
+  client.subscribe("home/commands");
-            // Start de Kobuki automatisch
-            robot.startCommunication("/dev/ttyUSB0", true, nullptr);
-        }
-        std::this_thread::sleep_for(std::chrono::seconds(5)); // Controleer elke 5 seconden
-    }
 }
 
 int main()
 {
-    setup();
+  setup();
-    std::thread image (CapnSend);
+  std::thread image(CapnSend);
-    std::thread safety([&]() { robot.robotSafety(&message); });
+  std::thread safety([&](){ robot.robotSafety(&message); });
-    std::thread sendMqtt([&]() { sendKobukiData(robot.parser.data); });
+  std::thread sendMqtt([&](){ sendKobukiData(robot.parser.data); });
-    std::thread connectionThread(checkKobukiConnection);
+  std::thread connectionChecker(checkKobukiConnection);
-    
+  connectionChecker.detach(); // Laat deze thread onafhankelijk draaien
-    while(true){
+ 
-        std::string message = readMQTT();
-        if (!message.empty()){
-            parseMQTT(message);
-        }
-        
-    }
 
-    sendMqtt.join();
+  while (true)
-    safety.join();
+  {
-    image.join();
+    std::string message = readMQTT();
-    connectionThread.join();
+    if (!message.empty())
+    {
+      parseMQTT(message);
+    }
+  }
+
+  sendMqtt.join();
+  safety.join();
+  image.join();
+}
+
+std::mutex connectionMutex;
+
+void checkKobukiConnection()
+{
+    while (true)
+    {
+        std::lock_guard<std::mutex> lock(connectionMutex);
+        std::string port = findKobukiPort();
+
+        if (port.empty()) {
+            if (kobuki_connected) {
+                cout << "Kobuki disconnected: No USB device found." << endl;
+                kobuki_connected = false;
+            }
+            std::this_thread::sleep_for(std::chrono::seconds(5));
+            continue; // Probeer later opnieuw
+        }
+
+        // Controleer of de Kobuki verbonden is
+        if (!robot.isConnected()){
+            if (kobuki_connected){
+                cout << "Kobuki disconnected." << endl;
+                kobuki_connected = false;
+            }
+
+            cout << "Attempting to reconnect Kobuki..." << endl;
+            robot.startCommunication(const_cast<char*>(port.c_str()), true, nullptr);
+
+            if (robot.isConnected()){
+                cout << "Kobuki reconnected successfully!" << endl;
+                kobuki_connected = true;
+            }
+            else{
+                cout << "Failed to reconnect Kobuki, retrying in 5 seconds..." << endl;
+            }
+        }
+        // Wacht voordat je opnieuw controleert
+        std::this_thread::sleep_for(std::chrono::seconds(5));
+    }
 }
 
 std::string readMQTT()
 {
-    static std::string lastMessage;
+  static std::string lastMessage;
 
-    std::string message = client.getLastMessage();
+  std::string message = client.getLastMessage();
-    if (!message.empty() && message != lastMessage)
+  if (!message.empty() && message != lastMessage)
-    {
+  {
-        std::cout << "MQTT Message: " << message << std::endl;
+    std::cout << "MQTT Message: " << message << std::endl;
-        lastMessage = message;
+    lastMessage = message;
-    }
+  }
 
   // Add a small delay to avoid busy-waiting
   std::this_thread::sleep_for(std::chrono::milliseconds(100));
   return lastMessage;
 }
 
-void parseMQTT(std::string message) {
+void parseMQTT(std::string message)
-  if (message == "up") {
+{
+  if (message == "up")
+  {
     robot.forward(350);
-  } else if (message == "left") {
+  }
+  else if (message == "left")
+  {
     robot.setRotationSpeed(4);
-  } else if (message == "right") {
+  }
+  else if (message == "right")
+  {
     robot.setRotationSpeed(-4);
-  } else if (message == "down") {
+  }
+  else if (message == "down")
+  {
     robot.forward(-350);
-  } else if (message == "stop") {
+  }
+  else if (message == "stop")
+  {
     robot.sendNullMessage();
     robot.sendNullMessage();
-  } else if (message == "estop") {
+  }
+  else if (message == "estop")
+  {
     robot.forward(-400);
-  } else {
+  }
+  else
+  {
     std::cout << "Invalid command" << std::endl;
   }
 }
 
-void logToFile() {
+void logToFile()
-  while (true) {
+{
+  while (true)
+  {
     TKobukiData robotData = robot.parser.data;
     std::ofstream outputFile("log",
                              std::ios_base::app); // Open file in append mode to
                                                   // not overwrite own content
-    if (outputFile.is_open()) { // check if the file was opened successfully
+    if (outputFile.is_open())
+    { // check if the file was opened successfully
       // Get current time
       std::time_t now = std::time(nullptr);
       outputFile << "Timestamp: " << std::ctime(&now);
@@ -166,7 +226,9 @@ void logToFile() {
       outputFile << "UDID1: " << robotData.extraInfo.UDID1 << "\n";
       outputFile << "UDID2: " << robotData.extraInfo.UDID2 << "\n";
       outputFile.close();
-    } else {
+    }
+    else
+    {
       std::cerr << "Error opening file\n";
     }
 
@@ -174,8 +236,10 @@ void logToFile() {
   }
 }
 
-void sendIndividualKobukiData(const TKobukiData &data) {
+void sendIndividualKobukiData(const TKobukiData &data)
-  while (true) {
+{
+  while (true)
+  {
     std::cout << "Kobuki Data wordt gepubliceerd naar kobuki/data/timestamp: "
               << data.timestamp << std::endl;
     client.publishMessage("kobuki/data/timestamp",
@@ -263,7 +327,8 @@ void sendIndividualKobukiData(const TKobukiData &data) {
     client.publishMessage("kobuki/data/extraInfo/UDID2",
                           std::to_string(data.extraInfo.UDID2));
 
-    if (!data.gyroData.empty()) {
+    if (!data.gyroData.empty())
+    {
       const auto &latestGyro = data.gyroData.back();
       client.publishMessage("kobuki/data/gyroData/x",
                             std::to_string(latestGyro.x));
@@ -277,7 +342,8 @@ void sendIndividualKobukiData(const TKobukiData &data) {
   }
 }
 
-std::string serializeKobukiData(const TKobukiData &data) {
+std::string serializeKobukiData(const TKobukiData &data)
+{
   std::string json =
       "{\"timestamp\":" + std::to_string(data.timestamp) +
       ",\"BumperCenter\":" + std::to_string(data.BumperCenter) +
@@ -330,7 +396,8 @@ std::string serializeKobukiData(const TKobukiData &data) {
       ",\"UDID1\":" + std::to_string(data.extraInfo.UDID1) +
       ",\"UDID2\":" + std::to_string(data.extraInfo.UDID2) + "},\"gyroData\":[";
 
-  if (!data.gyroData.empty()) {
+  if (!data.gyroData.empty())
+  {
     const auto &latestGyro = data.gyroData.back();
     json += "{\"x\":" + std::to_string(latestGyro.x) +
             ",\"y\":" + std::to_string(latestGyro.y) +
@@ -340,18 +407,22 @@ std::string serializeKobukiData(const TKobukiData &data) {
   json += "]}";
   return json;
 }
+
 // create extra function to send the message every 100ms
 // needed it so it can be threaded
-void sendKobukiData(TKobukiData &data) {
+void sendKobukiData(TKobukiData &data)
-    while (true) {
+{
-        client.publishMessage("kobuki/data", serializeKobukiData(data));
+  while (true)
-        std::cout << "Sent data" << std::endl;
+  {
-        std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    client.publishMessage("kobuki/data", serializeKobukiData(data));
-    }
+    std::cout << "Sent data" << std::endl;
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+  }
 }
 
 void CapnSend() {
-    VideoCapture cap(0);
+  int COMPRESSION_LEVEL = 90;
+    VideoCapture cap(0); // Open the camera
     if (!cap.isOpened()) {
         cerr << "Error: Could not open camera" << endl;
         return;
@@ -359,20 +430,37 @@ void CapnSend() {
 
     Mat frame;
     while (true) {
-        cap >> frame; // Capture a new image frame
+        if (!cap.read(frame)) { 
-        if (frame.empty()) {
+            cout << "Reconnecting camera" << endl;
-            cerr << "Error: Could not capture image" << endl;
+            cap.release();
-            continue;
+            std::this_thread::sleep_for(std::chrono::seconds(1));
+            // Attempt to reconnect to the camera
+            cap.open(0);
+            if (!cap.isOpened()) {
+                cerr << "Error: Could not reconnect to camera" << endl;
+                std::this_thread::sleep_for(std::chrono::seconds(1)); // Wait before retrying
+                continue;
+            } else {
+                cout << "Reconnected to camera" << endl;
+                continue;
+            }
         }
 
-        // Convert the image to a byte array
-        vector<uchar> buf;
-        imencode(".jpg", frame, buf);
-        auto* enc_msg = reinterpret_cast<unsigned char*>(buf.data());
 
-        // Publish the image data
+        vector<uchar> imgbuf;
-        client.publishMessage("kobuki/cam", string(enc_msg, enc_msg + buf.size()));
+        vector<int> compression_params;
-        cout << "Sent image" << endl;
+        compression_params.push_back(IMWRITE_JPEG_QUALITY); // Set JPEG quality
+        compression_params.push_back(COMPRESSION_LEVEL);  // Adjust the quality level (0-100, lower = more compression)
+
+        // Encode the image into the byte buffer with the specified compression parameters
+        imencode(".jpg", frame, imgbuf, compression_params);
+
+        // Convert the vector<uchar> buffer to a string (no casting)
+        string enc_msg(imgbuf.begin(), imgbuf.end());
+
+        // Publish the compressed image data (MQTT, in this case)
+        client.publishMessage("kobuki/cam", enc_msg);
+        cout << "Sent compressed image" << endl;
 
         std::this_thread::sleep_for(std::chrono::milliseconds(200)); // Send image every 200ms
     }

src/Python/flask/web/app.py

@@ -85,6 +85,7 @@ def index():
 @app.route('/control', methods=["GET", "POST"])
 def control():
     if request.authorization and request.authorization.username == 'ishak' and request.authorization.password == 'kobuki':
+        yolo_results_db()
         return render_template('control.html')
     else:
         return ('Unauthorized', 401, {'WWW-Authenticate': 'Basic realm="Login Required"'})
@@ -160,20 +161,25 @@ def image():
 
 @app.route('/yolo_results', methods=['GET'])
 def yolo_results_endpoint():
-    global yolo_results
+    global yolo_results        
-    with lock:
+    return jsonify(yolo_results)
-        return jsonify(yolo_results)
 
 def yolo_results_db():
-    global yolo_results
+    global yolo_results    
-    db = get_db()
+    with lock:
-    with db.cursor() as cursor:
+        try:
-        sql_yolo = "INSERT INTO yolo_results (object, confidence) VALUES (%s, %s)"
+            db = get_db()
-        yolo_tuples = [(result["class"], result["confidence"]) for result in yolo_results]
+            with db.cursor() as cursor:
-        cursor.executemany(sql_yolo, yolo_tuples)
+                sql_yolo = "INSERT INTO image (class, confidence) VALUES (%s, %s)"
-        db.commit()
+                yolo_tuples = [(result["class"], result["confidence"]) for result in yolo_results]
-        cursor.close()
+                print(f"YOLO Tuples: {yolo_tuples}")  # Debug statement
-
+                cursor.executemany(sql_yolo, yolo_tuples)
+                db.commit()
+                cursor.close()
+        except mysql.connector.Error as err:
+            print(f"Database error: {err}")
+        except Exception as e:
+            print(f"Unexpected error: {e}")
 
 if __name__ == '__main__':
     app.run(debug=True, port=5000)

src/Python/flask/web/static/script.js

@@ -43,19 +43,21 @@ document.addEventListener("DOMContentLoaded", function() {
     for (const [key, value] of Object.entries(data)) {
       const dataElement = document.createElement("p");
       dataElement.textContent = `${key}: ${value}`;
-      sensorDataContainer.appendChild(dataElement); // Voeg het element toe aan de container
+      sensorDataContainer.appendChild(dataElement); // Add the element to the container
     }
   }
 
   // Update the image
-  function updateImage() {
+  async function updateImage() {
-    var img = document.getElementById("robot-image");
+    let img = document.getElementById("robot-image");
     img.src = "/image?" + new Date().getTime(); // Add timestamp to avoid caching
+    // Wait for 200 milliseconds before fetching the next image
+    setTimeout(updateImage, 200);
   }
 
   // Fetch and display sensor data every 1 second
   setInterval(parseData, 1000);
 
-  // Update the image every 200 milliseconds
+  // Start updating the image
-  setInterval(updateImage, 100);
+  updateImage();
 });