diff --git a/.gitignore b/.gitignore index 9f06cae..917df1e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ src/Socket/a.out src/C++/Driver/cmake_install.cmake src/C++/Socket/a.out src/C++/Driver/Makefile -src/C++/Driver/vgcore* +vgcore* src/C++/Driver/cmake_install.cmake src/C++/Driver/Makefile src/C++/Driver/log @@ -31,3 +31,4 @@ CMakeFiles/ Makefile CMakeCache.txt cmake_install.cmake +src/C++/OpenCV/main diff --git a/docs/code/OpenCV.md b/docs/code/OpenCV.md new file mode 100644 index 0000000..5b087a2 --- /dev/null +++ b/docs/code/OpenCV.md @@ -0,0 +1,20 @@ +# OpenCV +## Requirements +For the camera we want it to detect what is happening on the video feed and identify it so it can identify dangers. + + +## Issues + +* OpenCL not grabbing gpu +* Solution: https://github.com/Smorodov/Multitarget-tracker/issues/93 + +## Installation +### Dependencies +* glew +* opencv + + + + +## Sources +* https://github.com/UnaNancyOwen/OpenCVDNNSample/tree/master \ No newline at end of file diff --git a/src/C++/Driver/CMakeLists.txt b/src/C++/Driver/CMakeLists.txt index e6c35e2..78d5be1 100644 --- a/src/C++/Driver/CMakeLists.txt +++ b/src/C++/Driver/CMakeLists.txt @@ -6,7 +6,10 @@ set(CMAKE_CXX_STANDARD 23) find_library(PAHO_MQTTPP_LIBRARY paho-mqttpp3 PATHS /usr/local/lib) find_library(PAHO_MQTT_LIBRARY paho-mqtt3a PATHS /usr/local/lib) -include_directories(/usr/local/include) +# Find OpenCV package +find_package(OpenCV REQUIRED) +find_package(OpenEXR REQUIRED) +include_directories(${OpenCV_INCLUDE_DIRS}) set(SOURCE_FILES src/KobukiDriver/KobukiParser.cpp @@ -20,4 +23,4 @@ set(SOURCE_FILES add_executable(kobuki_control ${SOURCE_FILES}) # Link the static libraries -target_link_libraries(kobuki_control ${PAHO_MQTTPP_LIBRARY} ${PAHO_MQTT_LIBRARY} pthread) \ No newline at end of file +target_link_libraries(kobuki_control ${PAHO_MQTTPP_LIBRARY} ${PAHO_MQTT_LIBRARY} ${OpenCV_LIBS} pthread OpenEXR::OpenEXR) diff --git a/src/C++/Driver/src/KobukiDriver/CKobuki.cpp b/src/C++/Driver/src/KobukiDriver/CKobuki.cpp index 429e7f9..587c50c 100755 --- a/src/C++/Driver/src/KobukiDriver/CKobuki.cpp +++ b/src/C++/Driver/src/KobukiDriver/CKobuki.cpp @@ -509,32 +509,6 @@ void CKobuki::doRotation(long double th) { usleep(25 * 1000); } -// combines navigation to a coordinate and rotation by an angle, performs -// movement to the selected coordinate in the robot's coordinate system -void CKobuki::goToXy(long double xx, long double yy) { - long double th; - - yy = yy * -1; - - th = atan2(yy, xx); - doRotation(th); - - long double s = sqrt(pow(xx, 2) + pow(yy, 2)); - - // resetnem suradnicovu sustavu robota - x = 0; - y = 0; - iterationCount = 0; - theta = 0; - - // std::cout << "mam prejst: " << s << "[m]" << std::endl; - - goStraight(s); - - usleep(25 * 1000); - return; -} - /// @brief Makes the robot move forward for 3 seconds /// @param speedvalue How fast it will drive forward from 0 - 1024 void CKobuki::forward(int speedvalue) { diff --git a/src/C++/Driver/src/KobukiDriver/CKobuki.h b/src/C++/Driver/src/KobukiDriver/CKobuki.h index 0567df1..a3fb1c1 100755 --- a/src/C++/Driver/src/KobukiDriver/CKobuki.h +++ b/src/C++/Driver/src/KobukiDriver/CKobuki.h @@ -31,7 +31,6 @@ #include #include #include "KobukiParser.h" -#include "graph.h" using namespace std; diff --git a/src/C++/Driver/src/KobukiDriver/graph.h b/src/C++/Driver/src/KobukiDriver/graph.h deleted file mode 100644 index 89c963c..0000000 --- a/src/C++/Driver/src/KobukiDriver/graph.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef GRAPH1010 -#define GRAPH1010 -#include -#include -#include - -using namespace std; -#define GRAPH_ENABLED true - -class plot { -public: - FILE *gp; - bool enabled,persist; - plot(bool _persist=false,bool _enabled=GRAPH_ENABLED) { - enabled=_enabled; - persist=_persist; - if (enabled) { - if(persist) - gp=popen("gnuplot -persist","w"); - else - gp=popen("gnuplot","w"); - } - } - - void plot_data(vector x,const char* style="points",const char* title="Data") { - if(!enabled) - return; - fprintf(gp,"set title '%s' \n",title); - fprintf(gp,"plot '-' w %s \n",style); - for(int k=0;k x,vector y,const char* style="points",const char* title="Data") { - if(!enabled) - return; - fprintf(gp,"set title '%s' \n",title); - fprintf(gp,"plot '-' w %s \n",style); - for(int k=0;k x,y; - for(int k=a;k #include #include -#include "KobukiDriver/graph.h" #include "MQTT/MqttClient.h" #include "KobukiDriver/CKobuki.h" +#include +#include using namespace std; +using namespace cv; CKobuki robot; std::string readMQTT(); void parseMQTT(std::string message); +void CapnSend(); //ip, clientID, username, password -MqttClient client("mqtt://145.92.224.21:1884", "KobukiRPI", "rpi", "rpiwachtwoordofzo"); // create a client object +MqttClient client("ws://145.92.224.21/ws/", "KobukiRPI", "rpi", "rpiwachtwoordofzo"); // create a client object std::string message = "stop"; std::string serializeKobukiData(const TKobukiData &data); void sendKobukiData(TKobukiData &data); @@ -20,6 +23,7 @@ void setup() unsigned char *null_ptr(0); robot.startCommunication("/dev/ttyUSB0", true, null_ptr); //connect mqtt server and sub to commands + client.connect(); client.subscribe("home/commands"); } @@ -27,15 +31,17 @@ void setup() int main() { setup(); + std::thread image (CapnSend); std::thread safety([&]() { robot.robotSafety(&message); }); std::thread sendMqtt([&]() { sendKobukiData(robot.parser.data); }); while(true){ - parseMQTT(readMQTT()); + parseMQTT(readMQTT()); } + sendMqtt.join(); safety.join(); - return 0; + image.join(); } std::string readMQTT() @@ -55,7 +61,7 @@ void parseMQTT(std::string message) { if (message == "up") { - robot.forward(1024); + robot.forward(350); } else if (message == "left") { @@ -67,7 +73,7 @@ void parseMQTT(std::string message) } else if (message == "down") { - robot.forward(-800); + robot.forward(-350); } else if (message == "stop") { @@ -270,6 +276,35 @@ std::string serializeKobukiData(const TKobukiData &data) { void sendKobukiData(TKobukiData &data) { while (true) { client.publishMessage("kobuki/data", serializeKobukiData(data)); + std::cout << "Sent data" << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } } + +void CapnSend() { + VideoCapture cap(0); + if (!cap.isOpened()) { + cerr << "Error: Could not open camera" << endl; + return; + } + + Mat frame; + while (true) { + cap >> frame; // Capture a new image frame + if (frame.empty()) { + cerr << "Error: Could not capture image" << endl; + continue; + } + + // Convert the image to a byte array + vector buf; + imencode(".jpg", frame, buf); + auto* enc_msg = reinterpret_cast(buf.data()); + + // Publish the image data + client.publishMessage("kobuki/cam", string(enc_msg, enc_msg + buf.size())); + cout << "Sent image" << endl; + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); // Send image every 1000ms + } +} \ No newline at end of file diff --git a/src/C++/MQTT/CMakeLists.txt b/src/C++/MQTT/CMakeLists.txt deleted file mode 100644 index 1f31293..0000000 --- a/src/C++/MQTT/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -set(CMAKE_CXX_STANDARD 23) - -# Find the Paho MQTT C++ library -find_library(PAHO_MQTTPP_LIBRARY paho-mqttpp3 PATHS /usr/local/lib) -find_library(PAHO_MQTT_LIBRARY paho-mqtt3a PATHS /usr/local/lib) - -# Include the headers -include_directories(/usr/local/include) - -# Add the executable -add_executable(my_program main.cpp) - -# Link the libraries -target_link_libraries(my_program ${PAHO_MQTTPP_LIBRARY} ${PAHO_MQTT_LIBRARY}) diff --git a/src/C++/MQTT/main.cpp b/src/C++/MQTT/main.cpp deleted file mode 100644 index 94747e9..0000000 --- a/src/C++/MQTT/main.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include -#include -#include // For std::this_thread::sleep_for -#include // For std::chrono::seconds - -// Define the address of the MQTT broker, the client ID, and the topic to subscribe to. -const std::string ADDRESS("mqtt://localhost:1883"); // Broker address (Raspberry Pi) -const std::string CLIENT_ID("raspberry_pi_client"); -const std::string TOPIC("home/commands"); - -// Define a callback class that handles incoming messages and connection events. -class callback : public virtual mqtt::callback { - // Called when a message arrives on a subscribed topic. - void message_arrived(mqtt::const_message_ptr msg) override { - std::cout << "Received message: '" << msg->get_topic()<< "' : " << msg->to_string() << std::endl; - } - - // Called when the connection to the broker is lost. - void connection_lost(const std::string& cause) override { - std::cerr << "Connection lost. Reason: " << cause << std::endl; - } - - // Called when a message delivery is complete. - void delivery_complete(mqtt::delivery_token_ptr token) override { - std::cout << "Message delivered!" << std::endl; - } -}; - -int main() { - // Create an MQTT async client and set up the callback class. - mqtt::async_client client(ADDRESS, CLIENT_ID); - callback cb; - client.set_callback(cb); - - // Set up the connection options (such as username and password). - mqtt::connect_options connOpts; - connOpts.set_clean_session(true); - connOpts.set_user_name("ishak"); - connOpts.set_password("kobuki"); - connOpts.set_mqtt_version(MQTTVERSION_3_1_1); - - try { - // Try to connect to the broker and wait until successful. - std::cout << "Connecting to broker..." << std::endl; - client.connect(connOpts)->wait(); // Connect with the provided options - std::cout << "Connected!" << std::endl; - - // Subscribe to the specified topic and wait for confirmation. - std::cout << "Subscribing to topic: " << TOPIC << std::endl; - client.subscribe(TOPIC, 1)->wait(); // Subscribe with QoS level 1 - - // Keep the program running to continue receiving messages from the broker. - while (true) { - std::this_thread::sleep_for(std::chrono::seconds(1)); // Sleep to reduce CPU usage - } - - } catch (const mqtt::exception &exc) { - // Catch any MQTT exceptions and display the error message. - std::cerr << "Error: " << exc.what() << std::endl; - return 1; - } - - return 0; // Return 0 to indicate successful execution -} diff --git a/src/C++/OpenCV/CMakeLists.txt b/src/C++/OpenCV/CMakeLists.txt new file mode 100644 index 0000000..ac79831 --- /dev/null +++ b/src/C++/OpenCV/CMakeLists.txt @@ -0,0 +1,44 @@ +cmake_minimum_required( VERSION 3.6 ) + +# Require C++11 (or later) +set( CMAKE_CXX_STANDARD 23 ) +set( CMAKE_CXX_STANDARD_REQUIRED ON ) +set( CMAKE_CXX_EXTENSIONS OFF ) +set(BUILD_MODE Debug) +# Create Project +project( Sample ) +add_executable( YOLOv4 util.h main.cpp ) + +# Set StartUp Project +set_property( DIRECTORY PROPERTY VS_STARTUP_PROJECT "YOLOv4" ) + +# Find Package +# OpenCV +find_package( OpenCV REQUIRED ) + +if( OpenCV_FOUND ) + # Additional Include Directories + include_directories( ${OpenCV_INCLUDE_DIRS} ) + + # Additional Dependencies + target_link_libraries( YOLOv4 ${OpenCV_LIBS} ) +endif() + +# Download Model +set( MODEL https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights ) +file( DOWNLOAD + "${MODEL}" + "${CMAKE_CURRENT_LIST_DIR}/yolov4.weights" + EXPECTED_HASH SHA256=e8a4f6c62188738d86dc6898d82724ec0964d0eb9d2ae0f0a9d53d65d108d562 + SHOW_PROGRESS +) + + +# Download Config +set( CONFIG https://raw.githubusercontent.com/AlexeyAB/darknet/master/cfg/yolov4.cfg ) +file( DOWNLOAD + "${CONFIG}" + "${CMAKE_CURRENT_LIST_DIR}/yolov4.cfg" + EXPECTED_HASH SHA256=a6d0f8e5c62cc8378384f75a8159b95fa2964d4162e33351b00ac82e0fc46a34 + SHOW_PROGRESS +) diff --git a/src/C++/OpenCV/YOLOv4 b/src/C++/OpenCV/YOLOv4 new file mode 100755 index 0000000..943662c Binary files /dev/null and b/src/C++/OpenCV/YOLOv4 differ diff --git a/src/C++/OpenCV/coco.names b/src/C++/OpenCV/coco.names new file mode 100644 index 0000000..ca76c80 --- /dev/null +++ b/src/C++/OpenCV/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/src/C++/OpenCV/main.cpp b/src/C++/OpenCV/main.cpp new file mode 100644 index 0000000..2f340fe --- /dev/null +++ b/src/C++/OpenCV/main.cpp @@ -0,0 +1,209 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" + +// Helper function to check if a file exists +bool fileExists(const std::string &path) +{ + return std::filesystem::exists(path); +} + +// Function to read class names from a file +std::vector _readClassNameList(const std::string &path) +{ + std::vector classes; + + // Check if file exists + if (!fileExists(path)) + { + throw std::runtime_error("Class names file not found: " + path); + } + + // Try to open and read file + std::ifstream file(path); + if (!file.is_open()) + { + throw std::runtime_error("Unable to open class names file: " + path); + } + + std::string line; + while (std::getline(file, line)) + { + if (!line.empty()) + { + classes.push_back(line); + } + } + + if (classes.empty()) + { + throw std::runtime_error("No classes found in file: " + path); + } + + return classes; +} + +int main(int argc, char *argv[]) +{ + try + { + // Open Video Capture + cv::VideoCapture capture = cv::VideoCapture(0); + if (!capture.isOpened()) + { + std::cerr << "Failed to open camera device" << std::endl; + return -1; + } + + // Read Class Name List and Color Table + const std::string list = "coco.names"; + const std::vector classes = _readClassNameList(list); + const std::vector colors = getClassColors(classes.size()); + + // Debug: Print the size of the colors vector + std::cout << "Number of colors: " << colors.size() << std::endl; + + // Read Darknet + const std::string model = "yolov4.weights"; + const std::string config = "yolov4.cfg"; + cv::dnn::Net net = cv::dnn::readNet(model, config); + if (net.empty()) + { + std::cerr << "Failed to load network" << std::endl; + return -1; + } + + // Set Preferable Backend + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + + // Set Preferable Target + net.setPreferableTarget(cv::dnn::DNN_TARGET_OPENCL); + + while (true) + { + // Read Frame + cv::Mat frame; + capture >> frame; + if (frame.empty()) + { + cv::waitKey(0); + break; + } + if (frame.channels() == 4) + { + cv::cvtColor(frame, frame, cv::COLOR_BGRA2BGR); + } + + // Create Blob from Input Image + cv::Mat blob = cv::dnn::blobFromImage(frame, 1 / 255.f, cv::Size(416, 416), cv::Scalar(), true, false); + + // Set Input Blob + net.setInput(blob); + + // Run Forward Network + std::vector detections; + net.forward(detections, getOutputsNames(net)); + + // Draw Region + std::vector class_ids; + std::vector confidences; + std::vector rectangles; + for (cv::Mat &detection : detections) + { + if (detection.empty()) + { + std::cerr << "Detection matrix is empty!" << std::endl; + continue; + } + for (int32_t i = 0; i < detection.rows; i++) + { + cv::Mat region = detection.row(i); + + // Retrieve Max Confidence and Class Index + cv::Mat scores = region.colRange(5, detection.cols); + cv::Point class_id; + double confidence; + cv::minMaxLoc(scores, 0, &confidence, 0, &class_id); + + // Check Confidence + constexpr float threshold = 0.2; + if (threshold > confidence) + { + continue; + } + + // Retrieve Object Position + const int32_t x_center = static_cast(region.at(0) * frame.cols); + const int32_t y_center = static_cast(region.at(1) * frame.rows); + const int32_t width = static_cast(region.at(2) * frame.cols); + const int32_t height = static_cast(region.at(3) * frame.rows); + const cv::Rect rectangle = cv::Rect(x_center - (width / 2), y_center - (height / 2), width, height); + + // Add Class ID, Confidence, Rectangle + class_ids.push_back(class_id.x); + confidences.push_back(confidence); + rectangles.push_back(rectangle); + } + } + + // Remove Overlap Rectangles using Non-Maximum Suppression + constexpr float confidence_threshold = 0.5; // Confidence + constexpr float nms_threshold = 0.5; // IoU (Intersection over Union) + std::vector indices; + cv::dnn::NMSBoxes(rectangles, confidences, confidence_threshold, nms_threshold, indices); + + // Draw Rectangle + for (const int32_t &index : indices) + { + // Bounds checking + if (class_ids[index] >= colors.size()) + { + std::cerr << "Color index out of bounds: " << class_ids[index] << " (max: " << colors.size() - 1 << ")" << std::endl; + continue; + } + + const cv::Rect rectangle = rectangles[index]; + const cv::Scalar color = colors[class_ids[index]]; + + // Debug: Print the index and color + std::cout << "Drawing rectangle with color index: " << class_ids[index] << std::endl; + + constexpr int32_t thickness = 3; + cv::rectangle(frame, rectangle, color, thickness); + + std::string label = classes[class_ids[index]] + ": " + std::to_string(static_cast(confidences[index] * 100)) + "%"; + + int baseLine; + cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + int top = std::max(rectangle.y, labelSize.height); + cv::rectangle(frame, cv::Point(rectangle.x, top - labelSize.height), + cv::Point(rectangle.x + labelSize.width, top + baseLine), color, cv::FILLED); + cv::putText(frame, label, cv::Point(rectangle.x, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 255, 255), 1); + } + + // Show Image + cv::imshow("Object Detection", frame); + const int32_t key = cv::waitKey(1); + if (key == 'q') + { + break; + } + } + + cv::destroyAllWindows(); + return 0; + } + catch (const std::exception &e) + { + std::cerr << "Error: " << e.what() << std::endl; + return -1; + } +} + +// cloned and fixed from https://github.com/UnaNancyOwen/OpenCVDNNSample/tree/master \ No newline at end of file diff --git a/src/C++/OpenCV/util.h b/src/C++/OpenCV/util.h new file mode 100644 index 0000000..cfbf904 --- /dev/null +++ b/src/C++/OpenCV/util.h @@ -0,0 +1,61 @@ +#ifndef __UTIL__ +#define __UTIL__ + +#include +#include +#include +#include +#include +#include + +// Get Output Layers Name +std::vector getOutputsNames( const cv::dnn::Net& net ) +{ + static std::vector names; + if( names.empty() ){ + std::vector out_layers = net.getUnconnectedOutLayers(); + std::vector layers_names = net.getLayerNames(); + names.resize( out_layers.size() ); + for( size_t i = 0; i < out_layers.size(); ++i ){ + names[i] = layers_names[out_layers[i] - 1]; + } + } + return names; +} + +// Get Output Layer Type +std::string getOutputLayerType( cv::dnn::Net& net ) +{ + const std::vector out_layers = net.getUnconnectedOutLayers(); + const std::string output_layer_type = net.getLayer( out_layers[0] )->type; + return output_layer_type; +} + +// Read Class Name List +std::vector readClassNameList( const std::string list_path ) +{ + std::vector classes; + std::ifstream ifs( list_path ); + if( !ifs.is_open() ){ + return classes; + } + std::string class_name = ""; + while( std::getline( ifs, class_name ) ){ + classes.push_back( class_name ); + } + return classes; +} + +// Get Class Color Table for Visualize +std::vector getClassColors( const int32_t number_of_colors ) +{ + cv::RNG random; + std::vector colors; + for( int32_t i = 0; i < number_of_colors; i++ ){ + cv::Scalar color( random.uniform( 0, 255 ), random.uniform( 0, 255 ), random.uniform( 0, 255 ) ); + colors.push_back( color ); + } + return colors; +} + +#endif // __UTIL__ diff --git a/src/C++/OpenCV/yolov4.cfg b/src/C++/OpenCV/yolov4.cfg new file mode 100644 index 0000000..a7be12b --- /dev/null +++ b/src/C++/OpenCV/yolov4.cfg @@ -0,0 +1,1158 @@ +[net] +batch=64 +subdivisions=8 +# Training +#width=512 +#height=512 +width=608 +height=608 +channels=3 +momentum=0.949 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.0013 +burn_in=1000 +max_batches = 500500 +policy=steps +steps=400000,450000 +scales=.1,.1 + +#cutmix=1 +mosaic=1 + +#:104x104 54:52x52 85:26x26 104:13x13 for 416 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-7 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-10 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-28 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -2 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=mish + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=mish + +[route] +layers = -1,-16 + +[convolutional] +batch_normalize=1 +filters=1024 +size=1 +stride=1 +pad=1 +activation=mish + +########################## + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +### SPP ### +[maxpool] +stride=1 +size=5 + +[route] +layers=-2 + +[maxpool] +stride=1 +size=9 + +[route] +layers=-4 + +[maxpool] +stride=1 +size=13 + +[route] +layers=-1,-3,-5,-6 +### End SPP ### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 85 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = 54 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[route] +layers = -1, -3 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +########################## + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.2 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=256 +activation=leaky + +[route] +layers = -1, -16 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +scale_x_y = 1.1 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +size=3 +stride=2 +pad=1 +filters=512 +activation=leaky + +[route] +layers = -1, -37 + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 +scale_x_y = 1.05 +iou_thresh=0.213 +cls_normalizer=1.0 +iou_normalizer=0.07 +iou_loss=ciou +nms_kind=greedynms +beta_nms=0.6 +max_delta=5 + diff --git a/src/C++/OpenCV/yolov4.weights b/src/C++/OpenCV/yolov4.weights new file mode 100644 index 0000000..921f440 Binary files /dev/null and b/src/C++/OpenCV/yolov4.weights differ diff --git a/src/Python/flask/web/app.py b/src/Python/flask/web/app.py index 3fc45f6..f6350ab 100644 --- a/src/Python/flask/web/app.py +++ b/src/Python/flask/web/app.py @@ -1,13 +1,15 @@ -from flask import Flask, request, render_template, jsonify +from flask import Flask, Response, request, render_template, jsonify import paho.mqtt.client as mqtt app = Flask(__name__) kobuki_message = "empty" def on_message(client, userdata, message): - global kobuki_message #set scope for this variable - kobuki_message = str(message.payload.decode("utf-8")) - print(kobuki_message) + global kobuki_message, latest_image + if message.topic == "kobuki/data": + kobuki_message = str(message.payload.decode("utf-8")) + elif message.topic == "kobuki/cam": + latest_image = message.payload # Create an MQTT client instance mqtt_client = mqtt.Client() @@ -15,6 +17,8 @@ mqtt_client.username_pw_set("server", "serverwachtwoordofzo") mqtt_client.connect("localhost", 80, 60) mqtt_client.loop_start() mqtt_client.subscribe("kobuki/data") +mqtt_client.subscribe("kobuki/cam") + mqtt_client.on_message = on_message # this lines needs to be under the function definition otherwise it cant find which function it needs to use @app.route('/') @@ -45,6 +49,16 @@ def move(): def data(): return kobuki_message + +@app.route('/image') +def image(): + global latest_image + if latest_image is not None: + return Response(latest_image, mimetype='image/jpeg') + else: + return "No image available", 404 + + @app.route('/phpmyadmin/') def phpmyadmin_passthrough(path): # Laat Apache deze route direct afhandelen @@ -52,5 +66,6 @@ def phpmyadmin_passthrough(path): + if __name__ == '__main__': app.run(debug=True, port=5000) diff --git a/src/Python/flask/web/static/script.js b/src/Python/flask/web/static/script.js index 109916d..db8cd8c 100644 --- a/src/Python/flask/web/static/script.js +++ b/src/Python/flask/web/static/script.js @@ -1,24 +1,25 @@ -// Selecteer alle knoppen en voeg een event listener toe aan elke knop -document.querySelectorAll(".btn").forEach(button => { - button.addEventListener("click", function(event) { - event.preventDefault(); // voorkomt pagina-verversing +document.addEventListener("DOMContentLoaded", function() { + document.querySelectorAll(".btn").forEach(button => { + button.addEventListener("click", function(event) { + event.preventDefault(); // prevents page refresh - // Haal de waarde van de knop op - const direction = event.target.value; + // Get the value of the button + const direction = event.target.value; - fetch("/move", { - method: "POST", - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify({ direction: direction }) - }) - .then(response => response.json()) - .then(data => { - console.log("Success:", data); - }) - .catch(error => { - console.error("Error:", error); + fetch("/move", { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ direction: direction }) + }) + .then(response => response.json()) + .then(data => {script + console.log("Success:", data); + }) + .catch(error => { + console.error("Error:", error); + }); }); }); @@ -34,7 +35,7 @@ document.querySelectorAll(".btn").forEach(button => { const data = await fetchData(); const sensorDataContainer = document.getElementById("sensor-data"); sensorDataContainer.innerHTML = ""; // Clear previous data - //for each object in json array create a new paragraph element and append it to the sensorDataContainer + // For each object in JSON array, create a new paragraph element and append it to the sensorDataContainer for (const [key, value] of Object.entries(data)) { const dataElement = document.createElement("p"); dataElement.textContent = `${key}: ${value}`; @@ -42,6 +43,15 @@ document.querySelectorAll(".btn").forEach(button => { } } + // Update the image + function updateImage() { + var img = document.getElementById("robot-image"); + img.src = "/image?" + new Date().getTime(); // Add timestamp to avoid caching + } + // Fetch and display sensor data every 5 seconds - setInterval(parseData, 5000); + setInterval(parseData, 1000); + + // Update the image every 5 seconds + setInterval(updateImage, 200); }); \ No newline at end of file diff --git a/src/Python/flask/web/static/style.css b/src/Python/flask/web/static/style.css index dc848db..2ab49cb 100644 --- a/src/Python/flask/web/static/style.css +++ b/src/Python/flask/web/static/style.css @@ -167,4 +167,4 @@ th,td { th { background-color: #f2f2f2; text-align: left; -} +} \ No newline at end of file diff --git a/src/Python/flask/web/templates/control.html b/src/Python/flask/web/templates/control.html index 519402f..93b9b46 100644 --- a/src/Python/flask/web/templates/control.html +++ b/src/Python/flask/web/templates/control.html @@ -1,6 +1,8 @@ -{% extends 'base.html' %} {% block head %} +{% extends 'base.html' %} +{% block head %} -{% endblock %} {% block content %} +{% endblock %} +{% block content %} @@ -11,8 +13,8 @@
-
- Kobuki Robot +
+ Kobuki Camera Feed
@@ -42,7 +44,8 @@
+ -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/src/config/mosquitto.conf b/src/config/mosquitto.conf new file mode 100644 index 0000000..8e4330a --- /dev/null +++ b/src/config/mosquitto.conf @@ -0,0 +1,7 @@ +allow_anonymous false +password_file /etc/mosquitto/passwordfile +listener 8080 +protocol websockets + +listener 1884 +protocol mqtt diff --git a/src/config/nginx-sites.conf b/src/config/nginx-sites.conf new file mode 100644 index 0000000..e213464 --- /dev/null +++ b/src/config/nginx-sites.conf @@ -0,0 +1,22 @@ +server { + listen 80; + server_name 145.92.224.21; + + # Proxy WebSocket connections for MQTT + location /ws/ { + proxy_pass http://localhost:9001; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + + # Proxy HTTP connections for Flask + location / { + proxy_pass http://localhost:5000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/src/config/nginx.conf b/src/config/nginx.conf new file mode 100644 index 0000000..0f13dc0 --- /dev/null +++ b/src/config/nginx.conf @@ -0,0 +1,7 @@ +stream { + server { + listen 9001; + proxy_pass localhost:8080; + } +} +