diff --git a/components/modules/ai/who_color_detection.cpp b/components/modules/ai/who_color_detection.cpp new file mode 100644 index 0000000..5825445 --- /dev/null +++ b/components/modules/ai/who_color_detection.cpp @@ -0,0 +1,293 @@ +#include "who_color_detection.hpp" + +#include "esp_log.h" +#include "esp_camera.h" + +#include "dl_image.hpp" +#include "fb_gfx.h" +#include "color_detector.hpp" + +#include "who_ai_utils.hpp" + +using namespace std; +using namespace dl; + +static const char *TAG = "color_detection"; + +static QueueHandle_t xQueueFrameI = NULL; +static QueueHandle_t xQueueEvent = NULL; +static QueueHandle_t xQueueFrameO = NULL; +static QueueHandle_t xQueueResult = NULL; + +static color_detection_state_t gEvent = COLOR_DETECTION_IDLE; +static bool register_mode = false; +static bool gReturnFB = true; +static bool draw_box = true; + +static SemaphoreHandle_t xMutex; + +vector std_color_info = {{{156, 10, 70, 255, 90, 255}, 64, "red"}, + {{11, 22, 70, 255, 90, 255}, 64, "orange"}, + {{23, 33, 70, 255, 90, 255}, 64, "yellow"}, + {{34, 75, 70, 255, 90, 255}, 64, "green"}, + {{76, 96, 70, 255, 90, 255}, 64, "cyan"}, + {{97, 124, 70, 255, 90, 255}, 64, "blue"}, + {{125, 155, 70, 255, 90, 255}, 64, "purple"}, + {{0, 180, 0, 40, 220, 255}, 64, "white"}, + {{0, 180, 0, 50, 50, 219}, 64, "gray"}, + // {{0, 180, 0, 255, 0, 45}, 64, "black"} + }; + +#define RGB565_LCD_RED 0x00F8 +#define RGB565_LCD_ORANGE 0x20FD +#define RGB565_LCD_YELLOW 0xE0FF +#define RGB565_LCD_GREEN 0xE007 +#define RGB565_LCD_CYAN 0xFF07 +#define RGB565_LCD_BLUE 0x1F00 +#define RGB565_LCD_PURPLE 0x1EA1 +#define RGB565_LCD_WHITE 0xFFFF +#define RGB565_LCD_GRAY 0x1084 +#define RGB565_LCD_BLACK 0x0000 + +#define FRAME_DELAY_NUM 16 + +static void rgb_print(camera_fb_t *fb, uint32_t color, const char *str) +{ + fb_gfx_print(fb, (fb->width - (strlen(str) * 14)) / 2, 10, color, str); +} + +static int rgb_printf(camera_fb_t *fb, uint32_t color, const char *format, ...) +{ + char loc_buf[64]; + char *temp = loc_buf; + int len; + va_list arg; + va_list copy; + va_start(arg, format); + va_copy(copy, arg); + len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg); + va_end(copy); + if (len >= sizeof(loc_buf)) + { + temp = (char *)malloc(len + 1); + if (temp == NULL) + { + return 0; + } + } + vsnprintf(temp, len + 1, format, arg); + va_end(arg); + rgb_print(fb, color, temp); + if (len > 64) + { + free(temp); + } + return len; +} + +static void draw_color_detection_result(uint16_t *image_ptr, int image_height, int image_width, vector &results, uint16_t color) +{ + for (int i = 0; i < results.size(); ++i) + { + dl::image::draw_hollow_rectangle(image_ptr, image_height, image_width, + results[i].box[0], + results[i].box[1], + results[i].box[2], + results[i].box[3], + color); + } +} + +static void task_process_handler(void *arg) +{ + camera_fb_t *frame = NULL; + + ColorDetector detector; + detector.set_detection_shape({80, 80, 1}); + for (int i = 0; i < std_color_info.size(); ++i) + { + detector.register_color(std_color_info[i].color_thresh, std_color_info[i].area_thresh, std_color_info[i].name); + } + + vector> color_thresh_boxes = {{110, 110, 130, 130}, {100, 100, 140, 140}, {90, 90, 150, 150}, {80, 80, 160, 160}, {60, 60, 180, 180}, {40, 40, 200, 200}, {20, 20, 220, 220}}; + int color_thresh_boxes_num = color_thresh_boxes.size(); + int color_thresh_boxes_index = color_thresh_boxes_num / 2; + vector color_area_threshes = {1, 4, 16, 32, 64, 128, 256, 512, 1024}; + int color_area_thresh_num = color_area_threshes.size(); + int color_area_thresh_index = color_area_thresh_num / 2; + + detector.set_area_thresh({color_area_threshes[color_area_thresh_index]}); + + + vector draw_lcd_colors = {RGB565_LCD_RED, + RGB565_LCD_ORANGE, + RGB565_LCD_YELLOW, + RGB565_LCD_GREEN, + RGB565_LCD_CYAN, + RGB565_LCD_BLUE, + RGB565_LCD_PURPLE, + RGB565_LCD_WHITE, + RGB565_LCD_GRAY, + // RGB565_LCD_BLACK + }; + int draw_colors_num = draw_lcd_colors.size(); + + color_detection_state_t _gEvent; + vector color_thresh; + + while (true) + { + xSemaphoreTake(xMutex, portMAX_DELAY); + _gEvent = gEvent; + gEvent = COLOR_DETECTION_IDLE; + xSemaphoreGive(xMutex); + + if (xQueueReceive(xQueueFrameI, &frame, portMAX_DELAY)) + { + if (register_mode) + { + switch (_gEvent) + { + case INCREASE_COLOR_AREA: + color_thresh_boxes_index = min(color_thresh_boxes_num - 1, color_thresh_boxes_index + 1); + ets_printf("increase color area\n"); + dl::image::draw_hollow_rectangle((uint16_t *)frame->buf, (int)frame->height, (int)frame->width, + color_thresh_boxes[color_thresh_boxes_index][0], + color_thresh_boxes[color_thresh_boxes_index][1], + color_thresh_boxes[color_thresh_boxes_index][2], + color_thresh_boxes[color_thresh_boxes_index][3], + draw_lcd_colors[1]); + break; + + case DECREASE_COLOR_AREA: + color_thresh_boxes_index = max(0, color_thresh_boxes_index - 1); + ets_printf("decrease color area\n"); + dl::image::draw_hollow_rectangle((uint16_t *)frame->buf, (int)frame->height, (int)frame->width, + color_thresh_boxes[color_thresh_boxes_index][0], + color_thresh_boxes[color_thresh_boxes_index][1], + color_thresh_boxes[color_thresh_boxes_index][2], + color_thresh_boxes[color_thresh_boxes_index][3], + draw_lcd_colors[1]); + break; + + case REGISTER_COLOR: + color_thresh = detector.cal_color_thresh((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, color_thresh_boxes[color_thresh_boxes_index]); + detector.register_color(color_thresh); + ets_printf("register color, color_thresh: %d, %d, %d, %d, %d, %d\n", color_thresh[0], color_thresh[1], color_thresh[2], color_thresh[3], color_thresh[4], color_thresh[5]); + xSemaphoreTake(xMutex, portMAX_DELAY); + register_mode = false; + xSemaphoreGive(xMutex); + break; + + case CLOSE_REGISTER_COLOR_BOX: + ets_printf("close register color box \n"); + xSemaphoreTake(xMutex, portMAX_DELAY); + register_mode = false; + xSemaphoreGive(xMutex); + break; + + default: + dl::image::draw_hollow_rectangle((uint16_t *)frame->buf, (int)frame->height, (int)frame->width, + color_thresh_boxes[color_thresh_boxes_index][0], + color_thresh_boxes[color_thresh_boxes_index][1], + color_thresh_boxes[color_thresh_boxes_index][2], + color_thresh_boxes[color_thresh_boxes_index][3], + draw_lcd_colors[1]); + break; + } + } + else + { + switch (_gEvent) + { + case INCREASE_COLOR_AREA: + color_area_thresh_index = min(color_area_thresh_num - 1, color_area_thresh_index + 1); + detector.set_area_thresh({color_area_threshes[color_area_thresh_index]}); + ets_printf("increase color area thresh to %d\n", color_area_threshes[color_area_thresh_index]); + break; + + case DECREASE_COLOR_AREA: + color_area_thresh_index = max(0, color_area_thresh_index - 1); + detector.set_area_thresh({color_area_threshes[color_area_thresh_index]}); + ets_printf("decrease color area thresh to %d\n", color_area_threshes[color_area_thresh_index]); + break; + + case DELETE_COLOR: + detector.delete_color(); + ets_printf("delete color \n"); + break; + + default: + std::vector> &results = detector.detect((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}); + if (draw_box) + { + for (int i = 0; i < results.size(); ++i) + { + draw_color_detection_result((uint16_t *)frame->buf, (int)frame->height, (int)frame->width, results[i], draw_lcd_colors[i % draw_colors_num]); + } + } + else + { + detector.draw_segmentation_results((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, draw_lcd_colors, true, 0x0000); + } + break; + } + + } + } + + if (xQueueFrameO) + { + + xQueueSend(xQueueFrameO, &frame, portMAX_DELAY); + } + else if (gReturnFB) + { + esp_camera_fb_return(frame); + } + else + { + free(frame); + } + } +} + +static void task_event_handler(void *arg) +{ + color_detection_state_t _gEvent; + while (true) + { + xQueueReceive(xQueueEvent, &(_gEvent), portMAX_DELAY); + xSemaphoreTake(xMutex, portMAX_DELAY); + gEvent = _gEvent; + xSemaphoreGive(xMutex); + if (gEvent == OPEN_REGISTER_COLOR_BOX) + { + xSemaphoreTake(xMutex, portMAX_DELAY); + register_mode = true; + xSemaphoreGive(xMutex); + } + else if(gEvent == SWITCH_RESULT) + { + draw_box = !draw_box; + } + } +} + +void register_color_detection(const QueueHandle_t frame_i, + const QueueHandle_t event, + const QueueHandle_t result, + const QueueHandle_t frame_o, + const bool camera_fb_return) +{ + xQueueFrameI = frame_i; + xQueueFrameO = frame_o; + xQueueEvent = event; + xQueueResult = result; + gReturnFB = camera_fb_return; + xMutex = xSemaphoreCreateMutex(); + + xTaskCreatePinnedToCore(task_process_handler, TAG, 4 * 1024, NULL, 5, NULL, 1); + if (xQueueEvent) + xTaskCreatePinnedToCore(task_event_handler, TAG, 4 * 1024, NULL, 5, NULL, 1); +} diff --git a/components/modules/ai/who_color_detection.hpp b/components/modules/ai/who_color_detection.hpp new file mode 100644 index 0000000..616ef05 --- /dev/null +++ b/components/modules/ai/who_color_detection.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "freertos/semphr.h" + +typedef enum +{ + COLOR_DETECTION_IDLE = 0, + OPEN_REGISTER_COLOR_BOX, + CLOSE_REGISTER_COLOR_BOX, + REGISTER_COLOR, + DELETE_COLOR, + INCREASE_COLOR_AREA, + DECREASE_COLOR_AREA, + SWITCH_RESULT, +} color_detection_state_t; + +void register_color_detection(QueueHandle_t frame_i, + QueueHandle_t event, + QueueHandle_t result, + QueueHandle_t frame_o = NULL, + const bool camera_fb_return = false); diff --git a/examples/color_detection/README.md b/examples/color_detection/README.md new file mode 100644 index 0000000..52e4f3c --- /dev/null +++ b/examples/color_detection/README.md @@ -0,0 +1,16 @@ +# Color Detection Example [[中文]](./README_ZH.md) + +This example currently only supports the ESP32-S3-EYE development board. To use other development boards, you need to modify the button input logic configuration yourself. + +## How to Use Example + +- The interactive buttons are the BOOT , MENU, Play, UP, and DN. +- Press the Boot button: Switch the output display mode. +- Press MENU: Switch between color entry mode and color detection mode. +- Press the PLAY (entry mode): Entry the color in the indicator box. +- Press the PLAY (detection mode): Delete the last color that are entered. +- Press the UP (entry mode): Increase the size of the indicator box. +- Press the UP (detection mode): Increase the area of the smallest color block filtered by color detection. +- Press the DN (entry mode): Decrease the size of the indicator box. +- Press the DN (detection mode): Decrease the area of the smallest color block filtered by color detection. + diff --git a/examples/color_detection/README_CN.md b/examples/color_detection/README_CN.md new file mode 100644 index 0000000..9eeb4b9 --- /dev/null +++ b/examples/color_detection/README_CN.md @@ -0,0 +1,15 @@ +# Color Detection Example [[English]](./README.md) + +目前该示例仅支持ESP32-S3-EYE开发板，若要使用其他开发板，需要自己修改按键输入逻辑配置。 + +## 如何使用示例 + +- 交互按键为Boot键，MENU键，PLAY键，UP键和DN键。 +- 按下BOOT键：切换输出显示模式。 +- 按下MENU：切换录入模式和检测模式。 +- 按下PLAY键（录入模式）： 录入指示框内的颜色。 +- 按下PLAY键（检测模式）： 删除最后一个被录入的颜色。 +- 按下UP键（录入模式）： 增大指示框的尺寸。 +- 按下UP键（检测模式）： 增大颜色检测过滤的最小色块的面积。 +- 按下DN键（录入模式）： 减小指示框的尺寸。 +- 按下DN键（检测模式）： 减小颜色检测过滤的最小色块的面积。 diff --git a/examples/color_detection/lcd/CMakeLists.txt b/examples/color_detection/lcd/CMakeLists.txt new file mode 100644 index 0000000..bba4acb --- /dev/null +++ b/examples/color_detection/lcd/CMakeLists.txt @@ -0,0 +1,8 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +set(EXTRA_COMPONENT_DIRS ../../../components) +add_compile_options(-fdiagnostics-color=always) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(color_detection_lcd) diff --git a/examples/color_detection/lcd/main/CMakeLists.txt b/examples/color_detection/lcd/main/CMakeLists.txt new file mode 100644 index 0000000..0f71747 --- /dev/null +++ b/examples/color_detection/lcd/main/CMakeLists.txt @@ -0,0 +1,5 @@ +set(src_dirs .) + +set(include_dirs .) + +idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs}) \ No newline at end of file diff --git a/examples/color_detection/lcd/main/app_main.cpp b/examples/color_detection/lcd/main/app_main.cpp new file mode 100755 index 0000000..c351160 --- /dev/null +++ b/examples/color_detection/lcd/main/app_main.cpp @@ -0,0 +1,38 @@ +#include "who_camera.h" +#include "who_color_detection.hpp" +#include "who_lcd.h" +#include "who_button.h" +#include "event_logic.hpp" +#include "who_adc_button.h" + +static QueueHandle_t xQueueAIFrame = NULL; +static QueueHandle_t xQueueLCDFrame = NULL; +static QueueHandle_t xQueueADCKeyState = NULL; +static QueueHandle_t xQueueGPIOKeyState = NULL; +static QueueHandle_t xQueueEventLogic = NULL; +static button_adc_config_t buttons[4] = {{1, 2800, 3000}, {2, 2250, 2450}, {3, 300, 500}, {4, 850, 1050}}; + +#define GPIO_BOOT GPIO_NUM_0 + +extern "C" void app_main() +{ + gpio_config_t gpio_conf; + gpio_conf.mode = GPIO_MODE_OUTPUT_OD; + gpio_conf.intr_type = GPIO_INTR_DISABLE; + gpio_conf.pin_bit_mask = 1LL << GPIO_NUM_3; + gpio_config(&gpio_conf); + + xQueueAIFrame = xQueueCreate(2, sizeof(camera_fb_t *)); + xQueueLCDFrame = xQueueCreate(2, sizeof(camera_fb_t *)); + xQueueADCKeyState = xQueueCreate(1, sizeof(int)); + xQueueGPIOKeyState = xQueueCreate(1, sizeof(int)); + xQueueEventLogic = xQueueCreate(1, sizeof(int)); + + register_camera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueAIFrame); + register_adc_button(buttons, 4, xQueueADCKeyState); + register_button(GPIO_NUM_0, xQueueGPIOKeyState); + register_event(xQueueADCKeyState, xQueueGPIOKeyState, xQueueEventLogic); + register_color_detection(xQueueAIFrame, xQueueEventLogic, NULL, xQueueLCDFrame, false); + register_lcd(xQueueLCDFrame, NULL, true); + +} diff --git a/examples/color_detection/lcd/main/event_logic.cpp b/examples/color_detection/lcd/main/event_logic.cpp new file mode 100644 index 0000000..ed7b084 --- /dev/null +++ b/examples/color_detection/lcd/main/event_logic.cpp @@ -0,0 +1,74 @@ +#include +#include "event_logic.hpp" +#include "who_button.h" +#include "who_color_detection.hpp" + +typedef enum +{ + MENU = 1, + PLAY, + UP, + DOWN +} key_name_t; + +static QueueHandle_t xQueueKeyStateIGPIO = NULL; +static QueueHandle_t xQueueKeyStateIADC = NULL; +static QueueHandle_t xQueueEventO = NULL; +static key_state_t key_state; +static key_name_t adc_button_name; +static color_detection_state_t detector_state; + +void event_generate_from_gpio_button(void *arg) +{ + color_detection_state_t switch_mode = SWITCH_RESULT; + while (1) + { + xQueueReceive(xQueueKeyStateIGPIO, &key_state, portMAX_DELAY); + xQueueSend(xQueueEventO, &switch_mode, portMAX_DELAY); + } +} + +void event_generate_from_adc_button(void *arg) +{ + bool register_mode = false; + while (1) + { + xQueueReceive(xQueueKeyStateIADC, &adc_button_name, portMAX_DELAY); + switch (adc_button_name) + { + case MENU: + detector_state = register_mode ? CLOSE_REGISTER_COLOR_BOX : OPEN_REGISTER_COLOR_BOX; + register_mode = !register_mode; + break; + + case PLAY: + detector_state = register_mode ? REGISTER_COLOR : DELETE_COLOR; + register_mode = false; + break; + + case UP: + detector_state = INCREASE_COLOR_AREA; + break; + + case DOWN: + detector_state = DECREASE_COLOR_AREA; + break; + + default: + detector_state = COLOR_DETECTION_IDLE; + break; + } + xQueueSend(xQueueEventO, &detector_state, portMAX_DELAY); + } +} + +void register_event(const QueueHandle_t key_state_i_adc, const QueueHandle_t key_state_i_gpio, const QueueHandle_t event_o) +{ + xQueueKeyStateIADC = key_state_i_adc; + xQueueKeyStateIGPIO = key_state_i_gpio; + xQueueEventO = event_o; + if (xQueueKeyStateIADC != NULL) + xTaskCreatePinnedToCore(event_generate_from_adc_button, "event_logic_task1", 1024, NULL, 5, NULL, 0); + if (xQueueKeyStateIGPIO != NULL) + xTaskCreatePinnedToCore(event_generate_from_gpio_button, "event_logic_task2", 1024, NULL, 5, NULL, 0); +} \ No newline at end of file diff --git a/examples/color_detection/lcd/main/event_logic.hpp b/examples/color_detection/lcd/main/event_logic.hpp new file mode 100644 index 0000000..4ed980d --- /dev/null +++ b/examples/color_detection/lcd/main/event_logic.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" + +/** + * @brief + * + * @param key_state_i_adc + * @param key_state_i_gpio + * @param event_o + */ +void register_event(const QueueHandle_t key_state_i_adc, const QueueHandle_t key_state_i_gpio, const QueueHandle_t event_o); \ No newline at end of file diff --git a/examples/color_detection/lcd/partitions.csv b/examples/color_detection/lcd/partitions.csv new file mode 100644 index 0000000..44ffcd0 --- /dev/null +++ b/examples/color_detection/lcd/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +factory, app, factory, , 3840K +nvs, data, nvs, , 16K +fr, data, , , 128K diff --git a/examples/color_detection/lcd/sdkconfig.defaults b/examples/color_detection/lcd/sdkconfig.defaults new file mode 100644 index 0000000..ed577b9 --- /dev/null +++ b/examples/color_detection/lcd/sdkconfig.defaults @@ -0,0 +1,14 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y + +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y + +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +CONFIG_S16=y +CONFIG_ESP_INT_WDT=n +CONFIG_ESP_TASK_WDT=n \ No newline at end of file diff --git a/examples/color_detection/lcd/sdkconfig.defaults.esp32s3 b/examples/color_detection/lcd/sdkconfig.defaults.esp32s3 new file mode 100644 index 0000000..cb7a777 --- /dev/null +++ b/examples/color_detection/lcd/sdkconfig.defaults.esp32s3 @@ -0,0 +1,17 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y + +CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S3_SPIRAM_SUPPORT=y + +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_8WAYS=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y + +CONFIG_CAMERA_MODULE_ESP_S3_EYE=y +CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y +CONFIG_ESPTOOLPY_NO_STUB=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y + +CONFIG_SPIRAM_MODE_OCT=y + +CONFIG_S8=y