vcc_gnd
/
esp-who
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ feature: color detection

pull/190/head
yuanjiong 2021-11-26 20:08:19 +08:00
parent d30a0e0610
commit ec73674356
12 changed files with 523 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
components/modules/ai/who_color_detection.cpp Normal file
View File

@ -0,0 +1,293 @@
#include "who_color_detection.hpp"
#include "esp_log.h"
#include "esp_camera.h"
#include "dl_image.hpp"
#include "fb_gfx.h"
#include "color_detector.hpp"
#include "who_ai_utils.hpp"
using namespace std;
using namespace dl;
static const char *TAG = "color_detection";
static QueueHandle_t xQueueFrameI = NULL;
static QueueHandle_t xQueueEvent = NULL;
static QueueHandle_t xQueueFrameO = NULL;
static QueueHandle_t xQueueResult = NULL;
static color_detection_state_t gEvent = COLOR_DETECTION_IDLE;
static bool register_mode = false;
static bool gReturnFB = true;
static bool draw_box = true;
static SemaphoreHandle_t xMutex;
vector<color_info_t> std_color_info = {{{156, 10, 70, 255, 90, 255}, 64, "red"},
{{11, 22, 70, 255, 90, 255}, 64, "orange"},
{{23, 33, 70, 255, 90, 255}, 64, "yellow"},
{{34, 75, 70, 255, 90, 255}, 64, "green"},
{{76, 96, 70, 255, 90, 255}, 64, "cyan"},
{{97, 124, 70, 255, 90, 255}, 64, "blue"},
{{125, 155, 70, 255, 90, 255}, 64, "purple"},
{{0, 180, 0, 40, 220, 255}, 64, "white"},
{{0, 180, 0, 50, 50, 219}, 64, "gray"},
// {{0, 180, 0, 255, 0, 45}, 64, "black"}
};
#define RGB565_LCD_RED 0x00F8
#define RGB565_LCD_ORANGE 0x20FD
#define RGB565_LCD_YELLOW 0xE0FF
#define RGB565_LCD_GREEN 0xE007
#define RGB565_LCD_CYAN 0xFF07
#define RGB565_LCD_BLUE 0x1F00
#define RGB565_LCD_PURPLE 0x1EA1
#define RGB565_LCD_WHITE 0xFFFF
#define RGB565_LCD_GRAY 0x1084
#define RGB565_LCD_BLACK 0x0000
#define FRAME_DELAY_NUM 16
static void rgb_print(camera_fb_t *fb, uint32_t color, const char *str)
{
fb_gfx_print(fb, (fb->width - (strlen(str) * 14)) / 2, 10, color, str);
}
static int rgb_printf(camera_fb_t *fb, uint32_t color, const char *format, ...)
{
char loc_buf[64];
char *temp = loc_buf;
int len;
va_list arg;
va_list copy;
va_start(arg, format);
va_copy(copy, arg);
len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg);
va_end(copy);
if (len >= sizeof(loc_buf))
{
temp = (char *)malloc(len + 1);
if (temp == NULL)
{
return 0;
}
}
vsnprintf(temp, len + 1, format, arg);
va_end(arg);
rgb_print(fb, color, temp);
if (len > 64)
{
free(temp);
}
return len;
}
static void draw_color_detection_result(uint16_t *image_ptr, int image_height, int image_width, vector<color_detect_result_t> &results, uint16_t color)
{
for (int i = 0; i < results.size(); ++i)
{
dl::image::draw_hollow_rectangle(image_ptr, image_height, image_width,
results[i].box[0],
results[i].box[1],
results[i].box[2],
results[i].box[3],
color);
}
}
static void task_process_handler(void *arg)
{
camera_fb_t *frame = NULL;
ColorDetector detector;
detector.set_detection_shape({80, 80, 1});
for (int i = 0; i < std_color_info.size(); ++i)
{
detector.register_color(std_color_info[i].color_thresh, std_color_info[i].area_thresh, std_color_info[i].name);
}
vector<vector<int>> color_thresh_boxes = {{110, 110, 130, 130}, {100, 100, 140, 140}, {90, 90, 150, 150}, {80, 80, 160, 160}, {60, 60, 180, 180}, {40, 40, 200, 200}, {20, 20, 220, 220}};
int color_thresh_boxes_num = color_thresh_boxes.size();
int color_thresh_boxes_index = color_thresh_boxes_num / 2;
vector<int> color_area_threshes = {1, 4, 16, 32, 64, 128, 256, 512, 1024};
int color_area_thresh_num = color_area_threshes.size();
int color_area_thresh_index = color_area_thresh_num / 2;
detector.set_area_thresh({color_area_threshes[color_area_thresh_index]});
vector<uint16_t> draw_lcd_colors = {RGB565_LCD_RED,
RGB565_LCD_ORANGE,
RGB565_LCD_YELLOW,
RGB565_LCD_GREEN,
RGB565_LCD_CYAN,
RGB565_LCD_BLUE,
RGB565_LCD_PURPLE,
RGB565_LCD_WHITE,
RGB565_LCD_GRAY,
// RGB565_LCD_BLACK
};
int draw_colors_num = draw_lcd_colors.size();
color_detection_state_t _gEvent;
vector<uint8_t> color_thresh;
while (true)
{
xSemaphoreTake(xMutex, portMAX_DELAY);
_gEvent = gEvent;
gEvent = COLOR_DETECTION_IDLE;
xSemaphoreGive(xMutex);
if (xQueueReceive(xQueueFrameI, &frame, portMAX_DELAY))
{
if (register_mode)
{
switch (_gEvent)
{
case INCREASE_COLOR_AREA:
color_thresh_boxes_index = min(color_thresh_boxes_num - 1, color_thresh_boxes_index + 1);
ets_printf("increase color area\n");
dl::image::draw_hollow_rectangle((uint16_t *)frame->buf, (int)frame->height, (int)frame->width,
color_thresh_boxes[color_thresh_boxes_index][0],
color_thresh_boxes[color_thresh_boxes_index][1],
color_thresh_boxes[color_thresh_boxes_index][2],
color_thresh_boxes[color_thresh_boxes_index][3],
draw_lcd_colors[1]);
break;
case DECREASE_COLOR_AREA:
color_thresh_boxes_index = max(0, color_thresh_boxes_index - 1);
ets_printf("decrease color area\n");
dl::image::draw_hollow_rectangle((uint16_t *)frame->buf, (int)frame->height, (int)frame->width,
color_thresh_boxes[color_thresh_boxes_index][0],
color_thresh_boxes[color_thresh_boxes_index][1],
color_thresh_boxes[color_thresh_boxes_index][2],
color_thresh_boxes[color_thresh_boxes_index][3],
draw_lcd_colors[1]);
break;
case REGISTER_COLOR:
color_thresh = detector.cal_color_thresh((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, color_thresh_boxes[color_thresh_boxes_index]);
detector.register_color(color_thresh);
ets_printf("register color, color_thresh: %d, %d, %d, %d, %d, %d\n", color_thresh[0], color_thresh[1], color_thresh[2], color_thresh[3], color_thresh[4], color_thresh[5]);
xSemaphoreTake(xMutex, portMAX_DELAY);
register_mode = false;
xSemaphoreGive(xMutex);
break;
case CLOSE_REGISTER_COLOR_BOX:
ets_printf("close register color box \n");
xSemaphoreTake(xMutex, portMAX_DELAY);
register_mode = false;
xSemaphoreGive(xMutex);
break;
default:
dl::image::draw_hollow_rectangle((uint16_t *)frame->buf, (int)frame->height, (int)frame->width,
color_thresh_boxes[color_thresh_boxes_index][0],
color_thresh_boxes[color_thresh_boxes_index][1],
color_thresh_boxes[color_thresh_boxes_index][2],
color_thresh_boxes[color_thresh_boxes_index][3],
draw_lcd_colors[1]);
break;
}
}
else
{
switch (_gEvent)
{
case INCREASE_COLOR_AREA:
color_area_thresh_index = min(color_area_thresh_num - 1, color_area_thresh_index + 1);
detector.set_area_thresh({color_area_threshes[color_area_thresh_index]});
ets_printf("increase color area thresh to %d\n", color_area_threshes[color_area_thresh_index]);
break;
case DECREASE_COLOR_AREA:
color_area_thresh_index = max(0, color_area_thresh_index - 1);
detector.set_area_thresh({color_area_threshes[color_area_thresh_index]});
ets_printf("decrease color area thresh to %d\n", color_area_threshes[color_area_thresh_index]);
break;
case DELETE_COLOR:
detector.delete_color();
ets_printf("delete color \n");
break;
default:
std::vector<std::vector<color_detect_result_t>> &results = detector.detect((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3});
if (draw_box)
{
for (int i = 0; i < results.size(); ++i)
{
draw_color_detection_result((uint16_t *)frame->buf, (int)frame->height, (int)frame->width, results[i], draw_lcd_colors[i % draw_colors_num]);
}
}
else
{
detector.draw_segmentation_results((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, draw_lcd_colors, true, 0x0000);
}
break;
}
}
}
if (xQueueFrameO)
{
xQueueSend(xQueueFrameO, &frame, portMAX_DELAY);
}
else if (gReturnFB)
{
esp_camera_fb_return(frame);
}
else
{
free(frame);
}
}
}
static void task_event_handler(void *arg)
{
color_detection_state_t _gEvent;
while (true)
{
xQueueReceive(xQueueEvent, &(_gEvent), portMAX_DELAY);
xSemaphoreTake(xMutex, portMAX_DELAY);
gEvent = _gEvent;
xSemaphoreGive(xMutex);
if (gEvent == OPEN_REGISTER_COLOR_BOX)
{
xSemaphoreTake(xMutex, portMAX_DELAY);
register_mode = true;
xSemaphoreGive(xMutex);
}
else if(gEvent == SWITCH_RESULT)
{
draw_box = !draw_box;
}
}
}
void register_color_detection(const QueueHandle_t frame_i,
const QueueHandle_t event,
const QueueHandle_t result,
const QueueHandle_t frame_o,
const bool camera_fb_return)
{
xQueueFrameI = frame_i;
xQueueFrameO = frame_o;
xQueueEvent = event;
xQueueResult = result;
gReturnFB = camera_fb_return;
xMutex = xSemaphoreCreateMutex();
xTaskCreatePinnedToCore(task_process_handler, TAG, 4 * 1024, NULL, 5, NULL, 1);
if (xQueueEvent)
xTaskCreatePinnedToCore(task_event_handler, TAG, 4 * 1024, NULL, 5, NULL, 1);
}

24
components/modules/ai/who_color_detection.hpp Normal file
View File

@ -0,0 +1,24 @@
#pragma once
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/task.h"
#include "freertos/semphr.h"
typedef enum
{
COLOR_DETECTION_IDLE = 0,
OPEN_REGISTER_COLOR_BOX,
CLOSE_REGISTER_COLOR_BOX,
REGISTER_COLOR,
DELETE_COLOR,
INCREASE_COLOR_AREA,
DECREASE_COLOR_AREA,
SWITCH_RESULT,
} color_detection_state_t;
void register_color_detection(QueueHandle_t frame_i,
QueueHandle_t event,
QueueHandle_t result,
QueueHandle_t frame_o = NULL,
const bool camera_fb_return = false);

16
examples/color_detection/README.md Normal file
View File

@ -0,0 +1,16 @@
# Color Detection Example [[中文]](./README_ZH.md)
This example currently only supports the ESP32-S3-EYE development board. To use other development boards, you need to modify the button input logic configuration yourself.
## How to Use Example
- The interactive buttons are the BOOT , MENU, Play, UP, and DN.
- Press the Boot button: Switch the output display mode.
- Press MENU: Switch between color entry mode and color detection mode.
- Press the PLAY (entry mode): Entry the color in the indicator box.
- Press the PLAY (detection mode): Delete the last color that are entered.
- Press the UP (entry mode): Increase the size of the indicator box.
- Press the UP (detection mode): Increase the area of the smallest color block filtered by color detection.
- Press the DN (entry mode): Decrease the size of the indicator box.
- Press the DN (detection mode): Decrease the area of the smallest color block filtered by color detection.

15
examples/color_detection/README_CN.md Normal file
View File

@ -0,0 +1,15 @@
# Color Detection Example [[English]](./README.md)
目前该示例仅支持ESP32-S3-EYE开发板，若要使用其他开发板，需要自己修改按键输入逻辑配置。
## 如何使用示例
- 交互按键为Boot键，MENU键，PLAY键，UP键和DN键。
- 按下BOOT键：切换输出显示模式。
- 按下MENU：切换录入模式和检测模式。
- 按下PLAY键（录入模式）： 录入指示框内的颜色。
- 按下PLAY键（检测模式）： 删除最后一个被录入的颜色。
- 按下UP键（录入模式）： 增大指示框的尺寸。
- 按下UP键（检测模式）： 增大颜色检测过滤的最小色块的面积。
- 按下DN键（录入模式）： 减小指示框的尺寸。
- 按下DN键（检测模式）： 减小颜色检测过滤的最小色块的面积。

8
examples/color_detection/lcd/CMakeLists.txt Normal file
View File

@ -0,0 +1,8 @@
# The following lines of boilerplate have to be in your project's
# CMakeLists in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.5)
set(EXTRA_COMPONENT_DIRS ../../../components)
add_compile_options(-fdiagnostics-color=always)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(color_detection_lcd)

5
examples/color_detection/lcd/main/CMakeLists.txt Normal file
View File

@ -0,0 +1,5 @@
set(src_dirs .)
set(include_dirs .)
idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs})

38
examples/color_detection/lcd/main/app_main.cpp Executable file
View File

@ -0,0 +1,38 @@
#include "who_camera.h"
#include "who_color_detection.hpp"
#include "who_lcd.h"
#include "who_button.h"
#include "event_logic.hpp"
#include "who_adc_button.h"
static QueueHandle_t xQueueAIFrame = NULL;
static QueueHandle_t xQueueLCDFrame = NULL;
static QueueHandle_t xQueueADCKeyState = NULL;
static QueueHandle_t xQueueGPIOKeyState = NULL;
static QueueHandle_t xQueueEventLogic = NULL;
static button_adc_config_t buttons[4] = {{1, 2800, 3000}, {2, 2250, 2450}, {3, 300, 500}, {4, 850, 1050}};
#define GPIO_BOOT GPIO_NUM_0
extern "C" void app_main()
{
gpio_config_t gpio_conf;
gpio_conf.mode = GPIO_MODE_OUTPUT_OD;
gpio_conf.intr_type = GPIO_INTR_DISABLE;
gpio_conf.pin_bit_mask = 1LL << GPIO_NUM_3;
gpio_config(&gpio_conf);
xQueueAIFrame = xQueueCreate(2, sizeof(camera_fb_t *));
xQueueLCDFrame = xQueueCreate(2, sizeof(camera_fb_t *));
xQueueADCKeyState = xQueueCreate(1, sizeof(int));
xQueueGPIOKeyState = xQueueCreate(1, sizeof(int));
xQueueEventLogic = xQueueCreate(1, sizeof(int));
register_camera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueAIFrame);
register_adc_button(buttons, 4, xQueueADCKeyState);
register_button(GPIO_NUM_0, xQueueGPIOKeyState);
register_event(xQueueADCKeyState, xQueueGPIOKeyState, xQueueEventLogic);
register_color_detection(xQueueAIFrame, xQueueEventLogic, NULL, xQueueLCDFrame, false);
register_lcd(xQueueLCDFrame, NULL, true);
}

74
examples/color_detection/lcd/main/event_logic.cpp Normal file
View File

@ -0,0 +1,74 @@
#include <stdio.h>
#include "event_logic.hpp"
#include "who_button.h"
#include "who_color_detection.hpp"
typedef enum
{
MENU = 1,
PLAY,
UP,
DOWN
} key_name_t;
static QueueHandle_t xQueueKeyStateIGPIO = NULL;
static QueueHandle_t xQueueKeyStateIADC = NULL;
static QueueHandle_t xQueueEventO = NULL;
static key_state_t key_state;
static key_name_t adc_button_name;
static color_detection_state_t detector_state;
void event_generate_from_gpio_button(void *arg)
{
color_detection_state_t switch_mode = SWITCH_RESULT;
while (1)
{
xQueueReceive(xQueueKeyStateIGPIO, &key_state, portMAX_DELAY);
xQueueSend(xQueueEventO, &switch_mode, portMAX_DELAY);
}
}
void event_generate_from_adc_button(void *arg)
{
bool register_mode = false;
while (1)
{
xQueueReceive(xQueueKeyStateIADC, &adc_button_name, portMAX_DELAY);
switch (adc_button_name)
{
case MENU:
detector_state = register_mode ? CLOSE_REGISTER_COLOR_BOX : OPEN_REGISTER_COLOR_BOX;
register_mode = !register_mode;
break;
case PLAY:
detector_state = register_mode ? REGISTER_COLOR : DELETE_COLOR;
register_mode = false;
break;
case UP:
detector_state = INCREASE_COLOR_AREA;
break;
case DOWN:
detector_state = DECREASE_COLOR_AREA;
break;
default:
detector_state = COLOR_DETECTION_IDLE;
break;
}
xQueueSend(xQueueEventO, &detector_state, portMAX_DELAY);
}
}
void register_event(const QueueHandle_t key_state_i_adc, const QueueHandle_t key_state_i_gpio, const QueueHandle_t event_o)
{
xQueueKeyStateIADC = key_state_i_adc;
xQueueKeyStateIGPIO = key_state_i_gpio;
xQueueEventO = event_o;
if (xQueueKeyStateIADC != NULL)
xTaskCreatePinnedToCore(event_generate_from_adc_button, "event_logic_task1", 1024, NULL, 5, NULL, 0);
if (xQueueKeyStateIGPIO != NULL)
xTaskCreatePinnedToCore(event_generate_from_gpio_button, "event_logic_task2", 1024, NULL, 5, NULL, 0);
}

14
examples/color_detection/lcd/main/event_logic.hpp Normal file
View File

@ -0,0 +1,14 @@
#pragma once
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/task.h"
/**
* @brief
*
* @param key_state_i_adc
* @param key_state_i_gpio
* @param event_o
*/
void register_event(const QueueHandle_t key_state_i_adc, const QueueHandle_t key_state_i_gpio, const QueueHandle_t event_o);

5
examples/color_detection/lcd/partitions.csv Normal file
View File

@ -0,0 +1,5 @@
# Name, Type, SubType, Offset, Size, Flags
# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
factory, app, factory, , 3840K
nvs, data, nvs, , 16K
fr, data, , , 128K
1 # Name, Type, SubType, Offset, Size, Flags
2 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
3 factory, app, factory, , 3840K
4 nvs, data, nvs, , 16K
5 fr, data, , , 128K

14
examples/color_detection/lcd/sdkconfig.defaults Normal file
View File

@ -0,0 +1,14 @@
CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
CONFIG_SPIRAM_SPEED_80M=y
CONFIG_SPIRAM_USE_CAPS_ALLOC=y
CONFIG_PARTITION_TABLE_CUSTOM=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
CONFIG_S16=y
CONFIG_ESP_INT_WDT=n
CONFIG_ESP_TASK_WDT=n

17
examples/color_detection/lcd/sdkconfig.defaults.esp32s3 Normal file
View File

@ -0,0 +1,17 @@
CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y
CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
CONFIG_ESP32S3_SPIRAM_SUPPORT=y
CONFIG_ESP32S3_DATA_CACHE_64KB=y
CONFIG_ESP32S3_DATA_CACHE_8WAYS=y
CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
CONFIG_CAMERA_MODULE_ESP_S3_EYE=y
CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y
CONFIG_ESPTOOLPY_NO_STUB=y
CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y
CONFIG_SPIRAM_MODE_OCT=y
CONFIG_S8=y