diff --git a/components/esp-dl b/components/esp-dl index bad67b5..c4735f4 160000 --- a/components/esp-dl +++ b/components/esp-dl @@ -1 +1 @@ -Subproject commit bad67b57f8123e76c740ba64e0abe39aff71509d +Subproject commit c4735f45b64dd529006db116c0bf1d695e34de63 diff --git a/components/fb_gfx/CMakeLists.txt b/components/fb_gfx/CMakeLists.txt index 5b6febc..4edefcb 100644 --- a/components/fb_gfx/CMakeLists.txt +++ b/components/fb_gfx/CMakeLists.txt @@ -1,5 +1,5 @@ -set(COMPONENT_SRCS "fb_gfx.c") -set(COMPONENT_ADD_INCLUDEDIRS "include") -set(COMPONENT_PRIV_INCLUDEDIRS "") -set(COMPONENT_PRIV_REQUIRES newlib) -register_component() \ No newline at end of file +set(src_dirs .) +set(include_dirs . + ./include) +set(requires esp32-camera) +idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs} REQUIRES ${requires}) \ No newline at end of file diff --git a/components/fb_gfx/fb_gfx.c b/components/fb_gfx/fb_gfx.c index 1c0a691..f539562 100644 --- a/components/fb_gfx/fb_gfx.c +++ b/components/fb_gfx/fb_gfx.c @@ -38,10 +38,24 @@ typedef struct #include "FreeMonoBold12pt7b.h" //14x24 #define gfxFont ((GFXfont *)(&FreeMonoBold12pt7b)) -void fb_gfx_fillRect(fb_data_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) +void fb_gfx_fillRect(camera_fb_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) { + int bytes_per_pixel = 0; + switch (fb->format) + { + case PIXFORMAT_GRAYSCALE: + bytes_per_pixel = 1; + break; + case PIXFORMAT_RGB565: + bytes_per_pixel = 2; + break; + case PIXFORMAT_RGB888: + bytes_per_pixel = 3; + default: + break; + } int32_t line_step = (fb->width - w) * 3; - uint8_t *data = fb->data + ((x + (y * fb->width)) * 3); + uint8_t *data = fb->buf + ((x + (y * fb->width)) * bytes_per_pixel); uint8_t c0 = color >> 16; uint8_t c1 = color >> 8; uint8_t c2 = color; @@ -49,26 +63,41 @@ void fb_gfx_fillRect(fb_data_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, { for (int j = 0; j < w; j++) { - data[0] = c0; - data[1] = c1; - data[2] = c2; - data += 3; + switch (bytes_per_pixel) + { + case 1: + data[0] = c2; + data++; + break; + case 2: + data[0] = c1; + data[1] = c2; + data += 2; + break; + case 3: + data[0] = c0; + data[1] = c1; + data[2] = c2; + data += 3; + default: + break; + } } data += line_step; } } -void fb_gfx_drawFastHLine(fb_data_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color) +void fb_gfx_drawFastHLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color) { fb_gfx_fillRect(fb, x, y, w, 1, color); } -void fb_gfx_drawFastVLine(fb_data_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color) +void fb_gfx_drawFastVLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color) { fb_gfx_fillRect(fb, x, y, 1, h, color); } -uint8_t fb_gfx_putc(fb_data_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c) +uint8_t fb_gfx_putc(camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c) { uint16_t line_width; uint8_t xa = 0, bit = 0, bits = 0, xx, yy; @@ -120,7 +149,7 @@ uint8_t fb_gfx_putc(fb_data_t *fb, int32_t x, int32_t y, uint32_t color, unsigne return xa; } -uint32_t fb_gfx_print(fb_data_t *fb, int x, int y, uint32_t color, const char *str) +uint32_t fb_gfx_print(camera_fb_t *fb, int x, int y, uint32_t color, const char *str) { uint32_t l = 0; int xc = x, yc = y, lc = fb->width - gfxFont->glyph[0].xAdvance; @@ -151,7 +180,7 @@ uint32_t fb_gfx_print(fb_data_t *fb, int x, int y, uint32_t color, const char *s return l; } -uint32_t fb_gfx_printf(fb_data_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...) +uint32_t fb_gfx_printf(camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...) { char loc_buf[64]; char *temp = loc_buf; diff --git a/components/fb_gfx/include/fb_gfx.h b/components/fb_gfx/include/fb_gfx.h index 079ff7b..8cd4686 100644 --- a/components/fb_gfx/include/fb_gfx.h +++ b/components/fb_gfx/include/fb_gfx.h @@ -13,29 +13,30 @@ // limitations under the License. #ifndef _FB_GFX_H_ #define _FB_GFX_H_ +#include "esp_camera.h" #ifdef __cplusplus extern "C" { #endif - typedef enum { - FB_RGB888, FB_BGR888, FB_RGB565, FB_BGR565 - } fb_format_t; + // typedef enum { + // FB_RGB888, FB_BGR888, FB_RGB565, FB_BGR565 + // } fb_format_t; - typedef struct { - int width; - int height; - int bytes_per_pixel; - fb_format_t format; - uint8_t * data; - } fb_data_t; + // typedef struct { + // int width; + // int height; + // int bytes_per_pixel; + // fb_format_t format; + // uint8_t * data; + // } fb_data_t; - void fb_gfx_fillRect (fb_data_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color); - void fb_gfx_drawFastHLine(fb_data_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color); - void fb_gfx_drawFastVLine(fb_data_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color); - uint8_t fb_gfx_putc (fb_data_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c); - uint32_t fb_gfx_print (fb_data_t *fb, int32_t x, int32_t y, uint32_t color, const char * str); - uint32_t fb_gfx_printf (fb_data_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...); + void fb_gfx_fillRect (camera_fb_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color); + void fb_gfx_drawFastHLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color); + void fb_gfx_drawFastVLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color); + uint8_t fb_gfx_putc (camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c); + uint32_t fb_gfx_print (camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, const char * str); + uint32_t fb_gfx_printf (camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...); #ifdef __cplusplus } diff --git a/components/modules/CMakeLists.txt b/components/modules/CMakeLists.txt index eb75a3b..7d9770d 100644 --- a/components/modules/CMakeLists.txt +++ b/components/modules/CMakeLists.txt @@ -8,6 +8,7 @@ set(src_dirs camera lcd led + button web trace) @@ -16,6 +17,7 @@ set(include_dirs camera lcd led + button web trace) @@ -26,6 +28,7 @@ set(requires esp32-camera esp_http_server nvs_flash mdns + esp_adc_cal fb_gfx) idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs} REQUIRES ${requires} EMBED_FILES ${embed_files}) diff --git a/components/modules/Kconfig b/components/modules/Kconfig index 33a17e0..1fb5363 100644 --- a/components/modules/Kconfig +++ b/components/modules/Kconfig @@ -215,4 +215,34 @@ menu "ESP-WHO Configuration" Select Camera Y9 pin. endmenu + + menu "Model Configuration" + menu "Face Recognition" + choice FACE_RECOGNITION_MODEL + bool "Face Recognition Model" + default MFN_V1 + help + Select Face Recognition Model. + + config MFN_V1 + bool "mfn v1" + + endchoice + + choice QUANTIZATION + bool "Quantization" + default S8 + help + Select Face Recognition Model. + + config S8 + bool "8-bit" + config S16 + bool "16-bit" + + endchoice + endmenu + + endmenu + endmenu \ No newline at end of file diff --git a/components/modules/ai/who_human_face_recognition.cpp b/components/modules/ai/who_human_face_recognition.cpp new file mode 100644 index 0000000..c416e73 --- /dev/null +++ b/components/modules/ai/who_human_face_recognition.cpp @@ -0,0 +1,239 @@ +#include "who_human_face_recognition.hpp" + +#include "esp_log.h" +#include "esp_camera.h" + +#include "dl_image.hpp" +#include "fb_gfx.h" + +#include "human_face_detect_msr01.hpp" +#include "human_face_detect_mnp01.hpp" +#include "face_recognition_tool.hpp" + +#if CONFIG_MFN_V1 +#if CONFIG_S8 +#include "face_recognition_112_v1_s8.hpp" +#elif CONFIG_S16 +#include "face_recognition_112_v1_s16.hpp" +#endif +#endif + +#include "who_ai_utils.hpp" + +using namespace std; +using namespace dl; + +static const char *TAG = "human_face_detection"; + +static QueueHandle_t xQueueFrameI = NULL; +static QueueHandle_t xQueueEvent = NULL; +static QueueHandle_t xQueueFrameO = NULL; +static QueueHandle_t xQueueResult = NULL; + +static recognizer_state_t gEvent = DETECT; +static bool gReturnFB = true; +static face_info_t recognize_result; + +SemaphoreHandle_t xMutex; + +typedef enum +{ + SHOW_STATE_IDLE, + SHOW_STATE_DELETE, + SHOW_STATE_RECOGNIZE, + SHOW_STATE_ENROLL, +} show_state_t; + +#define RGB565_MASK_RED 0xF800 +#define RGB565_MASK_GREEN 0x07E0 +#define RGB565_MASK_BLUE 0x001F +#define FRAME_DELAY_NUM 16 + +static void rgb_print(camera_fb_t *fb, uint32_t color, const char *str) +{ + fb_gfx_print(fb, (fb->width - (strlen(str) * 14)) / 2, 10, color, str); +} + +static int rgb_printf(camera_fb_t *fb, uint32_t color, const char *format, ...) +{ + char loc_buf[64]; + char *temp = loc_buf; + int len; + va_list arg; + va_list copy; + va_start(arg, format); + va_copy(copy, arg); + len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg); + va_end(copy); + if (len >= sizeof(loc_buf)) + { + temp = (char *)malloc(len + 1); + if (temp == NULL) + { + return 0; + } + } + vsnprintf(temp, len + 1, format, arg); + va_end(arg); + rgb_print(fb, color, temp); + if (len > 64) + { + free(temp); + } + return len; +} + +static void task_process_handler(void *arg) +{ + camera_fb_t *frame = NULL; + HumanFaceDetectMSR01 detector(0.3F, 0.3F, 10, 0.3F); + HumanFaceDetectMNP01 detector2(0.4F, 0.3F, 10); + +#if CONFIG_MFN_V1 +#if CONFIG_S8 + FaceRecognition112V1S8 *recognizer = new FaceRecognition112V1S8(); +#elif CONFIG_S16 + FaceRecognition112V1S16 *recognizer = new FaceRecognition112V1S16(); +#endif +#endif + show_state_t frame_show_state = SHOW_STATE_IDLE; + recognizer_state_t _gEvent; + + while (true) + { + xSemaphoreTake(xMutex, portMAX_DELAY); + _gEvent = gEvent; + gEvent = DETECT; + xSemaphoreGive(xMutex); + + if (_gEvent) + { + bool is_detected = false; + + if (xQueueReceive(xQueueFrameI, &frame, portMAX_DELAY)) + { + std::list &detect_candidates = detector.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}); + std::list &detect_results = detector2.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_candidates); + + if (detect_results.size() == 1) + is_detected = true; + + if (is_detected) + { + switch (_gEvent) + { + case ENROLL: + recognizer->enroll_id((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint); + ESP_LOGW("ENROLL", "ID %d is enrolled", recognizer->get_enrolled_ids().back().id); + frame_show_state = SHOW_STATE_ENROLL; + break; + + case RECOGNIZE: + recognize_result = recognizer->recognize((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint); + print_detection_result(detect_results); + if (recognize_result.id > 0) + ESP_LOGI("RECOGNIZE", "Similarity: %f, Match ID: %d", recognize_result.similarity, recognize_result.id); + else + ESP_LOGE("RECOGNIZE", "Similarity: %f, Match ID: %d", recognize_result.similarity, recognize_result.id); + frame_show_state = SHOW_STATE_RECOGNIZE; + break; + + case DELETE: + recognizer->delete_id(); + ESP_LOGE("DELETE", "% d IDs left", recognizer->get_enrolled_id_num()); + frame_show_state = SHOW_STATE_DELETE; + break; + + default: + break; + } + } + + if (frame_show_state != SHOW_STATE_IDLE) + { + static int frame_count = 0; + switch (frame_show_state) + { + case SHOW_STATE_DELETE: + rgb_printf(frame, RGB565_MASK_RED, "%d IDs left", recognizer->get_enrolled_id_num()); + break; + + case SHOW_STATE_RECOGNIZE: + if (recognize_result.id > 0) + rgb_printf(frame, RGB565_MASK_GREEN, "ID %d", recognize_result.id); + else + rgb_print(frame, RGB565_MASK_RED, "who ?"); + break; + + case SHOW_STATE_ENROLL: + rgb_printf(frame, RGB565_MASK_BLUE, "Enroll: ID %d", recognizer->get_enrolled_ids().back().id); + break; + + default: + break; + } + + if (++frame_count > FRAME_DELAY_NUM) + { + frame_count = 0; + frame_show_state = SHOW_STATE_IDLE; + } + } + + if (detect_results.size()) + { + draw_detection_result((uint16_t *)frame->buf, frame->height, frame->width, detect_results); + } + } + + if (xQueueFrameO) + { + + xQueueSend(xQueueFrameO, &frame, portMAX_DELAY); + } + else if (gReturnFB) + { + esp_camera_fb_return(frame); + } + else + { + free(frame); + } + + if (xQueueResult && is_detected) + { + xQueueSend(xQueueResult, &recognize_result, portMAX_DELAY); + } + } + } +} + +static void task_event_handler(void *arg) +{ + recognizer_state_t _gEvent; + while (true) + { + xQueueReceive(xQueueEvent, &(_gEvent), portMAX_DELAY); + xSemaphoreTake(xMutex, portMAX_DELAY); + gEvent = _gEvent; + xSemaphoreGive(xMutex); + } +} + +void register_human_face_recognition(const QueueHandle_t frame_i, + const QueueHandle_t event, + const QueueHandle_t result, + const QueueHandle_t frame_o, + const bool camera_fb_return) +{ + xQueueFrameI = frame_i; + xQueueFrameO = frame_o; + xQueueEvent = event; + xQueueResult = result; + gReturnFB = camera_fb_return; + xMutex = xSemaphoreCreateMutex(); + + xTaskCreatePinnedToCore(task_process_handler, TAG, 4 * 1024, NULL, 5, NULL, 0); + if (xQueueEvent) + xTaskCreatePinnedToCore(task_event_handler, TAG, 4 * 1024, NULL, 5, NULL, 1); +} diff --git a/components/modules/ai/who_human_face_recognition.hpp b/components/modules/ai/who_human_face_recognition.hpp new file mode 100644 index 0000000..b7841d8 --- /dev/null +++ b/components/modules/ai/who_human_face_recognition.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "freertos/semphr.h" + +typedef enum +{ + IDLE = 0, + DETECT, + ENROLL, + RECOGNIZE, + DELETE, +} recognizer_state_t; + +void register_human_face_recognition(QueueHandle_t frame_i, + QueueHandle_t event, + QueueHandle_t result, + QueueHandle_t frame_o = NULL, + const bool camera_fb_return = false); diff --git a/components/modules/button/who_adc_button.c b/components/modules/button/who_adc_button.c new file mode 100644 index 0000000..59a0a41 --- /dev/null +++ b/components/modules/button/who_adc_button.c @@ -0,0 +1,110 @@ +/* ADC1 Example + + This example code is in the Public Domain (or CC0 licensed, at your option.) + + Unless required by applicable law or agreed to in writing, this + software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + CONDITIONS OF ANY KIND, either express or implied. +*/ +#include +#include +#include "esp_log.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "driver/adc_common.h" +#include "esp_adc_cal.h" +#include "who_adc_button.h" + +//ADC Channels +#define ADC1_EXAMPLE_CHAN0 ADC1_CHANNEL_0 +//ADC Attenuation +#define ADC_EXAMPLE_ATTEN ADC_ATTEN_DB_11 +//ADC Calibration +#if CONFIG_IDF_TARGET_ESP32 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_VREF +#elif CONFIG_IDF_TARGET_ESP32S2 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP +#elif CONFIG_IDF_TARGET_ESP32C3 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP +#elif CONFIG_IDF_TARGET_ESP32S3 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP_FIT +#endif + +#define PRESS_INTERVAL 500000 + +static uint32_t voltage = 0; + +static const char *TAG = "ADC SINGLE"; + +static esp_adc_cal_characteristics_t adc1_chars; + +button_adc_config_t *adc_buttons; +int adc_button_num; +static QueueHandle_t xQueueKeyStateO = NULL; + +static bool adc_calibration_init(void) +{ + esp_err_t ret; + bool cali_enable = false; + + ret = esp_adc_cal_check_efuse(ADC_EXAMPLE_CALI_SCHEME); + if (ret == ESP_ERR_NOT_SUPPORTED) + { + ESP_LOGW(TAG, "Calibration scheme not supported, skip software calibration"); + } + else if (ret == ESP_ERR_INVALID_VERSION) + { + ESP_LOGW(TAG, "eFuse not burnt, skip software calibration"); + } + else if (ret == ESP_OK) + { + cali_enable = true; + esp_adc_cal_characterize(ADC_UNIT_1, ADC_EXAMPLE_ATTEN, ADC_WIDTH_BIT_DEFAULT, 0, &adc1_chars); + } + else + { + ESP_LOGE(TAG, "Invalid arg"); + } + return cali_enable; +} + +void adc_button_task(void *arg) +{ + int last_button_pressed = -1; + int button_pressed = -1; + int64_t backup_time = esp_timer_get_time(); + int64_t last_time = esp_timer_get_time(); + + //ADC1 config + ESP_ERROR_CHECK(adc1_config_width(ADC_WIDTH_BIT_DEFAULT)); + ESP_ERROR_CHECK(adc1_config_channel_atten(ADC1_EXAMPLE_CHAN0, ADC_EXAMPLE_ATTEN)); + + while (1) + { + voltage = adc1_get_raw(ADC1_EXAMPLE_CHAN0); + backup_time = esp_timer_get_time(); + for (int i = 0; i < adc_button_num; ++i) + { + if ((voltage >= adc_buttons[i].min) && (voltage <= adc_buttons[i].max)) + { + button_pressed = adc_buttons[i].button_index; + if ((button_pressed != last_button_pressed) || ((backup_time - last_time) > PRESS_INTERVAL)) + { + last_button_pressed = button_pressed; + last_time = backup_time; + xQueueOverwrite(xQueueKeyStateO, &button_pressed); + break; + } + } + } + vTaskDelay(pdMS_TO_TICKS(10)); + } +} + +void register_adc_button(button_adc_config_t *buttons_ptr, int button_num, const QueueHandle_t key_state_o) +{ + xQueueKeyStateO = key_state_o; + adc_buttons = buttons_ptr; + adc_button_num = button_num; + xTaskCreatePinnedToCore(adc_button_task, "adc_button_scan_task", 1024, NULL, 5, NULL, 0); +} \ No newline at end of file diff --git a/components/modules/button/who_adc_button.h b/components/modules/button/who_adc_button.h new file mode 100644 index 0000000..25fb0af --- /dev/null +++ b/components/modules/button/who_adc_button.h @@ -0,0 +1,30 @@ +#pragma once +#include "esp_event_loop.h" +#include "soc/system_reg.h" +#include "driver/gpio.h" +#include "esp_log.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef struct + { + int button_index; /**< button index on the channel */ + int min; /**< min voltage in mv corresponding to the button */ + int max; /**< max voltage in mv corresponding to the button */ + } button_adc_config_t; + + /** + * @brief initialize adc button + * + * @param buttons_ptr the pointer of adc button configuration + * @param button_num the numbers of adc buttons + * @param key_state_o the queue to send which button is pressed + */ + void register_adc_button(button_adc_config_t *buttons_ptr, int button_num, const QueueHandle_t key_state_o); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/components/modules/button/who_button.c b/components/modules/button/who_button.c new file mode 100644 index 0000000..1bf7f9f --- /dev/null +++ b/components/modules/button/who_button.c @@ -0,0 +1,100 @@ +#include +#include +#include "who_button.h" + +typedef struct +{ + gpio_num_t io_num; + key_state_t state; +} key_scan_state_t; + +#define LONG_PRESS_THRESH 700000 +#define DOUBLE_CLICK_THRESH 300000 + +static xQueueHandle gpio_evt_queue = NULL; +static QueueHandle_t xQueueKeyStateO = NULL; + +static void IRAM_ATTR gpio_isr_handler_key(void *arg) +{ + uint32_t gpio_num = (uint32_t)arg; + xQueueSendFromISR(gpio_evt_queue, &gpio_num, NULL); +} + +int key_scan(TickType_t ticks_to_wait) +{ + gpio_num_t io_num; + BaseType_t press_key = pdFALSE; + BaseType_t lift_key = pdFALSE; + int64_t backup_time = 0; + int64_t interval_time = 0; + static int64_t last_time = 0; + + for (;;) + { + xQueueReceive(gpio_evt_queue, &io_num, ticks_to_wait); + + if (gpio_get_level(io_num) == 0) + { + press_key = pdTRUE; + backup_time = esp_timer_get_time(); + interval_time = backup_time - last_time; + } + else if (press_key) + { + lift_key = pdTRUE; + last_time = esp_timer_get_time(); + backup_time = last_time - backup_time; + } + + if (press_key & lift_key) + { + press_key = pdFALSE; + lift_key = pdFALSE; + + if (backup_time > LONG_PRESS_THRESH) + { + return KEY_LONG_PRESS; + } + else + { + if ((interval_time < DOUBLE_CLICK_THRESH) && (interval_time > 0)) + return KEY_DOUBLE_CLICK; + else + return KEY_SHORT_PRESS; + } + } + } +} + +void key_trigger(void *arg) +{ + int ret = 0; + + while (1) + { + ret = key_scan(portMAX_DELAY); + xQueueOverwrite(xQueueKeyStateO, &ret); + } + + vTaskDelete(NULL); +} + +void key_init(gpio_num_t gpio_num) +{ + gpio_config_t io_conf = {0}; + io_conf.mode = GPIO_MODE_INPUT; + io_conf.intr_type = GPIO_INTR_ANYEDGE; + io_conf.pin_bit_mask = 1LL << gpio_num; + io_conf.pull_up_en = GPIO_PULLUP_ENABLE; + gpio_config(&io_conf); + gpio_evt_queue = xQueueCreate(5, sizeof(uint32_t)); + gpio_install_isr_service(0); + gpio_isr_handler_add(gpio_num, gpio_isr_handler_key, (void *)gpio_num); +} + +void register_button(const gpio_num_t key_io_num, const QueueHandle_t key_state_o) +{ + xQueueKeyStateO = key_state_o; + key_init(key_io_num); + xTaskCreatePinnedToCore(key_trigger, "key_scan_task", 1024, NULL, 5, NULL, 0); +} \ No newline at end of file diff --git a/components/modules/button/who_button.h b/components/modules/button/who_button.h new file mode 100644 index 0000000..e5989c0 --- /dev/null +++ b/components/modules/button/who_button.h @@ -0,0 +1,29 @@ +#pragma once +#include "esp_event_loop.h" +#include "soc/system_reg.h" +#include "driver/gpio.h" +#include "esp_log.h" + +typedef enum +{ + KEY_SHORT_PRESS = 1, + KEY_LONG_PRESS, + KEY_DOUBLE_CLICK, +} key_state_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * @brief initialize gpio button + * + * @param key_io_num the gpio number of the button + * @param key_state_o the queue to send the button state + */ +void register_button(const gpio_num_t key_io_num, const QueueHandle_t key_state_o); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/components/modules/camera/who_camera.h b/components/modules/camera/who_camera.h index 188bd9c..474154c 100644 --- a/components/modules/camera/who_camera.h +++ b/components/modules/camera/who_camera.h @@ -192,7 +192,7 @@ #define CAMERA_PIN_PCLK CONFIG_CAMERA_PIN_PCLK #endif -#define XCLK_FREQ_HZ 20000000 +#define XCLK_FREQ_HZ 16000000 #ifdef __cplusplus extern "C" diff --git a/examples/human_face_recognition/README.md b/examples/human_face_recognition/README.md new file mode 100644 index 0000000..4c9007e --- /dev/null +++ b/examples/human_face_recognition/README.md @@ -0,0 +1,18 @@ +# Human Face Recognition Example [[中文]](./README_ZH.md) + +The following is the special configuration of face recognition examples. For general configuration, please refer to [this](../../README.md) + + +## Configure the face recognition model + +Enter `idf.py menuconfig` in the terminal and click (Top) -> Component config -> ESP-WHO Configuration -> Model Configuration -> Face Recognition to enter the face recognition model configuration interface, as shown below: +![](../../img/face_recognition_model_config.png) +You can configure the version and quantification type of the model here. + +## How to Use Example + +- The interactive button is the Boot button. +- Short press the button: recognize the face captured by the camera at this time. +- Long press the button: enroll the face captured by the camera at this time. +- Double click the button: delete the last enrolled face. + diff --git a/examples/human_face_recognition/README_CN.md b/examples/human_face_recognition/README_CN.md new file mode 100644 index 0000000..6f47a1c --- /dev/null +++ b/examples/human_face_recognition/README_CN.md @@ -0,0 +1,18 @@ +# Human Face Recognition Example [[English]](./README.md) + +以下为使用人脸识别示例的特殊配置,若要进行通用配置可参考此[说明](../../README_ZH.md)。 + +## 人脸识别模型配置 + +在终端输入 `idf.py menuconfig` ,依次 (Top) -> Component config -> ESP-WHO Configuration -> Model Configuration -> Face Recognition 可进入人脸识别模型配置界面,如下图所示: + +![](../../img/face_recognition_model_config.png) + +您可以在这里配置模型的版本和量化方式。 + +## 如何使用示例 + +- 交互按键为Boot键。 +- 短按按键:识别此时摄像头拍到的人脸。 +- 长按按键:录入此时摄像头拍到的人脸。 +- 双击按键:删除最后一个被录入的人脸。 diff --git a/examples/human_face_recognition/lcd/CMakeLists.txt b/examples/human_face_recognition/lcd/CMakeLists.txt new file mode 100644 index 0000000..f3c0cfb --- /dev/null +++ b/examples/human_face_recognition/lcd/CMakeLists.txt @@ -0,0 +1,8 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +set(EXTRA_COMPONENT_DIRS ../../../components) +add_compile_options(-fdiagnostics-color=always) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(human_face_recognition_lcd) diff --git a/examples/human_face_recognition/lcd/main/CMakeLists.txt b/examples/human_face_recognition/lcd/main/CMakeLists.txt new file mode 100644 index 0000000..0f71747 --- /dev/null +++ b/examples/human_face_recognition/lcd/main/CMakeLists.txt @@ -0,0 +1,5 @@ +set(src_dirs .) + +set(include_dirs .) + +idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs}) \ No newline at end of file diff --git a/examples/human_face_recognition/lcd/main/app_main.cpp b/examples/human_face_recognition/lcd/main/app_main.cpp new file mode 100755 index 0000000..6bb5a3b --- /dev/null +++ b/examples/human_face_recognition/lcd/main/app_main.cpp @@ -0,0 +1,30 @@ +#include "who_camera.h" +#include "who_human_face_recognition.hpp" +#include "who_lcd.h" +#include "who_button.h" +#include "event_logic.hpp" +#include "who_adc_button.h" + +static QueueHandle_t xQueueAIFrame = NULL; +static QueueHandle_t xQueueLCDFrame = NULL; +static QueueHandle_t xQueueKeyState = NULL; +static QueueHandle_t xQueueEventLogic = NULL; +static button_adc_config_t buttons[4] = {{1, 2800, 3000}, {2, 2250, 2450}, {3, 300, 500}, {4, 850, 1050}}; + +#define GPIO_BOOT GPIO_NUM_0 + +extern "C" void app_main() +{ + xQueueAIFrame = xQueueCreate(2, sizeof(camera_fb_t *)); + xQueueLCDFrame = xQueueCreate(2, sizeof(camera_fb_t *)); + xQueueKeyState = xQueueCreate(1, sizeof(int *)); + xQueueEventLogic = xQueueCreate(1, sizeof(int *)); + + register_camera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueAIFrame); + register_button(GPIO_BOOT, xQueueKeyState); + // register_adc_button(buttons, 4, xQueueKeyState); + register_event(xQueueKeyState, xQueueEventLogic); + register_human_face_recognition(xQueueAIFrame, xQueueEventLogic, NULL, xQueueLCDFrame, false); + register_lcd(xQueueLCDFrame, NULL, true); + +} diff --git a/examples/human_face_recognition/lcd/main/event_logic.cpp b/examples/human_face_recognition/lcd/main/event_logic.cpp new file mode 100644 index 0000000..08adeaa --- /dev/null +++ b/examples/human_face_recognition/lcd/main/event_logic.cpp @@ -0,0 +1,84 @@ +#include +#include "event_logic.hpp" +#include "who_button.h" +#include "who_human_face_recognition.hpp" + +typedef enum +{ + MENU = 1, + PLAY, + UP, + DOWN +}key_name_t; + +static QueueHandle_t xQueueKeyStateI = NULL; +static QueueHandle_t xQueueEventO = NULL; +static key_state_t key_state; +static key_name_t adc_button_name; +static recognizer_state_t recognizer_state; + +void event_generate(void *arg) +{ + while (1) + { + xQueueReceive(xQueueKeyStateI, &key_state, portMAX_DELAY); + switch (key_state) + { + case KEY_SHORT_PRESS: + recognizer_state = RECOGNIZE; + break; + + case KEY_LONG_PRESS: + recognizer_state = ENROLL; + break; + + case KEY_DOUBLE_CLICK: + recognizer_state = DELETE; + break; + + default: + recognizer_state = DETECT; + break; + } + xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY); + } +} + + +void event_generate_from_adc_button(void *arg) +{ + while (1) + { + xQueueReceive(xQueueKeyStateI, &adc_button_name, portMAX_DELAY); + switch (adc_button_name) + { + case MENU: + recognizer_state = ENROLL; + break; + + case PLAY: + recognizer_state = DELETE; + break; + + case UP: + recognizer_state = RECOGNIZE; + break; + + case DOWN: + recognizer_state = RECOGNIZE; + break; + + default: + recognizer_state = DETECT; + break; + } + xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY); + } +} + +void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o) +{ + xQueueKeyStateI = key_state_i; + xQueueEventO = event_o; + xTaskCreatePinnedToCore(event_generate, "event_logic_task", 1024, NULL, 5, NULL, 0); +} \ No newline at end of file diff --git a/examples/human_face_recognition/lcd/main/event_logic.hpp b/examples/human_face_recognition/lcd/main/event_logic.hpp new file mode 100644 index 0000000..9468ba0 --- /dev/null +++ b/examples/human_face_recognition/lcd/main/event_logic.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" + +/** + * @brief + * + * @param key_state_i + * @param event_o + */ +void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o); \ No newline at end of file diff --git a/examples/human_face_recognition/lcd/partitions.csv b/examples/human_face_recognition/lcd/partitions.csv new file mode 100644 index 0000000..d6fdf11 --- /dev/null +++ b/examples/human_face_recognition/lcd/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +factory, app, factory, 0x010000, 3840K +nvs, data, nvs, 0x3D0000, 16K +fr, 32, 32, 0x3E0000, 128K diff --git a/examples/human_face_recognition/lcd/sdkconfig.defaults b/examples/human_face_recognition/lcd/sdkconfig.defaults new file mode 100644 index 0000000..2b2b87f --- /dev/null +++ b/examples/human_face_recognition/lcd/sdkconfig.defaults @@ -0,0 +1,12 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y + +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y + +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +CONFIG_S16=y \ No newline at end of file diff --git a/examples/human_face_recognition/lcd/sdkconfig.defaults.esp32s3 b/examples/human_face_recognition/lcd/sdkconfig.defaults.esp32s3 new file mode 100644 index 0000000..522de61 --- /dev/null +++ b/examples/human_face_recognition/lcd/sdkconfig.defaults.esp32s3 @@ -0,0 +1,15 @@ +CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S3_SPIRAM_SUPPORT=y + +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_8WAYS=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y + +CONFIG_CAMERA_MODULE_ESP_S3_EYE=y +CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y +CONFIG_ESPTOOLPY_NO_STUB=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y + +CONFIG_SPIRAM_MODE_OCT=y + +CONFIG_S8=y diff --git a/examples/human_face_recognition/terminal/CMakeLists.txt b/examples/human_face_recognition/terminal/CMakeLists.txt new file mode 100644 index 0000000..fa8a4b8 --- /dev/null +++ b/examples/human_face_recognition/terminal/CMakeLists.txt @@ -0,0 +1,8 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +set(EXTRA_COMPONENT_DIRS ../../../components) +add_compile_options(-fdiagnostics-color=always) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(human_face_recognition_terminal) diff --git a/examples/human_face_recognition/terminal/main/CMakeLists.txt b/examples/human_face_recognition/terminal/main/CMakeLists.txt new file mode 100644 index 0000000..0f71747 --- /dev/null +++ b/examples/human_face_recognition/terminal/main/CMakeLists.txt @@ -0,0 +1,5 @@ +set(src_dirs .) + +set(include_dirs .) + +idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs}) \ No newline at end of file diff --git a/examples/human_face_recognition/terminal/main/app_main.cpp b/examples/human_face_recognition/terminal/main/app_main.cpp new file mode 100755 index 0000000..a96e98a --- /dev/null +++ b/examples/human_face_recognition/terminal/main/app_main.cpp @@ -0,0 +1,25 @@ +#include "who_camera.h" +#include "who_human_face_recognition.hpp" +#include "who_button.h" +#include "event_logic.hpp" +#include "who_adc_button.h" + +static QueueHandle_t xQueueAIFrame = NULL; +static QueueHandle_t xQueueKeyState = NULL; +static QueueHandle_t xQueueEventLogic = NULL; +static button_adc_config_t buttons[4] = {{1, 2800, 3000}, {2, 2250, 2450}, {3, 300, 500}, {4, 850, 1050}}; + +#define GPIO_BOOT GPIO_NUM_0 + +extern "C" void app_main() +{ + xQueueAIFrame = xQueueCreate(2, sizeof(camera_fb_t *)); + xQueueKeyState = xQueueCreate(1, sizeof(int *)); + xQueueEventLogic = xQueueCreate(1, sizeof(int *)); + + register_camera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueAIFrame); + register_button(GPIO_BOOT, xQueueKeyState); + register_event(xQueueKeyState, xQueueEventLogic); + register_human_face_recognition(xQueueAIFrame, xQueueEventLogic, NULL, NULL, true); + +} diff --git a/examples/human_face_recognition/terminal/main/event_logic.cpp b/examples/human_face_recognition/terminal/main/event_logic.cpp new file mode 100644 index 0000000..08adeaa --- /dev/null +++ b/examples/human_face_recognition/terminal/main/event_logic.cpp @@ -0,0 +1,84 @@ +#include +#include "event_logic.hpp" +#include "who_button.h" +#include "who_human_face_recognition.hpp" + +typedef enum +{ + MENU = 1, + PLAY, + UP, + DOWN +}key_name_t; + +static QueueHandle_t xQueueKeyStateI = NULL; +static QueueHandle_t xQueueEventO = NULL; +static key_state_t key_state; +static key_name_t adc_button_name; +static recognizer_state_t recognizer_state; + +void event_generate(void *arg) +{ + while (1) + { + xQueueReceive(xQueueKeyStateI, &key_state, portMAX_DELAY); + switch (key_state) + { + case KEY_SHORT_PRESS: + recognizer_state = RECOGNIZE; + break; + + case KEY_LONG_PRESS: + recognizer_state = ENROLL; + break; + + case KEY_DOUBLE_CLICK: + recognizer_state = DELETE; + break; + + default: + recognizer_state = DETECT; + break; + } + xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY); + } +} + + +void event_generate_from_adc_button(void *arg) +{ + while (1) + { + xQueueReceive(xQueueKeyStateI, &adc_button_name, portMAX_DELAY); + switch (adc_button_name) + { + case MENU: + recognizer_state = ENROLL; + break; + + case PLAY: + recognizer_state = DELETE; + break; + + case UP: + recognizer_state = RECOGNIZE; + break; + + case DOWN: + recognizer_state = RECOGNIZE; + break; + + default: + recognizer_state = DETECT; + break; + } + xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY); + } +} + +void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o) +{ + xQueueKeyStateI = key_state_i; + xQueueEventO = event_o; + xTaskCreatePinnedToCore(event_generate, "event_logic_task", 1024, NULL, 5, NULL, 0); +} \ No newline at end of file diff --git a/examples/human_face_recognition/terminal/main/event_logic.hpp b/examples/human_face_recognition/terminal/main/event_logic.hpp new file mode 100644 index 0000000..9468ba0 --- /dev/null +++ b/examples/human_face_recognition/terminal/main/event_logic.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" + +/** + * @brief + * + * @param key_state_i + * @param event_o + */ +void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o); \ No newline at end of file diff --git a/examples/human_face_recognition/terminal/partitions.csv b/examples/human_face_recognition/terminal/partitions.csv new file mode 100644 index 0000000..d6fdf11 --- /dev/null +++ b/examples/human_face_recognition/terminal/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +factory, app, factory, 0x010000, 3840K +nvs, data, nvs, 0x3D0000, 16K +fr, 32, 32, 0x3E0000, 128K diff --git a/examples/human_face_recognition/terminal/sdkconfig.defaults b/examples/human_face_recognition/terminal/sdkconfig.defaults new file mode 100644 index 0000000..2b2b87f --- /dev/null +++ b/examples/human_face_recognition/terminal/sdkconfig.defaults @@ -0,0 +1,12 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y + +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y + +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +CONFIG_S16=y \ No newline at end of file diff --git a/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32 b/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32 new file mode 100644 index 0000000..ecc2d87 --- /dev/null +++ b/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32 @@ -0,0 +1,4 @@ +CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32_SPIRAM_SUPPORT=y + +CONFIG_CAMERA_MODULE_ESP_EYE=y \ No newline at end of file diff --git a/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s2 b/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s2 new file mode 100644 index 0000000..3b5e99c --- /dev/null +++ b/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s2 @@ -0,0 +1,7 @@ +CONFIG_ESP32S2_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S2_SPIRAM_SUPPORT=y + +CONFIG_ESP32S2_DATA_CACHE_16KB=y +ESP32S2_DATA_CACHE_LINE_32B=y + +CONFIG_CAMERA_MODULE_ESP_S2_KALUGA=y \ No newline at end of file diff --git a/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s3 b/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s3 new file mode 100644 index 0000000..522de61 --- /dev/null +++ b/examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s3 @@ -0,0 +1,15 @@ +CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S3_SPIRAM_SUPPORT=y + +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_8WAYS=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y + +CONFIG_CAMERA_MODULE_ESP_S3_EYE=y +CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y +CONFIG_ESPTOOLPY_NO_STUB=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y + +CONFIG_SPIRAM_MODE_OCT=y + +CONFIG_S8=y diff --git a/img/face_recognition_model_config.png b/img/face_recognition_model_config.png new file mode 100644 index 0000000..cb93b15 Binary files /dev/null and b/img/face_recognition_model_config.png differ