From 0684cc5db3413f4c6590c523a348571ad6f99e9a Mon Sep 17 00:00:00 2001 From: yehangyang Date: Wed, 2 Mar 2022 19:38:29 +0800 Subject: [PATCH] :sparkles: esp32-s3-eye --- examples/esp32-s3-eye/CMakeLists.txt | 8 + examples/esp32-s3-eye/README.md | 65 +++++ examples/esp32-s3-eye/main/CMakeLists.txt | 11 + examples/esp32-s3-eye/main/Kconfig.projbuild | 13 + examples/esp32-s3-eye/main/app_main.cpp | 50 ++++ .../esp32-s3-eye/main/include/__base__.hpp | 79 ++++++ .../esp32-s3-eye/main/include/app_buttom.hpp | 35 +++ .../esp32-s3-eye/main/include/app_camera.hpp | 210 ++++++++++++++++ .../esp32-s3-eye/main/include/app_face.hpp | 58 +++++ .../esp32-s3-eye/main/include/app_lcd.hpp | 51 ++++ .../esp32-s3-eye/main/include/app_led.hpp | 23 ++ .../esp32-s3-eye/main/include/app_motion.hpp | 26 ++ .../esp32-s3-eye/main/include/app_speech.hpp | 23 ++ examples/esp32-s3-eye/main/src/app_buttom.cpp | 79 ++++++ examples/esp32-s3-eye/main/src/app_camera.cpp | 98 ++++++++ examples/esp32-s3-eye/main/src/app_face.cpp | 231 ++++++++++++++++++ examples/esp32-s3-eye/main/src/app_lcd.cpp | 178 ++++++++++++++ examples/esp32-s3-eye/main/src/app_led.cpp | 124 ++++++++++ examples/esp32-s3-eye/main/src/app_motion.cpp | 82 +++++++ examples/esp32-s3-eye/main/src/app_speech.cpp | 214 ++++++++++++++++ examples/esp32-s3-eye/partitions.csv | 7 + examples/esp32-s3-eye/sdkconfig.defaults | 59 +++++ 22 files changed, 1724 insertions(+) create mode 100644 examples/esp32-s3-eye/CMakeLists.txt create mode 100644 examples/esp32-s3-eye/README.md create mode 100644 examples/esp32-s3-eye/main/CMakeLists.txt create mode 100644 examples/esp32-s3-eye/main/Kconfig.projbuild create mode 100755 examples/esp32-s3-eye/main/app_main.cpp create mode 100644 examples/esp32-s3-eye/main/include/__base__.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_buttom.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_camera.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_face.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_lcd.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_led.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_motion.hpp create mode 100644 examples/esp32-s3-eye/main/include/app_speech.hpp create mode 100644 examples/esp32-s3-eye/main/src/app_buttom.cpp create mode 100644 examples/esp32-s3-eye/main/src/app_camera.cpp create mode 100644 examples/esp32-s3-eye/main/src/app_face.cpp create mode 100644 examples/esp32-s3-eye/main/src/app_lcd.cpp create mode 100644 examples/esp32-s3-eye/main/src/app_led.cpp create mode 100644 examples/esp32-s3-eye/main/src/app_motion.cpp create mode 100644 examples/esp32-s3-eye/main/src/app_speech.cpp create mode 100644 examples/esp32-s3-eye/partitions.csv create mode 100644 examples/esp32-s3-eye/sdkconfig.defaults diff --git a/examples/esp32-s3-eye/CMakeLists.txt b/examples/esp32-s3-eye/CMakeLists.txt new file mode 100644 index 0000000..dedeb46 --- /dev/null +++ b/examples/esp32-s3-eye/CMakeLists.txt @@ -0,0 +1,8 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +set(EXTRA_COMPONENT_DIRS ../../components) +add_compile_options(-fdiagnostics-color=always) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(esp32-s3-eye) diff --git a/examples/esp32-s3-eye/README.md b/examples/esp32-s3-eye/README.md new file mode 100644 index 0000000..966d69a --- /dev/null +++ b/examples/esp32-s3-eye/README.md @@ -0,0 +1,65 @@ +# ESP-S3-EYE 示例说明 + +## 交互系统 + +### 语音 + +语音交互逻辑遵循“先说唤醒词，再说命令词”。 + + + +### 按键 + +#### + +## 工作模式 + +该示例共包含四个工作模式：“待机模式”、“实时显示”、“人脸识别”和“移动侦测”。下面对这四个模式做简要介绍。 + +### 待机模式 + +待机模式下，LCD 屏幕上仅显示 Espressif Logo。该模式也是开发板上电以后的默认模式。 + +### 实时显示 + +实时显示模式下，LCD 屏幕上会实时显示摄像头采集的图片。 + +### 人脸识别 + +人脸识别模式下，LCD 屏幕上会实时显示摄像头采集的图片。并且实时检测画面中的人脸，并显示检测框和关键点。通过按键和语音交互可以实现人脸识别相关的操作，详见下表： + +| 操作 | 说明 | 按键触发 | 语音触发 | +| :------: | :------------------------------------------: | :------: | :------------------------------------: | +| 识别人脸 | 识别当前画面中的人脸，并显示识别结果 | “PLAY” | “Hi，乐鑫”唤醒后，说出命令词“识别一下” | +| 添加人脸 | 添加当前画面中的人脸，并显示添加的 ID 号 | “UP” | “Hi，乐鑫”唤醒后，说出命令词“添加人脸” | +| 删除人脸 | 删除人脸库中的最后一个 ID， 并显示剩余 ID 数 | “DOWN” | “Hi，乐鑫”唤醒后，说出命令词“删除人脸” | + +### 移动侦测 + +移动侦测模式下，LCD 屏幕上会实时显示摄像头采集的图片。并且实时检测画面中是否出现物体移动，如果物体移动，画面左上角会显示蓝点。 + + + +#### 进入默认固件 +上电后等待3秒，不对开发板做任何操作，当LCD屏幕显示Espressif LOGO表示已进入默认固件。 + +#### 默认固件模式 +默认固件存在四种运行模式，固件会默认先进入人脸识别模式： +1. 待机模式：该模式下开发板只有LCD工作，显示公司LOGO。 +2. 实时显示模式：该模式下LCD会实时显示摄像头拍摄到的画面。 +3. 人脸识别模式：该模式下LCD会实时显示摄像头拍摄到的画面，如果画面中存在人脸，LCD显示的画面上会框出人脸，并画出人脸的五个特征点。该模式下提供三种功能：录入人脸、识别人脸、删除人脸。 + - 录入人脸： 按下MENU键。或者使用“Hi，乐鑫”唤醒词唤醒，唤醒后摄像头旁的LED会被点亮，随后说出“添加人脸”。 + - 识别人脸： 按下UP键。或者使用“Hi，乐鑫”唤醒词唤醒，唤醒后摄像头旁的LED会被点亮，随后说出“识别一下”。 + - 删除人脸： 按下PLAY键。或者使用“Hi，乐鑫”唤醒词唤醒，唤醒后摄像头旁的LED会被点亮，随后说出“删除人脸”。 +4. 移动侦测模式： 该模式下LCD会实时显示摄像头拍摄到的画面。若画面中存在物体移动，LCD中左上角会显示蓝色色块。 + +#### 模式切换 +在任何时候都可以使用“Hi，乐鑫”唤醒词唤醒开发板，四种模式分别对应四个命令词。说出相应的命令词，识别成功后随即切换到对应的模式。工作模式与命令词的对应关系如下： +1. 待机模式：“停止工作” +2. 实时显示模式：“仅显示” +3. 人脸识别模式：“人脸识别” +4. 移动侦测模式：“移动侦测” + +#### 语音系统 +- 语音唤醒：默认固件工作时，任何时刻都可以使用“Hi，乐鑫”唤醒词唤醒开发板。开发板被唤醒后，摄像头旁的LED会常亮。 +- 命令词识别：唤醒后会等待用户说出命令词。若识别成功，摄像头旁的LED会从常亮变为闪烁一秒, LED会随后熄灭。 \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/CMakeLists.txt b/examples/esp32-s3-eye/main/CMakeLists.txt new file mode 100644 index 0000000..700bdc9 --- /dev/null +++ b/examples/esp32-s3-eye/main/CMakeLists.txt @@ -0,0 +1,11 @@ +set(src_dirs . + src) + +set(include_dirs include + .) + +set(requires console + esp_adc_cal + ) + +idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs}) \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/Kconfig.projbuild b/examples/esp32-s3-eye/main/Kconfig.projbuild new file mode 100644 index 0000000..92afa98 --- /dev/null +++ b/examples/esp32-s3-eye/main/Kconfig.projbuild @@ -0,0 +1,13 @@ +menu "ESP32-S3-EYE Firmware" + choice ESP_SR_LANGUAGE + bool "esp-sr language" + default CN_MODEL + help + Select ESP-SR Language. + + config CN_MODEL + bool "Chinese" + config EN_MODEL + bool "English" + endchoice +endmenu \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/app_main.cpp b/examples/esp32-s3-eye/main/app_main.cpp new file mode 100755 index 0000000..c86a46b --- /dev/null +++ b/examples/esp32-s3-eye/main/app_main.cpp @@ -0,0 +1,50 @@ +#include "sdkconfig.h" +#include "driver/gpio.h" + +#if CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID +#include "who_trace.h" +#endif + +#include "app_buttom.hpp" +#include "app_camera.hpp" +#include "app_lcd.hpp" +#include "app_led.hpp" +#include "app_motion.hpp" +#include "app_speech.hpp" +#include "app_face.hpp" + +extern "C" void app_main() +{ +#if CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID + register_trace(); +#endif + + QueueHandle_t xQueueFrame_0 = xQueueCreate(2, sizeof(camera_fb_t *)); + QueueHandle_t xQueueFrame_1 = xQueueCreate(2, sizeof(camera_fb_t *)); + QueueHandle_t xQueueFrame_2 = xQueueCreate(2, sizeof(camera_fb_t *)); + + AppButtom *key = new AppButtom(); + AppSpeech *speech = new AppSpeech(); + AppCamera *camera = new AppCamera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueFrame_0); + AppFace *face = new AppFace(key, speech, xQueueFrame_0, xQueueFrame_1); + AppMotion *motion = new AppMotion(key, speech, xQueueFrame_1, xQueueFrame_2); + AppLCD *lcd = new AppLCD(key, speech, xQueueFrame_2); + LED *led = new LED(GPIO_NUM_3, key, speech); + + key->attach(face); + key->attach(motion); + key->attach(led); + key->attach(lcd); + + speech->attach(face); + speech->attach(motion); + speech->attach(led); + speech->attach(lcd); + + lcd->run(); + motion->run(); + face->run(); + camera->run(); + speech->run(); + key->run(); +} diff --git a/examples/esp32-s3-eye/main/include/__base__.hpp b/examples/esp32-s3-eye/main/include/__base__.hpp new file mode 100644 index 0000000..27b4751 --- /dev/null +++ b/examples/esp32-s3-eye/main/include/__base__.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "freertos/semphr.h" + +#include "esp_camera.h" + +typedef enum +{ + COMMAND_TIMEOUT = -2, + COMMAND_NOT_DETECTED = -1, + + MENU_STOP_WORKING = 0, + MENU_DISPLAY_ONLY = 1, + MENU_FACE_RECOGNITION = 2, + MENU_MOTION_DETECTION = 3, + + ACTION_ENROLL = 4, + ACTION_DELETE = 5, + ACTION_RECOGNIZE = 6 +} command_word_t; + +class Observer +{ +public: + virtual void update() = 0; +}; + +class Subject +{ +private: + std::list observers; + +public: + void attach(Observer *observer) + { + this->observers.push_back(observer); + } + + void detach(Observer *observer) + { + this->observers.remove(observer); + } + + void detach_all() + { + this->observers.clear(); + } + + void notify() + { + for (auto observer : this->observers) + observer->update(); + } +}; + +class Frame +{ +public: + QueueHandle_t queue_i; + QueueHandle_t queue_o; + void (*callback)(camera_fb_t *); + + Frame(QueueHandle_t queue_i = nullptr, + QueueHandle_t queue_o = nullptr, + void (*callback)(camera_fb_t *) = nullptr) : queue_i(queue_i), + queue_o(queue_o), + callback(callback) {} + + void set_io(QueueHandle_t queue_i, QueueHandle_t queue_o) + { + this->queue_i = queue_i; + this->queue_o = queue_o; + } +}; diff --git a/examples/esp32-s3-eye/main/include/app_buttom.hpp b/examples/esp32-s3-eye/main/include/app_buttom.hpp new file mode 100644 index 0000000..f1de93d --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_buttom.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "__base__.hpp" + +typedef enum +{ + _IDLE = 0, + _MENU, + _PLAY, + _UP, + _DOWN +} _key_name_t; + +typedef struct +{ + _key_name_t key; /**< button index on the channel */ + int min; /**< min voltage in mv corresponding to the button */ + int max; /**< max voltage in mv corresponding to the button */ +} key_config_t; + +class AppButtom : public Subject +{ +public: + std::vector key_configs; + _key_name_t pressed; + + uint8_t menu; + + AppButtom(); + ~AppButtom(); + + void run(); +}; diff --git a/examples/esp32-s3-eye/main/include/app_camera.hpp b/examples/esp32-s3-eye/main/include/app_camera.hpp new file mode 100644 index 0000000..2e02fa9 --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_camera.hpp @@ -0,0 +1,210 @@ +#pragma once + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "freertos/semphr.h" + +#include "esp_camera.h" + +#include "__base__.hpp" + +#if CONFIG_CAMERA_MODULE_WROVER_KIT +#define CAMERA_MODULE_NAME "Wrover Kit" +#define CAMERA_PIN_PWDN -1 +#define CAMERA_PIN_RESET -1 +#define CAMERA_PIN_XCLK 21 +#define CAMERA_PIN_SIOD 26 +#define CAMERA_PIN_SIOC 27 + +#define CAMERA_PIN_D7 35 +#define CAMERA_PIN_D6 34 +#define CAMERA_PIN_D5 39 +#define CAMERA_PIN_D4 36 +#define CAMERA_PIN_D3 19 +#define CAMERA_PIN_D2 18 +#define CAMERA_PIN_D1 5 +#define CAMERA_PIN_D0 4 +#define CAMERA_PIN_VSYNC 25 +#define CAMERA_PIN_HREF 23 +#define CAMERA_PIN_PCLK 22 + +#elif CONFIG_CAMERA_MODULE_ESP_EYE +#define CAMERA_MODULE_NAME "ESP-EYE" +#define CAMERA_PIN_PWDN -1 +#define CAMERA_PIN_RESET -1 +#define CAMERA_PIN_XCLK 4 +#define CAMERA_PIN_SIOD 18 +#define CAMERA_PIN_SIOC 23 + +#define CAMERA_PIN_D7 36 +#define CAMERA_PIN_D6 37 +#define CAMERA_PIN_D5 38 +#define CAMERA_PIN_D4 39 +#define CAMERA_PIN_D3 35 +#define CAMERA_PIN_D2 14 +#define CAMERA_PIN_D1 13 +#define CAMERA_PIN_D0 34 +#define CAMERA_PIN_VSYNC 5 +#define CAMERA_PIN_HREF 27 +#define CAMERA_PIN_PCLK 25 + +#elif CONFIG_CAMERA_MODULE_ESP_S2_KALUGA +#define CAMERA_MODULE_NAME "ESP-S2-KALUGA" +#define CAMERA_PIN_PWDN -1 +#define CAMERA_PIN_RESET -1 +#define CAMERA_PIN_XCLK 1 +#define CAMERA_PIN_SIOD 8 +#define CAMERA_PIN_SIOC 7 + +#define CAMERA_PIN_D7 38 +#define CAMERA_PIN_D6 21 +#define CAMERA_PIN_D5 40 +#define CAMERA_PIN_D4 39 +#define CAMERA_PIN_D3 42 +#define CAMERA_PIN_D2 41 +#define CAMERA_PIN_D1 37 +#define CAMERA_PIN_D0 36 +#define CAMERA_PIN_VSYNC 2 +#define CAMERA_PIN_HREF 3 +#define CAMERA_PIN_PCLK 33 + +#elif CONFIG_CAMERA_MODULE_ESP_S3_EYE +#define CAMERA_MODULE_NAME "ESP-S3-EYE" +#define CAMERA_PIN_PWDN -1 +#define CAMERA_PIN_RESET -1 + +#define CAMERA_PIN_VSYNC 6 +#define CAMERA_PIN_HREF 7 +#define CAMERA_PIN_PCLK 13 +#define CAMERA_PIN_XCLK 15 + +#define CAMERA_PIN_SIOD 4 +#define CAMERA_PIN_SIOC 5 + +#define CAMERA_PIN_D0 11 +#define CAMERA_PIN_D1 9 +#define CAMERA_PIN_D2 8 +#define CAMERA_PIN_D3 10 +#define CAMERA_PIN_D4 12 +#define CAMERA_PIN_D5 18 +#define CAMERA_PIN_D6 17 +#define CAMERA_PIN_D7 16 + +#elif CONFIG_CAMERA_MODEL_ESP32_CAM_BOARD +#define CAMERA_MODULE_NAME "ESP-DEVCAM" +#define CAMERA_PIN_PWDN 32 +#define CAMERA_PIN_RESET 33 + +#define CAMERA_PIN_XCLK 4 +#define CAMERA_PIN_SIOD 18 +#define CAMERA_PIN_SIOC 23 + +#define CAMERA_PIN_D7 36 +#define CAMERA_PIN_D6 19 +#define CAMERA_PIN_D5 21 +#define CAMERA_PIN_D4 39 +#define CAMERA_PIN_D3 35 +#define CAMERA_PIN_D2 14 +#define CAMERA_PIN_D1 13 +#define CAMERA_PIN_D0 34 +#define CAMERA_PIN_VSYNC 5 +#define CAMERA_PIN_HREF 27 +#define CAMERA_PIN_PCLK 25 + +#elif CONFIG_CAMERA_MODULE_M5STACK_PSRAM +#define CAMERA_MODULE_NAME "M5STACK-PSRAM" +#define CAMERA_PIN_PWDN -1 +#define CAMERA_PIN_RESET 15 + +#define CAMERA_PIN_XCLK 27 +#define CAMERA_PIN_SIOD 25 +#define CAMERA_PIN_SIOC 23 + +#define CAMERA_PIN_D7 19 +#define CAMERA_PIN_D6 36 +#define CAMERA_PIN_D5 18 +#define CAMERA_PIN_D4 39 +#define CAMERA_PIN_D3 5 +#define CAMERA_PIN_D2 34 +#define CAMERA_PIN_D1 35 +#define CAMERA_PIN_D0 32 +#define CAMERA_PIN_VSYNC 22 +#define CAMERA_PIN_HREF 26 +#define CAMERA_PIN_PCLK 21 + +#elif CONFIG_CAMERA_MODULE_M5STACK_WIDE +#define CAMERA_MODULE_NAME "M5STACK-WIDE" +#define CAMERA_PIN_PWDN -1 +#define CAMERA_PIN_RESET 15 +#define CAMERA_PIN_XCLK 27 +#define CAMERA_PIN_SIOD 22 +#define CAMERA_PIN_SIOC 23 + +#define CAMERA_PIN_D7 19 +#define CAMERA_PIN_D6 36 +#define CAMERA_PIN_D5 18 +#define CAMERA_PIN_D4 39 +#define CAMERA_PIN_D3 5 +#define CAMERA_PIN_D2 34 +#define CAMERA_PIN_D1 35 +#define CAMERA_PIN_D0 32 +#define CAMERA_PIN_VSYNC 25 +#define CAMERA_PIN_HREF 26 +#define CAMERA_PIN_PCLK 21 + +#elif CONFIG_CAMERA_MODULE_AI_THINKER +#define CAMERA_MODULE_NAME "AI-THINKER" +#define CAMERA_PIN_PWDN 32 +#define CAMERA_PIN_RESET -1 +#define CAMERA_PIN_XCLK 0 +#define CAMERA_PIN_SIOD 26 +#define CAMERA_PIN_SIOC 27 + +#define CAMERA_PIN_D7 35 +#define CAMERA_PIN_D6 34 +#define CAMERA_PIN_D5 39 +#define CAMERA_PIN_D4 36 +#define CAMERA_PIN_D3 21 +#define CAMERA_PIN_D2 19 +#define CAMERA_PIN_D1 18 +#define CAMERA_PIN_D0 5 +#define CAMERA_PIN_VSYNC 25 +#define CAMERA_PIN_HREF 23 +#define CAMERA_PIN_PCLK 22 + +#elif CONFIG_CAMERA_MODULE_CUSTOM +#define CAMERA_MODULE_NAME "CUSTOM" +#define CAMERA_PIN_PWDN CONFIG_CAMERA_PIN_PWDN +#define CAMERA_PIN_RESET CONFIG_CAMERA_PIN_RESET +#define CAMERA_PIN_XCLK CONFIG_CAMERA_PIN_XCLK +#define CAMERA_PIN_SIOD CONFIG_CAMERA_PIN_SIOD +#define CAMERA_PIN_SIOC CONFIG_CAMERA_PIN_SIOC + +#define CAMERA_PIN_D7 CONFIG_CAMERA_PIN_Y9 +#define CAMERA_PIN_D6 CONFIG_CAMERA_PIN_Y8 +#define CAMERA_PIN_D5 CONFIG_CAMERA_PIN_Y7 +#define CAMERA_PIN_D4 CONFIG_CAMERA_PIN_Y6 +#define CAMERA_PIN_D3 CONFIG_CAMERA_PIN_Y5 +#define CAMERA_PIN_D2 CONFIG_CAMERA_PIN_Y4 +#define CAMERA_PIN_D1 CONFIG_CAMERA_PIN_Y3 +#define CAMERA_PIN_D0 CONFIG_CAMERA_PIN_Y2 +#define CAMERA_PIN_VSYNC CONFIG_CAMERA_PIN_VSYNC +#define CAMERA_PIN_HREF CONFIG_CAMERA_PIN_HREF +#define CAMERA_PIN_PCLK CONFIG_CAMERA_PIN_PCLK +#endif + +#define XCLK_FREQ_HZ 15000000 + +class AppCamera : public Frame +{ +public: + AppCamera(const pixformat_t pixel_fromat, + const framesize_t frame_size, + const uint8_t fb_count, + QueueHandle_t queue_o = nullptr); + + void run(); +}; diff --git a/examples/esp32-s3-eye/main/include/app_face.hpp b/examples/esp32-s3-eye/main/include/app_face.hpp new file mode 100644 index 0000000..11bdb8e --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_face.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include "human_face_detect_msr01.hpp" +#include "human_face_detect_mnp01.hpp" +#include "face_recognition_tool.hpp" +// #if CONFIG_MFN_V1_Q8 +// #include "face_recognition_112_v1_s8.hpp" +// #elif CONFIG_MFN_V1_Q16 +#include "face_recognition_112_v1_s16.hpp" +// #endif + +#include "__base__.hpp" +#include "app_camera.hpp" +#include "app_buttom.hpp" +#include "app_speech.hpp" + +typedef enum +{ + IDLE = 0, + ENROLL = 1, + RECOGNIZE = 2, + DELETE = 3, +} recognizer_state_t; + +class AppFace : public Observer, public Frame +{ +private: + AppButtom *key; + AppSpeech *speech; + +public: + HumanFaceDetectMSR01 detector; + HumanFaceDetectMNP01 detector2; + + // #if CONFIG_MFN_V1_Q8 + // FaceRecognition112V1S8 *recognizer; + // #elif CONFIG_MFN_V1_Q16 + FaceRecognition112V1S16 *recognizer; + // #endif + + face_info_t recognize_result; + recognizer_state_t state; + recognizer_state_t state_previous; + + bool switch_on; + + uint8_t frame_count; + + AppFace(AppButtom *key, + AppSpeech *speech, + QueueHandle_t queue_i = nullptr, + QueueHandle_t queue_o = nullptr, + void (*callback)(camera_fb_t *) = esp_camera_fb_return); + ~AppFace(); + + void update(); + void run(); +}; diff --git a/examples/esp32-s3-eye/main/include/app_lcd.hpp b/examples/esp32-s3-eye/main/include/app_lcd.hpp new file mode 100644 index 0000000..6b11a4f --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_lcd.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include +#include "esp_log.h" +#include "screen_driver.h" + +#include "__base__.hpp" +#include "app_camera.hpp" +#include "app_buttom.hpp" +#include "app_speech.hpp" + +#define BOARD_LCD_MOSI 47 +#define BOARD_LCD_MISO -1 +#define BOARD_LCD_SCK 21 +#define BOARD_LCD_CS 44 +#define BOARD_LCD_DC 43 +#define BOARD_LCD_RST -1 +#define BOARD_LCD_BL 48 +#define BOARD_LCD_PIXEL_CLOCK_HZ (40 * 1000 * 1000) +#define BOARD_LCD_BK_LIGHT_ON_LEVEL 0 +#define BOARD_LCD_BK_LIGHT_OFF_LEVEL !BOARD_LCD_BK_LIGHT_ON_LEVEL +#define BOARD_LCD_H_RES 240 +#define BOARD_LCD_V_RES 240 +#define BOARD_LCD_CMD_BITS 8 +#define BOARD_LCD_PARAM_BITS 8 +#define LCD_HOST SPI2_HOST + +class AppLCD : public Observer, public Frame +{ +private: + AppButtom *key; + AppSpeech *speech; + +public: + scr_driver_t driver; + bool switch_on; + bool paper_drawn; + + AppLCD(AppButtom *key, + AppSpeech *speech, + QueueHandle_t xQueueFrameI = nullptr, + QueueHandle_t xQueueFrameO = nullptr, + void (*callback)(camera_fb_t *) = esp_camera_fb_return); + + void draw_wallpaper(); + void draw_color(int color); + + void update(); + + void run(); +}; diff --git a/examples/esp32-s3-eye/main/include/app_led.hpp b/examples/esp32-s3-eye/main/include/app_led.hpp new file mode 100644 index 0000000..8167916 --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_led.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "driver/gpio.h" + +#include "app_buttom.hpp" +#include "app_speech.hpp" + +class LED : public Observer +{ +private: + const gpio_num_t pin; + AppButtom *key; + AppSpeech *sr; + +public: + LED(const gpio_num_t pin, AppButtom *key, AppSpeech *sr); + ~LED(); + + void update(); +}; diff --git a/examples/esp32-s3-eye/main/include/app_motion.hpp b/examples/esp32-s3-eye/main/include/app_motion.hpp new file mode 100644 index 0000000..93a3909 --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_motion.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include "__base__.hpp" +#include "app_camera.hpp" +#include "app_buttom.hpp" +#include "app_speech.hpp" + +class AppMotion : public Observer, public Frame +{ +private: + AppButtom *key; + AppSpeech *speech; + +public: + bool switch_on; + + AppMotion(AppButtom *key, + AppSpeech *speech, + QueueHandle_t queue_i = nullptr, + QueueHandle_t queue_o = nullptr, + void (*callback)(camera_fb_t *) = esp_camera_fb_return); + + void update(); + + void run(); +}; diff --git a/examples/esp32-s3-eye/main/include/app_speech.hpp b/examples/esp32-s3-eye/main/include/app_speech.hpp new file mode 100644 index 0000000..3b6009a --- /dev/null +++ b/examples/esp32-s3-eye/main/include/app_speech.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "freertos/semphr.h" + +#include "esp_afe_sr_iface.h" + +#include "__base__.hpp" + +class AppSpeech : public Subject +{ +public: + const esp_afe_sr_iface_t *afe_handle; + esp_afe_sr_data_t *afe_data; + bool detected; + command_word_t command; + + AppSpeech(); + + void run(); +}; diff --git a/examples/esp32-s3-eye/main/src/app_buttom.cpp b/examples/esp32-s3-eye/main/src/app_buttom.cpp new file mode 100644 index 0000000..591022b --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_buttom.cpp @@ -0,0 +1,79 @@ +#include "app_buttom.hpp" + +#include +#include + +#include "driver/adc_common.h" +#include "esp_log.h" +#include "esp_adc_cal.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" + +// ADC Channels +#define ADC1_EXAMPLE_CHAN0 ADC1_CHANNEL_0 +// ADC Attenuation +#define ADC_EXAMPLE_ATTEN ADC_ATTEN_DB_11 +// ADC Calibration +#if CONFIG_IDF_TARGET_ESP32 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_VREF +#elif CONFIG_IDF_TARGET_ESP32S2 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP +#elif CONFIG_IDF_TARGET_ESP32C3 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP +#elif CONFIG_IDF_TARGET_ESP32S3 +#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP_FIT +#endif + +#define PRESS_INTERVAL 500000 + +static const char *TAG = "App/Buttom"; + +AppButtom::AppButtom() : key_configs({{_MENU, 2800, 3000}, {_PLAY, 2250, 2450}, {_UP, 300, 500}, {_DOWN, 850, 1050}}), + pressed(_IDLE) +{ + ESP_ERROR_CHECK(adc1_config_width((adc_bits_width_t)ADC_WIDTH_BIT_DEFAULT)); + ESP_ERROR_CHECK(adc1_config_channel_atten(ADC1_EXAMPLE_CHAN0, ADC_EXAMPLE_ATTEN)); +} + +static void task(AppButtom *self) +{ + int64_t backup_time = esp_timer_get_time(); + int64_t last_time = esp_timer_get_time(); + + uint8_t menu_count = 0; + + while (true) + { + uint32_t voltage = adc1_get_raw(ADC1_EXAMPLE_CHAN0); + backup_time = esp_timer_get_time(); + for (auto key_config : self->key_configs) + { + if ((voltage >= key_config.min) && (voltage <= key_config.max)) + { + if (((backup_time - last_time) > PRESS_INTERVAL)) + { + ESP_LOGD(TAG, "Key[%d] is pressed", self->pressed); + self->pressed = key_config.key; + + if (self->pressed == _MENU) + { + self->menu++; + self->menu %= (MENU_MOTION_DETECTION + 1); + } + + last_time = backup_time; + self->notify(); + + self->pressed = _IDLE; + break; + } + } + } + vTaskDelay(pdMS_TO_TICKS(10)); + } +} + +void AppButtom::run() +{ + xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 3 * 1024, this, 5, NULL, 0); +} diff --git a/examples/esp32-s3-eye/main/src/app_camera.cpp b/examples/esp32-s3-eye/main/src/app_camera.cpp new file mode 100644 index 0000000..824d91a --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_camera.cpp @@ -0,0 +1,98 @@ +#include "app_camera.hpp" + +#include "assert.h" + +#include "esp_log.h" +#include "esp_system.h" + +const static char TAG[] = "App/Camera"; + +AppCamera::AppCamera(const pixformat_t pixel_fromat, + const framesize_t frame_size, + const uint8_t fb_count, + QueueHandle_t queue_o) : Frame(nullptr, queue_o, nullptr) +{ + ESP_LOGI(TAG, "Camera module is %s", CAMERA_MODULE_NAME); + +#if CONFIG_CAMERA_MODEL_ESP_EYE || CONFIG_CAMERA_MODEL_ESP32_CAM_BOARD + /* IO13, IO14 is designed for JTAG by default, + * to use it as generalized input, + * firstly declair it as pullup input */ + gpio_config_t conf; + conf.mode = GPIO_MODE_INPUT; + conf.pull_up_en = GPIO_PULLUP_ENABLE; + conf.pull_down_en = GPIO_PULLDOWN_DISABLE; + conf.intr_type = GPIO_INTR_DISABLE; + conf.pin_bit_mask = 1LL << 13; + gpio_config(&conf); + conf.pin_bit_mask = 1LL << 14; + gpio_config(&conf); +#endif + + camera_config_t config; + config.ledc_channel = LEDC_CHANNEL_0; + config.ledc_timer = LEDC_TIMER_0; + config.pin_d0 = CAMERA_PIN_D0; + config.pin_d1 = CAMERA_PIN_D1; + config.pin_d2 = CAMERA_PIN_D2; + config.pin_d3 = CAMERA_PIN_D3; + config.pin_d4 = CAMERA_PIN_D4; + config.pin_d5 = CAMERA_PIN_D5; + config.pin_d6 = CAMERA_PIN_D6; + config.pin_d7 = CAMERA_PIN_D7; + config.pin_xclk = CAMERA_PIN_XCLK; + config.pin_pclk = CAMERA_PIN_PCLK; + config.pin_vsync = CAMERA_PIN_VSYNC; + config.pin_href = CAMERA_PIN_HREF; + config.pin_sscb_sda = CAMERA_PIN_SIOD; + config.pin_sscb_scl = CAMERA_PIN_SIOC; + config.pin_pwdn = CAMERA_PIN_PWDN; + config.pin_reset = CAMERA_PIN_RESET; + config.xclk_freq_hz = XCLK_FREQ_HZ; + config.pixel_format = pixel_fromat; + config.frame_size = frame_size; + config.jpeg_quality = 12; + config.fb_count = fb_count; + config.fb_location = CAMERA_FB_IN_PSRAM; + config.grab_mode = CAMERA_GRAB_WHEN_EMPTY; + + // camera init + esp_err_t err = esp_camera_init(&config); + if (err != ESP_OK) + { + ESP_LOGE(TAG, "Camera init failed with error 0x%x", err); + return; + } + + sensor_t *s = esp_camera_sensor_get(); + s->set_vflip(s, 1); // flip it back + // initial sensors are flipped vertically and colors are a bit saturated + if (s->id.PID == OV3660_PID) + { + s->set_brightness(s, 1); // up the blightness just a bit + s->set_saturation(s, -2); // lower the saturation + } + s->set_sharpness(s, 2); + s->set_awb_gain(s, 2); +} + +static void task(AppCamera *self) +{ + ESP_LOGD(TAG, "Start"); + while (true) + { + if (self->queue_o == nullptr) + break; + + camera_fb_t *frame = esp_camera_fb_get(); + if (frame) + xQueueSend(self->queue_o, &frame, portMAX_DELAY); + } + ESP_LOGD(TAG, "Stop"); + vTaskDelete(NULL); +} + +void AppCamera::run() +{ + xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 2 * 1024, this, 5, NULL, 1); +} diff --git a/examples/esp32-s3-eye/main/src/app_face.cpp b/examples/esp32-s3-eye/main/src/app_face.cpp new file mode 100644 index 0000000..56b7b9f --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_face.cpp @@ -0,0 +1,231 @@ +#include "app_face.hpp" + +#include + +#include "esp_log.h" +#include "esp_camera.h" + +#include "dl_image.hpp" +#include "fb_gfx.h" + +#include "who_ai_utils.hpp" + +static const char TAG[] = "App/Face"; + +#define RGB565_MASK_RED 0xF800 +#define RGB565_MASK_GREEN 0x07E0 +#define RGB565_MASK_BLUE 0x001F + +#define FRAME_DELAY_NUM 16 + +static void rgb_print(camera_fb_t *fb, uint32_t color, const char *str) +{ + fb_gfx_print(fb, (fb->width - (strlen(str) * 14)) / 2, 10, color, str); +} + +static int rgb_printf(camera_fb_t *fb, uint32_t color, const char *format, ...) +{ + char loc_buf[64]; + char *temp = loc_buf; + int len; + va_list arg; + va_list copy; + va_start(arg, format); + va_copy(copy, arg); + len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg); + va_end(copy); + if (len >= sizeof(loc_buf)) + { + temp = (char *)malloc(len + 1); + if (temp == NULL) + { + return 0; + } + } + vsnprintf(temp, len + 1, format, arg); + va_end(arg); + rgb_print(fb, color, temp); + if (len > 64) + { + free(temp); + } + return len; +} + +AppFace::AppFace(AppButtom *key, + AppSpeech *speech, + QueueHandle_t queue_i, + QueueHandle_t queue_o, + void (*callback)(camera_fb_t *)) : Frame(queue_i, queue_o, callback), + key(key), + speech(speech), + detector(0.3F, 0.3F, 10, 0.3F), + detector2(0.4F, 0.3F, 10), + state(IDLE), + switch_on(false) +// recognizer( +// #if CONFIG_MFN_V1_Q8 +// new FaceRecognition112V1S8() +// #elif CONFIG_MFN_V1_Q16 +// new FaceRecognition112V1S16() +// #endif +// ) +{ + // this->recognizer = new FaceRecognition112V1S8(); + this->recognizer = new FaceRecognition112V1S16(); + + this->recognizer->set_partition(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, "fr"); + int partition_result = this->recognizer->set_ids_from_flash(); +} + +AppFace::~AppFace() +{ +} + +void AppFace::update() +{ + // Parse key + if (this->key->pressed > _IDLE) + { + if (this->key->pressed == _MENU) + { + this->switch_on = (this->key->menu == MENU_FACE_RECOGNITION) ? true : false; + ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF"); + } + else if (this->key->pressed == _PLAY) + { + this->state = RECOGNIZE; + } + else if (this->key->pressed == _UP) + { + this->state = ENROLL; + } + else if (this->key->pressed == _DOWN) + { + this->state = DELETE; + } + } + + // Parse speech recognition + if (this->speech->command > COMMAND_NOT_DETECTED) + { + if (this->speech->command >= MENU_STOP_WORKING && this->speech->command <= MENU_MOTION_DETECTION) + { + this->switch_on = (this->speech->command == MENU_FACE_RECOGNITION) ? true : false; + ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF"); + } + else if (this->speech->command == ACTION_ENROLL) + { + this->state = ENROLL; + } + else if (this->speech->command == ACTION_RECOGNIZE) + { + this->state = RECOGNIZE; + } + else if (this->speech->command == ACTION_DELETE) + { + this->state = DELETE; + } + } + ESP_LOGD(TAG, "Human face recognition state = %d", this->state); +} + +static void task(AppFace *self) +{ + ESP_LOGD(TAG, "Start"); + camera_fb_t *frame = nullptr; + + while (true) + { + if (self->queue_i == nullptr) + break; + + if (xQueueReceive(self->queue_i, &frame, portMAX_DELAY)) + { + if (self->switch_on) + { + std::list &detect_candidates = self->detector.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}); + std::list &detect_results = self->detector2.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_candidates); + + if (detect_results.size()) + { + // print_detection_result(detect_results); + draw_detection_result((uint16_t *)frame->buf, frame->height, frame->width, detect_results); + } + + if (self->state && detect_results.size() == 1) + { + switch (self->state) + { + case ENROLL: + self->recognizer->enroll_id((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint, "", true); + ESP_LOGW(TAG, "Enroll ID %d", self->recognizer->get_enrolled_ids().back().id); + break; + + case RECOGNIZE: + self->recognize_result = self->recognizer->recognize((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint); + // print_detection_result(detect_results); + ESP_LOGD(TAG, "Similarity: %f", self->recognize_result.similarity); + if (self->recognize_result.id > 0) + ESP_LOGI(TAG, "Match ID: %d", self->recognize_result.id); + else + ESP_LOGE(TAG, "Match ID: %d", self->recognize_result.id); + break; + + case DELETE: + vTaskDelay(10); + self->recognizer->delete_id(true); + ESP_LOGE(TAG, "%d IDs left", self->recognizer->get_enrolled_id_num()); + break; + + default: + break; + } + + self->state_previous = self->state; + self->state = IDLE; + self->frame_count = FRAME_DELAY_NUM; + } + + // Write result on several frames of image + if (self->frame_count) + { + switch (self->state_previous) + { + case DELETE: + rgb_printf(frame, RGB565_MASK_RED, "%d IDs left", self->recognizer->get_enrolled_id_num()); + break; + + case RECOGNIZE: + if (self->recognize_result.id > 0) + rgb_printf(frame, RGB565_MASK_GREEN, "ID %d", self->recognize_result.id); + else + rgb_print(frame, RGB565_MASK_RED, "who ?"); + break; + + case ENROLL: + rgb_printf(frame, RGB565_MASK_BLUE, "Enroll: ID %d", self->recognizer->get_enrolled_ids().back().id); + break; + + default: + break; + } + + self->frame_count--; + } + } + + if (self->queue_o) + xQueueSend(self->queue_o, &frame, portMAX_DELAY); + else + self->callback(frame); + } + } + ESP_LOGD(TAG, "Stop"); + vTaskDelete(NULL); +} + +void AppFace::run() +{ + xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 5 * 1024, this, 5, NULL, 0); +} \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/src/app_lcd.cpp b/examples/esp32-s3-eye/main/src/app_lcd.cpp new file mode 100644 index 0000000..a71911b --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_lcd.cpp @@ -0,0 +1,178 @@ +#include "app_lcd.hpp" + +#include + +#include "esp_camera.h" +#include "esp_lcd_panel_io.h" +#include "esp_lcd_panel_vendor.h" +#include "esp_lcd_panel_ops.h" +#include "driver/spi_master.h" +#include "driver/gpio.h" + +#include "logo_en_240x240_lcd.h" + +static const char TAG[] = "App/LCD"; + +AppLCD::AppLCD(AppButtom *key, + AppSpeech *speech, + QueueHandle_t queue_i, + QueueHandle_t queue_o, + void (*callback)(camera_fb_t *)) : Frame(queue_i, queue_o, callback), + key(key), + speech(speech), + switch_on(false) +{ + do + { + spi_config_t bus_conf = { + .miso_io_num = (gpio_num_t)BOARD_LCD_MISO, + .mosi_io_num = (gpio_num_t)BOARD_LCD_MOSI, + .sclk_io_num = (gpio_num_t)BOARD_LCD_SCK, + .max_transfer_sz = 2 * 240 * 240 + 10, + }; + spi_bus_handle_t spi_bus = spi_bus_create(SPI2_HOST, &bus_conf); + + scr_interface_spi_config_t spi_lcd_cfg = { + .spi_bus = spi_bus, + .pin_num_cs = BOARD_LCD_CS, + .pin_num_dc = BOARD_LCD_DC, + .clk_freq = 40 * 1000000, + .swap_data = 0, + }; + + scr_interface_driver_t *iface_drv; + scr_interface_create(SCREEN_IFACE_SPI, &spi_lcd_cfg, &iface_drv); + if (ESP_OK != scr_find_driver(SCREEN_CONTROLLER_ST7789, &this->driver)) + { + ESP_LOGE(TAG, "screen find failed"); + break; + } + + scr_controller_config_t lcd_cfg = { + .interface_drv = iface_drv, + .pin_num_rst = BOARD_LCD_RST, + .pin_num_bckl = BOARD_LCD_BL, + .rst_active_level = 0, + .bckl_active_level = 0, + .width = 240, + .height = 240, + .offset_hor = 0, + .offset_ver = 0, + .rotate = (scr_dir_t)0, + }; + + if (ESP_OK != this->driver.init(&lcd_cfg)) + { + ESP_LOGE(TAG, "screen initialize failed"); + break; + } + + scr_info_t lcd_info; + this->driver.get_info(&lcd_info); + ESP_LOGI(TAG, "Screen name:%s | width:%d | height:%d", lcd_info.name, lcd_info.width, lcd_info.height); + + this->draw_color(0x000000); + vTaskDelay(pdMS_TO_TICKS(500)); + this->draw_wallpaper(); + vTaskDelay(pdMS_TO_TICKS(1000)); + } while (0); +} + +void AppLCD::draw_wallpaper() +{ + uint16_t *pixels = (uint16_t *)heap_caps_malloc((logo_en_240x240_lcd_width * logo_en_240x240_lcd_height) * sizeof(uint16_t), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM); + if (NULL == pixels) + { + ESP_LOGE(TAG, "Memory for bitmap is not enough"); + return; + } + memcpy(pixels, logo_en_240x240_lcd, (logo_en_240x240_lcd_width * logo_en_240x240_lcd_height) * sizeof(uint16_t)); + this->driver.draw_bitmap(0, 0, logo_en_240x240_lcd_width, logo_en_240x240_lcd_height, (uint16_t *)pixels); + heap_caps_free(pixels); + + this->paper_drawn = true; +} + +void AppLCD::draw_color(int color) +{ + scr_info_t lcd_info; + this->driver.get_info(&lcd_info); + uint16_t *buffer = (uint16_t *)malloc(lcd_info.width * sizeof(uint16_t)); + if (buffer) + { + for (size_t i = 0; i < lcd_info.width; i++) + { + buffer[i] = color; + } + + for (int y = 0; y < lcd_info.height; y++) + { + this->driver.draw_bitmap(0, y, lcd_info.width, 1, buffer); + } + + free(buffer); + } + else + { + ESP_LOGE(TAG, "Memory for bitmap is not enough"); + } +} + +void AppLCD::update() +{ + if (this->key->pressed > _IDLE) + { + if (this->key->pressed == _MENU) + { + this->switch_on = (this->key->menu == MENU_STOP_WORKING) ? false : true; + ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF"); + } + } + + if (this->speech->command > COMMAND_NOT_DETECTED) + { + if (this->speech->command >= MENU_STOP_WORKING && this->speech->command <= MENU_MOTION_DETECTION) + { + this->switch_on = (this->speech->command == MENU_STOP_WORKING) ? false : true; + ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF"); + } + } + + if (this->switch_on == false) + { + this->paper_drawn = false; + } +} + +static void task(AppLCD *self) +{ + ESP_LOGD(TAG, "Start"); + + camera_fb_t *frame = nullptr; + while (true) + { + if (self->queue_i == nullptr) + break; + + if (xQueueReceive(self->queue_i, &frame, portMAX_DELAY)) + { + if (self->switch_on) + self->driver.draw_bitmap(0, 0, frame->width, frame->height, (uint16_t *)frame->buf); + else if (self->paper_drawn == false) + self->draw_wallpaper(); + + if (self->queue_o) + xQueueSend(self->queue_o, &frame, portMAX_DELAY); + else + self->callback(frame); + } + } + ESP_LOGD(TAG, "Stop"); + self->draw_wallpaper(); + vTaskDelete(NULL); +} + +void AppLCD::run() +{ + xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 2 * 1024, this, 5, NULL, 0); +} \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/src/app_led.cpp b/examples/esp32-s3-eye/main/src/app_led.cpp new file mode 100644 index 0000000..3df0b8f --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_led.cpp @@ -0,0 +1,124 @@ +#include "app_led.hpp" + +#include "esp_log.h" + +const static char TAG[] = "App/LED"; + +typedef enum +{ + LED_ALWAYS_OFF = 0, + LED_ALWAYS_ON = 1, + LED_OFF_1S = 2, + LED_OFF_2S = 3, + LED_OFF_4S = 4, + LED_ON_1S = 5, + LED_ON_2S = 6, + LED_ON_4S = 7, + LED_BLINK_1S = 8, + LED_BLINK_2S = 9, + LED_BLINK_4S = 10, +} led_mode_t; + +LED::LED(const gpio_num_t pin, AppButtom *key, AppSpeech *sr) : pin(pin), key(key), sr(sr) +{ + // initialize GPIO + gpio_config_t gpio_conf; + gpio_conf.mode = GPIO_MODE_OUTPUT_OD; + gpio_conf.pull_up_en = GPIO_PULLUP_ENABLE; + + gpio_conf.intr_type = GPIO_INTR_DISABLE; + gpio_conf.pin_bit_mask = 1LL << this->pin; + gpio_config(&gpio_conf); + + gpio_set_level(this->pin, 0); +} + +void LED::update() +{ + led_mode_t mode = LED_ALWAYS_OFF; + + // parse key + if (this->key->pressed) + { + mode = LED_BLINK_2S; + } + // parse speech recognition + else if (this->sr->detected) + { + if (this->sr->command > COMMAND_NOT_DETECTED) + mode = LED_BLINK_1S; + else + mode = LED_ALWAYS_ON; + } + + // do + switch (mode) + { + case LED_ALWAYS_OFF: + gpio_set_level(this->pin, 0); + break; + case LED_ALWAYS_ON: + gpio_set_level(this->pin, 1); + break; + case LED_OFF_1S: + gpio_set_level(this->pin, 0); + vTaskDelay(1000 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 1); + break; + case LED_OFF_2S: + gpio_set_level(this->pin, 0); + vTaskDelay(2000 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 1); + break; + case LED_OFF_4S: + gpio_set_level(this->pin, 0); + vTaskDelay(4000 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 1); + break; + case LED_ON_1S: + gpio_set_level(this->pin, 1); + vTaskDelay(1000 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 0); + break; + case LED_ON_2S: + gpio_set_level(this->pin, 1); + vTaskDelay(2000 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 0); + break; + case LED_ON_4S: + gpio_set_level(this->pin, 1); + vTaskDelay(4000 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 0); + break; + case LED_BLINK_1S: + for (int i = 0; i < 2; ++i) + { + gpio_set_level(this->pin, 1); + vTaskDelay(250 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 0); + vTaskDelay(250 / portTICK_PERIOD_MS); + } + break; + case LED_BLINK_2S: + for (int i = 0; i < 4; ++i) + { + gpio_set_level(this->pin, 1); + vTaskDelay(250 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 0); + vTaskDelay(250 / portTICK_PERIOD_MS); + } + break; + case LED_BLINK_4S: + for (int i = 0; i < 8; ++i) + { + gpio_set_level(this->pin, 1); + vTaskDelay(250 / portTICK_PERIOD_MS); + gpio_set_level(this->pin, 0); + vTaskDelay(250 / portTICK_PERIOD_MS); + } + break; + + default: + break; + } +} \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/src/app_motion.cpp b/examples/esp32-s3-eye/main/src/app_motion.cpp new file mode 100644 index 0000000..3c78056 --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_motion.cpp @@ -0,0 +1,82 @@ +#include "app_motion.hpp" + +#include "assert.h" + +#include "esp_log.h" +#include "esp_camera.h" + +#include "dl_image.hpp" + +static const char TAG[] = "App/Motion"; + +AppMotion::AppMotion(AppButtom *key, + AppSpeech *speech, + QueueHandle_t queue_i, + QueueHandle_t queue_o, + void (*callback)(camera_fb_t *)) : Frame(queue_i, queue_o, callback), + key(key), + speech(speech), + switch_on(false) {} + +void AppMotion::update() +{ + if (this->key->pressed > _IDLE) + { + if (this->key->pressed == _MENU) + { + this->switch_on = (this->key->menu == MENU_MOTION_DETECTION) ? true : false; + ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF"); + } + } + + if (this->speech->command > COMMAND_NOT_DETECTED) + { + if (this->speech->command >= MENU_STOP_WORKING && this->speech->command <= MENU_MOTION_DETECTION) + { + this->switch_on = (this->speech->command == MENU_MOTION_DETECTION) ? true : false; + ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF"); + } + } +} + +static void task(AppMotion *self) +{ + ESP_LOGD(TAG, "Start"); + while (true) + { + if (self->queue_i == nullptr) + break; + + camera_fb_t *frame1 = NULL; + camera_fb_t *frame2 = NULL; + if (xQueueReceive(self->queue_i, &frame1, portMAX_DELAY)) + { + if (self->switch_on) + { + if (xQueueReceive(self->queue_i, &frame2, portMAX_DELAY)) + { + uint32_t moving_point_number = dl::image::get_moving_point_number((uint16_t *)frame1->buf, (uint16_t *)frame2->buf, frame1->height, frame1->width, 8, 15); + if (moving_point_number > 50) + { + ESP_LOGI(TAG, "Something moved!"); + dl::image::draw_filled_rectangle((uint16_t *)frame1->buf, frame1->height, frame1->width, 0, 0, 20, 20); + } + + self->callback(frame2); + } + } + + if (self->queue_o) + xQueueSend(self->queue_o, &frame1, portMAX_DELAY); + else + self->callback(frame1); + } + } + ESP_LOGD(TAG, "Stop"); + vTaskDelete(NULL); +} + +void AppMotion::run() +{ + xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 3 * 1024, this, 6, NULL, 0); +} \ No newline at end of file diff --git a/examples/esp32-s3-eye/main/src/app_speech.cpp b/examples/esp32-s3-eye/main/src/app_speech.cpp new file mode 100644 index 0000000..f566918 --- /dev/null +++ b/examples/esp32-s3-eye/main/src/app_speech.cpp @@ -0,0 +1,214 @@ +#include "app_speech.hpp" + +#include "driver/i2s.h" +#include "esp_log.h" + +#include "dl_lib_coefgetter_if.h" +#include "esp_wn_iface.h" +#include "esp_wn_models.h" +#include "esp_afe_sr_models.h" +#include "esp_mn_iface.h" +#include "esp_mn_models.h" +#include "esp_spiffs.h" + +#define I2S_CHANNEL_NUM 1 +#define I2S_CH ((i2s_port_t)1) + +static const char *TAG = "App/Speech"; + +static void spiffs_init(void) +{ + ESP_LOGI(TAG, "Initializing SPIFFS"); + + esp_vfs_spiffs_conf_t conf = { + .base_path = "/srmodel", + .partition_label = "model", + .max_files = 5, + .format_if_mount_failed = true}; + // Use settings defined above to initialize and mount SPIFFS filesystem. + // Note: esp_vfs_spiffs_register is an all-in-one convenience function. + esp_err_t ret = esp_vfs_spiffs_register(&conf); + if (ret != ESP_OK) + { + if (ret == ESP_FAIL) + { + ESP_LOGE(TAG, "Failed to mount or format filesystem"); + } + else if (ret == ESP_ERR_NOT_FOUND) + { + ESP_LOGE(TAG, "Failed to find SPIFFS partition"); + } + else + { + ESP_LOGE(TAG, "Failed to initialize SPIFFS (%s)", esp_err_to_name(ret)); + } + return; + } + size_t total = 0, used = 0; + ret = esp_spiffs_info("model", &total, &used); + if (ret != ESP_OK) + { + ESP_LOGE(TAG, "Failed to get SPIFFS partition information (%s)", esp_err_to_name(ret)); + } + else + { + ESP_LOGI(TAG, "Partition size: total: %d, used: %d", total, used); + } +} + +static void i2s_init(void) +{ + i2s_config_t i2s_config = { + .mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX), // the mode must be set according to DSP configuration + .sample_rate = 16000, // must be the same as DSP configuration + .bits_per_sample = (i2s_bits_per_sample_t)32, // must be the same as DSP configuration + .channel_format = I2S_CHANNEL_FMT_ONLY_LEFT, // must be the same as DSP configuration + .communication_format = I2S_COMM_FORMAT_STAND_I2S, + .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2, + .dma_buf_count = 3, + .dma_buf_len = 300, + }; + i2s_pin_config_t pin_config = { + .mck_io_num = I2S_PIN_NO_CHANGE, + .bck_io_num = 41, // IIS_SCLK + .ws_io_num = 42, // IIS_LCLK + .data_out_num = I2S_PIN_NO_CHANGE, // IIS_DSIN + .data_in_num = 2 // IIS_DOUT + }; + i2s_driver_install(I2S_CH, &i2s_config, 0, NULL); + i2s_set_pin(I2S_CH, &pin_config); + i2s_zero_dma_buffer(I2S_CH); +} + +static void feed_handler(AppSpeech *self) +{ + esp_afe_sr_data_t *afe_data = self->afe_data; + int audio_chunksize = self->afe_handle->get_feed_chunksize(afe_data); + int nch = self->afe_handle->get_channel_num(afe_data); + size_t samp_len = audio_chunksize * I2S_CHANNEL_NUM; + size_t samp_len_bytes = samp_len * sizeof(int32_t); + int32_t *i2s_buff = (int32_t *)malloc(samp_len_bytes); + assert(i2s_buff); + size_t bytes_read; + // FILE *fp = fopen("/sdcard/out", "a+"); + // if (fp == NULL) ESP_LOGE(TAG,"can not open file\n"); + + while (true) + { + i2s_read(I2S_CH, i2s_buff, samp_len_bytes, &bytes_read, portMAX_DELAY); + + for (int i = 0; i < samp_len; ++i) + { + i2s_buff[i] = i2s_buff[i] >> 14; // 32:8为有效位， 8:0为低8位， 全为0， AFE的输入为16位语音数据，拿29：13位是为了对语音信号放大。 + } + // FatfsComboWrite(i2s_buff, audio_chunksize * I2S_CHANNEL_NUM * sizeof(int16_t), 1, fp); + + self->afe_handle->feed(afe_data, (int16_t *)i2s_buff); + } + self->afe_handle->destroy(afe_data); + vTaskDelete(NULL); +} + +static void detect_hander(AppSpeech *self) +{ + esp_afe_sr_data_t *afe_data = self->afe_data; + int afe_chunksize = self->afe_handle->get_fetch_chunksize(afe_data); + int nch = self->afe_handle->get_channel_num(afe_data); + int16_t *buff = (int16_t *)malloc(afe_chunksize * sizeof(int16_t)); + assert(buff); + + static const esp_mn_iface_t *multinet = &MULTINET_MODEL; + model_iface_data_t *model_data = multinet->create((const model_coeff_getter_t *)&MULTINET_COEFF, 5760); + int mu_chunksize = multinet->get_samp_chunksize(model_data); + int chunk_num = multinet->get_samp_chunknum(model_data); + assert(mu_chunksize == afe_chunksize); + + // FILE *fp = fopen("/sdcard/out", "w"); + // if (fp == NULL) ESP_LOGE(TAG,"can not open file\n"); + + ESP_LOGI(TAG, "Ready"); + + self->detected = false; + + while (true) + { + int res = self->afe_handle->fetch(afe_data, buff); + + if (res == AFE_FETCH_WWE_DETECTED) + { + ESP_LOGI(TAG, ">>> Say your command <<<"); + self->detected = true; + self->afe_handle->disable_wakenet(afe_data); + self->notify(); + } + + if (self->detected) + { + // Detect command + self->command = (command_word_t)multinet->detect(model_data, buff); + // FatfsComboWrite(buff, afe_chunksize * sizeof(int16_t), 1, fp); + + if (self->command == COMMAND_NOT_DETECTED) + continue; + else if (self->command == COMMAND_TIMEOUT) + { + self->afe_handle->enable_wakenet(afe_data); + // self->afe_handle->enable_aec(afe_data); + + self->detected = false; + ESP_LOGI(TAG, ">>> Waiting to be waken up <<<"); + self->notify(); + } + else + { + self->notify(); + ESP_LOGD(TAG, "Command: %d", self->command); + +#ifndef CONFIG_SR_MN_CN_MULTINET3_CONTINUOUS_RECOGNITION + self->afe_handle->enable_wakenet(afe_data); + // self->afe_handle->enable_aec(afe_data); + self->detected = false; + ESP_LOGI(TAG, ">>> Waiting to be waken up <<<"); +#endif + self->command = COMMAND_TIMEOUT; + self->notify(); + } + } + } + self->afe_handle->destroy(afe_data); + vTaskDelete(NULL); +} + +AppSpeech::AppSpeech() : afe_handle(&esp_afe_sr_1mic), detected(false), command(COMMAND_TIMEOUT) +{ + spiffs_init(); + i2s_init(); + // sd_card_mount("/sdcard"); + afe_config_t afe_config = { + .aec_init = true, + .se_init = true, + .vad_init = true, + .wakenet_init = true, + .vad_mode = 3, + .wakenet_model = &WAKENET_MODEL, + .wakenet_coeff = (const model_coeff_getter_t *)&WAKENET_COEFF, + .wakenet_mode = DET_MODE_2CH_90, + .afe_mode = SR_MODE_LOW_COST, + .afe_perferred_core = 0, + .afe_perferred_priority = 5, + .afe_ringbuf_size = 50, + .alloc_from_psram = AFE_PSRAM_MEDIA_COST, + .agc_mode = 2, + }; + afe_config.aec_init = false; + afe_config.se_init = false; + afe_config.vad_init = false; + afe_config.afe_ringbuf_size = 10; + this->afe_data = this->afe_handle->create_from_config(&afe_config); +} + +void AppSpeech::run() +{ + xTaskCreatePinnedToCore((TaskFunction_t)feed_handler, "SR_feed", 4 * 1024, this, 5, NULL, 0); + xTaskCreatePinnedToCore((TaskFunction_t)detect_hander, "SR_detect", 5 * 1024, this, 5, NULL, 0); +} \ No newline at end of file diff --git a/examples/esp32-s3-eye/partitions.csv b/examples/esp32-s3-eye/partitions.csv new file mode 100644 index 0000000..39f732e --- /dev/null +++ b/examples/esp32-s3-eye/partitions.csv @@ -0,0 +1,7 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild + +factory, app, factory, 0x010000, 4000K, +model, data, spiffs, , 3900K, +nvs, data, nvs, , 16K, +fr, data, , , 128K, diff --git a/examples/esp32-s3-eye/sdkconfig.defaults b/examples/esp32-s3-eye/sdkconfig.defaults new file mode 100644 index 0000000..6d4de9f --- /dev/null +++ b/examples/esp32-s3-eye/sdkconfig.defaults @@ -0,0 +1,59 @@ +# General Configuration +CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y + +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y + +CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" + +CONFIG_ESPTOOLPY_NO_STUB=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_SPIRAM_MODE_OCT=y + +# ESP32-S3 Configuration +CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32S3_SPIRAM_SUPPORT=y +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_8WAYS=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y + + +# Image Relation +CONFIG_CAMERA_MODULE_ESP_S3_EYE=y +CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y +CONFIG_MFN_V1_Q8=y + + +# Speech Relation + +# Chinese +CONFIG_SR_WN_MODEL_WN7_QUANT8=y +CONFIG_SR_WN_WN7_HILEXIN=y + +CONFIG_SR_MN_CHINESE=y +CONFIG_SR_MN_CN_MULTINET4_5_SINGLE_RECOGNITION=y +CONFIG_CN_SPEECH_COMMAND_ID0="ting zhi gong zuo" +CONFIG_CN_SPEECH_COMMAND_ID1="jin xian shi" +CONFIG_CN_SPEECH_COMMAND_ID2="ren lian shi bie" +CONFIG_CN_SPEECH_COMMAND_ID3="yi dong zhen ce" +CONFIG_CN_SPEECH_COMMAND_ID4="tian jia ren lian" +CONFIG_CN_SPEECH_COMMAND_ID5="shan chu ren lian" +CONFIG_CN_SPEECH_COMMAND_ID6="shi bie yi xia" + +# English +# CONFIG_SR_WN_MODEL_WN8_QUANT8=y +# CONFIG_SR_WN_WN8_HIESP=y +# +# CONFIG_SR_MN_ENGLISH=y +# CONFIG_SR_MN_EN_MULTINET5_SINGLE_RECOGNITION_QUANT8=y +# CONFIG_EN_SPEECH_COMMAND_ID0="STnP WkKgl" +# CONFIG_EN_SPEECH_COMMAND_ID1="DgSPLd bNLm" +# CONFIG_EN_SPEECH_COMMAND_ID2="FdS RfKcGNgscN" +# CONFIG_EN_SPEECH_COMMAND_ID3="MbscN DgTfKscN" +# CONFIG_EN_SPEECH_COMMAND_ID4="fNTk FdS" +# CONFIG_EN_SPEECH_COMMAND_ID5="DgLmT FdS" +# CONFIG_EN_SPEECH_COMMAND_ID6="RfKcGNiZ FdS" \ No newline at end of file