✨ esp32-s3-eye

examples/esp32-s3-eye/CMakeLists.txt

@@ -0,0 +1,8 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(EXTRA_COMPONENT_DIRS ../../components)
+add_compile_options(-fdiagnostics-color=always)
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(esp32-s3-eye)

examples/esp32-s3-eye/README.md

@@ -0,0 +1,65 @@
+# ESP-S3-EYE 示例说明
+
+## 交互系统
+
+### 语音
+
+语音交互逻辑遵循“先说唤醒词，再说命令词”。
+
+
+
+### 按键
+
+#### 
+
+## 工作模式
+
+该示例共包含四个工作模式：“待机模式”、“实时显示”、“人脸识别”和“移动侦测”。下面对这四个模式做简要介绍。
+
+### 待机模式
+
+待机模式下，LCD 屏幕上仅显示 Espressif Logo。该模式也是开发板上电以后的默认模式。
+
+### 实时显示
+
+实时显示模式下，LCD 屏幕上会实时显示摄像头采集的图片。
+
+### 人脸识别
+
+人脸识别模式下，LCD 屏幕上会实时显示摄像头采集的图片。并且实时检测画面中的人脸，并显示检测框和关键点。通过按键和语音交互可以实现人脸识别相关的操作，详见下表：
+
+|   操作   |                     说明                     | 按键触发 |                语音触发                |
+| :------: | :------------------------------------------: | :------: | :------------------------------------: |
+| 识别人脸 |     识别当前画面中的人脸，并显示识别结果     |  “PLAY”  | “Hi，乐鑫”唤醒后，说出命令词“识别一下” |
+| 添加人脸 |   添加当前画面中的人脸，并显示添加的 ID 号   |   “UP”   | “Hi，乐鑫”唤醒后，说出命令词“添加人脸” |
+| 删除人脸 | 删除人脸库中的最后一个 ID， 并显示剩余 ID 数 |  “DOWN”  | “Hi，乐鑫”唤醒后，说出命令词“删除人脸” |
+
+### 移动侦测
+
+移动侦测模式下，LCD 屏幕上会实时显示摄像头采集的图片。并且实时检测画面中是否出现物体移动，如果物体移动，画面左上角会显示蓝点。
+
+
+
+#### 进入默认固件
+上电后等待3秒，不对开发板做任何操作，当LCD屏幕显示Espressif LOGO表示已进入默认固件。
+
+#### 默认固件模式
+默认固件存在四种运行模式，固件会默认先进入人脸识别模式：
+1. 待机模式：该模式下开发板只有LCD工作，显示公司LOGO。
+2. 实时显示模式：该模式下LCD会实时显示摄像头拍摄到的画面。
+3. 人脸识别模式：该模式下LCD会实时显示摄像头拍摄到的画面，如果画面中存在人脸，LCD显示的画面上会框出人脸，并画出人脸的五个特征点。该模式下提供三种功能：录入人脸、识别人脸、删除人脸。
+   - 录入人脸： 按下MENU键。或者使用“Hi，乐鑫”唤醒词唤醒，唤醒后摄像头旁的LED会被点亮，随后说出“添加人脸”。
+   - 识别人脸： 按下UP键。或者使用“Hi，乐鑫”唤醒词唤醒，唤醒后摄像头旁的LED会被点亮，随后说出“识别一下”。
+   - 删除人脸： 按下PLAY键。或者使用“Hi，乐鑫”唤醒词唤醒，唤醒后摄像头旁的LED会被点亮，随后说出“删除人脸”。
+4. 移动侦测模式： 该模式下LCD会实时显示摄像头拍摄到的画面。若画面中存在物体移动，LCD中左上角会显示蓝色色块。
+
+#### 模式切换
+在任何时候都可以使用“Hi，乐鑫”唤醒词唤醒开发板，四种模式分别对应四个命令词。说出相应的命令词，识别成功后随即切换到对应的模式。工作模式与命令词的对应关系如下：
+1. 待机模式：“停止工作”
+2. 实时显示模式：“仅显示”
+3. 人脸识别模式：“人脸识别”
+4. 移动侦测模式：“移动侦测”
+
+#### 语音系统
+- 语音唤醒：默认固件工作时，任何时刻都可以使用“Hi，乐鑫”唤醒词唤醒开发板。开发板被唤醒后，摄像头旁的LED会常亮。
+- 命令词识别：唤醒后会等待用户说出命令词。若识别成功，摄像头旁的LED会从常亮变为闪烁一秒, LED会随后熄灭。

examples/esp32-s3-eye/main/CMakeLists.txt

@@ -0,0 +1,11 @@
+set(src_dirs        .
+                    src)
+
+set(include_dirs    include
+                    .)
+
+set(requires    console
+                esp_adc_cal
+                        )
+
+idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs})

examples/esp32-s3-eye/main/Kconfig.projbuild

@@ -0,0 +1,13 @@
+menu "ESP32-S3-EYE Firmware"
+    choice ESP_SR_LANGUAGE
+        bool "esp-sr language"
+        default CN_MODEL
+        help
+             Select ESP-SR Language.
+        
+        config CN_MODEL
+            bool "Chinese"
+        config EN_MODEL
+            bool "English"
+    endchoice
+endmenu

examples/esp32-s3-eye/main/app_main.cpp

@@ -0,0 +1,50 @@
+#include "sdkconfig.h"
+#include "driver/gpio.h"
+
+#if CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID
+#include "who_trace.h"
+#endif
+
+#include "app_buttom.hpp"
+#include "app_camera.hpp"
+#include "app_lcd.hpp"
+#include "app_led.hpp"
+#include "app_motion.hpp"
+#include "app_speech.hpp"
+#include "app_face.hpp"
+
+extern "C" void app_main()
+{
+#if CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID
+    register_trace();
+#endif
+
+    QueueHandle_t xQueueFrame_0 = xQueueCreate(2, sizeof(camera_fb_t *));
+    QueueHandle_t xQueueFrame_1 = xQueueCreate(2, sizeof(camera_fb_t *));
+    QueueHandle_t xQueueFrame_2 = xQueueCreate(2, sizeof(camera_fb_t *));
+
+    AppButtom *key = new AppButtom();
+    AppSpeech *speech = new AppSpeech();
+    AppCamera *camera = new AppCamera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueFrame_0);
+    AppFace *face = new AppFace(key, speech, xQueueFrame_0, xQueueFrame_1);
+    AppMotion *motion = new AppMotion(key, speech, xQueueFrame_1, xQueueFrame_2);
+    AppLCD *lcd = new AppLCD(key, speech, xQueueFrame_2);
+    LED *led = new LED(GPIO_NUM_3, key, speech);
+
+    key->attach(face);
+    key->attach(motion);
+    key->attach(led);
+    key->attach(lcd);
+
+    speech->attach(face);
+    speech->attach(motion);
+    speech->attach(led);
+    speech->attach(lcd);
+
+    lcd->run();
+    motion->run();
+    face->run();
+    camera->run();
+    speech->run();
+    key->run();
+}

examples/esp32-s3-eye/main/include/__base__.hpp

@@ -0,0 +1,79 @@
+#pragma once
+
+#include <list>
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+
+#include "esp_camera.h"
+
+typedef enum
+{
+    COMMAND_TIMEOUT = -2,
+    COMMAND_NOT_DETECTED = -1,
+
+    MENU_STOP_WORKING = 0,
+    MENU_DISPLAY_ONLY = 1,
+    MENU_FACE_RECOGNITION = 2,
+    MENU_MOTION_DETECTION = 3,
+
+    ACTION_ENROLL = 4,
+    ACTION_DELETE = 5,
+    ACTION_RECOGNIZE = 6
+} command_word_t;
+
+class Observer
+{
+public:
+    virtual void update() = 0;
+};
+
+class Subject
+{
+private:
+    std::list<Observer *> observers;
+
+public:
+    void attach(Observer *observer)
+    {
+        this->observers.push_back(observer);
+    }
+
+    void detach(Observer *observer)
+    {
+        this->observers.remove(observer);
+    }
+
+    void detach_all()
+    {
+        this->observers.clear();
+    }
+
+    void notify()
+    {
+        for (auto observer : this->observers)
+            observer->update();
+    }
+};
+
+class Frame
+{
+public:
+    QueueHandle_t queue_i;
+    QueueHandle_t queue_o;
+    void (*callback)(camera_fb_t *);
+
+    Frame(QueueHandle_t queue_i = nullptr,
+          QueueHandle_t queue_o = nullptr,
+          void (*callback)(camera_fb_t *) = nullptr) : queue_i(queue_i),
+                                                       queue_o(queue_o),
+                                                       callback(callback) {}
+
+    void set_io(QueueHandle_t queue_i, QueueHandle_t queue_o)
+    {
+        this->queue_i = queue_i;
+        this->queue_o = queue_o;
+    }
+};

examples/esp32-s3-eye/main/include/app_buttom.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vector>
+
+#include "__base__.hpp"
+
+typedef enum
+{
+    _IDLE = 0,
+    _MENU,
+    _PLAY,
+    _UP,
+    _DOWN
+} _key_name_t;
+
+typedef struct
+{
+    _key_name_t key; /**< button index on the channel */
+    int min;         /**< min voltage in mv corresponding to the button */
+    int max;         /**< max voltage in mv corresponding to the button */
+} key_config_t;
+
+class AppButtom : public Subject
+{
+public:
+    std::vector<key_config_t> key_configs;
+    _key_name_t pressed;
+
+    uint8_t menu;
+
+    AppButtom();
+    ~AppButtom();
+
+    void run();
+};

examples/esp32-s3-eye/main/include/app_camera.hpp

@@ -0,0 +1,210 @@
+#pragma once
+
+#include <list>
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+
+#include "esp_camera.h"
+
+#include "__base__.hpp"
+
+#if CONFIG_CAMERA_MODULE_WROVER_KIT
+#define CAMERA_MODULE_NAME "Wrover Kit"
+#define CAMERA_PIN_PWDN -1
+#define CAMERA_PIN_RESET -1
+#define CAMERA_PIN_XCLK 21
+#define CAMERA_PIN_SIOD 26
+#define CAMERA_PIN_SIOC 27
+
+#define CAMERA_PIN_D7 35
+#define CAMERA_PIN_D6 34
+#define CAMERA_PIN_D5 39
+#define CAMERA_PIN_D4 36
+#define CAMERA_PIN_D3 19
+#define CAMERA_PIN_D2 18
+#define CAMERA_PIN_D1 5
+#define CAMERA_PIN_D0 4
+#define CAMERA_PIN_VSYNC 25
+#define CAMERA_PIN_HREF 23
+#define CAMERA_PIN_PCLK 22
+
+#elif CONFIG_CAMERA_MODULE_ESP_EYE
+#define CAMERA_MODULE_NAME "ESP-EYE"
+#define CAMERA_PIN_PWDN -1
+#define CAMERA_PIN_RESET -1
+#define CAMERA_PIN_XCLK 4
+#define CAMERA_PIN_SIOD 18
+#define CAMERA_PIN_SIOC 23
+
+#define CAMERA_PIN_D7 36
+#define CAMERA_PIN_D6 37
+#define CAMERA_PIN_D5 38
+#define CAMERA_PIN_D4 39
+#define CAMERA_PIN_D3 35
+#define CAMERA_PIN_D2 14
+#define CAMERA_PIN_D1 13
+#define CAMERA_PIN_D0 34
+#define CAMERA_PIN_VSYNC 5
+#define CAMERA_PIN_HREF 27
+#define CAMERA_PIN_PCLK 25
+
+#elif CONFIG_CAMERA_MODULE_ESP_S2_KALUGA
+#define CAMERA_MODULE_NAME "ESP-S2-KALUGA"
+#define CAMERA_PIN_PWDN -1
+#define CAMERA_PIN_RESET -1
+#define CAMERA_PIN_XCLK 1
+#define CAMERA_PIN_SIOD 8
+#define CAMERA_PIN_SIOC 7
+
+#define CAMERA_PIN_D7 38
+#define CAMERA_PIN_D6 21
+#define CAMERA_PIN_D5 40
+#define CAMERA_PIN_D4 39
+#define CAMERA_PIN_D3 42
+#define CAMERA_PIN_D2 41
+#define CAMERA_PIN_D1 37
+#define CAMERA_PIN_D0 36
+#define CAMERA_PIN_VSYNC 2
+#define CAMERA_PIN_HREF 3
+#define CAMERA_PIN_PCLK 33
+
+#elif CONFIG_CAMERA_MODULE_ESP_S3_EYE
+#define CAMERA_MODULE_NAME "ESP-S3-EYE"
+#define CAMERA_PIN_PWDN -1
+#define CAMERA_PIN_RESET -1
+
+#define CAMERA_PIN_VSYNC 6
+#define CAMERA_PIN_HREF 7
+#define CAMERA_PIN_PCLK 13
+#define CAMERA_PIN_XCLK 15
+
+#define CAMERA_PIN_SIOD 4
+#define CAMERA_PIN_SIOC 5
+
+#define CAMERA_PIN_D0 11
+#define CAMERA_PIN_D1 9
+#define CAMERA_PIN_D2 8
+#define CAMERA_PIN_D3 10
+#define CAMERA_PIN_D4 12
+#define CAMERA_PIN_D5 18
+#define CAMERA_PIN_D6 17
+#define CAMERA_PIN_D7 16
+
+#elif CONFIG_CAMERA_MODEL_ESP32_CAM_BOARD
+#define CAMERA_MODULE_NAME "ESP-DEVCAM"
+#define CAMERA_PIN_PWDN 32
+#define CAMERA_PIN_RESET 33
+
+#define CAMERA_PIN_XCLK 4
+#define CAMERA_PIN_SIOD 18
+#define CAMERA_PIN_SIOC 23
+
+#define CAMERA_PIN_D7 36
+#define CAMERA_PIN_D6 19
+#define CAMERA_PIN_D5 21
+#define CAMERA_PIN_D4 39
+#define CAMERA_PIN_D3 35
+#define CAMERA_PIN_D2 14
+#define CAMERA_PIN_D1 13
+#define CAMERA_PIN_D0 34
+#define CAMERA_PIN_VSYNC 5
+#define CAMERA_PIN_HREF 27
+#define CAMERA_PIN_PCLK 25
+
+#elif CONFIG_CAMERA_MODULE_M5STACK_PSRAM
+#define CAMERA_MODULE_NAME "M5STACK-PSRAM"
+#define CAMERA_PIN_PWDN -1
+#define CAMERA_PIN_RESET 15
+
+#define CAMERA_PIN_XCLK 27
+#define CAMERA_PIN_SIOD 25
+#define CAMERA_PIN_SIOC 23
+
+#define CAMERA_PIN_D7 19
+#define CAMERA_PIN_D6 36
+#define CAMERA_PIN_D5 18
+#define CAMERA_PIN_D4 39
+#define CAMERA_PIN_D3 5
+#define CAMERA_PIN_D2 34
+#define CAMERA_PIN_D1 35
+#define CAMERA_PIN_D0 32
+#define CAMERA_PIN_VSYNC 22
+#define CAMERA_PIN_HREF 26
+#define CAMERA_PIN_PCLK 21
+
+#elif CONFIG_CAMERA_MODULE_M5STACK_WIDE
+#define CAMERA_MODULE_NAME "M5STACK-WIDE"
+#define CAMERA_PIN_PWDN -1
+#define CAMERA_PIN_RESET 15
+#define CAMERA_PIN_XCLK 27
+#define CAMERA_PIN_SIOD 22
+#define CAMERA_PIN_SIOC 23
+
+#define CAMERA_PIN_D7 19
+#define CAMERA_PIN_D6 36
+#define CAMERA_PIN_D5 18
+#define CAMERA_PIN_D4 39
+#define CAMERA_PIN_D3 5
+#define CAMERA_PIN_D2 34
+#define CAMERA_PIN_D1 35
+#define CAMERA_PIN_D0 32
+#define CAMERA_PIN_VSYNC 25
+#define CAMERA_PIN_HREF 26
+#define CAMERA_PIN_PCLK 21
+
+#elif CONFIG_CAMERA_MODULE_AI_THINKER
+#define CAMERA_MODULE_NAME "AI-THINKER"
+#define CAMERA_PIN_PWDN 32
+#define CAMERA_PIN_RESET -1
+#define CAMERA_PIN_XCLK 0
+#define CAMERA_PIN_SIOD 26
+#define CAMERA_PIN_SIOC 27
+
+#define CAMERA_PIN_D7 35
+#define CAMERA_PIN_D6 34
+#define CAMERA_PIN_D5 39
+#define CAMERA_PIN_D4 36
+#define CAMERA_PIN_D3 21
+#define CAMERA_PIN_D2 19
+#define CAMERA_PIN_D1 18
+#define CAMERA_PIN_D0 5
+#define CAMERA_PIN_VSYNC 25
+#define CAMERA_PIN_HREF 23
+#define CAMERA_PIN_PCLK 22
+
+#elif CONFIG_CAMERA_MODULE_CUSTOM
+#define CAMERA_MODULE_NAME "CUSTOM"
+#define CAMERA_PIN_PWDN CONFIG_CAMERA_PIN_PWDN
+#define CAMERA_PIN_RESET CONFIG_CAMERA_PIN_RESET
+#define CAMERA_PIN_XCLK CONFIG_CAMERA_PIN_XCLK
+#define CAMERA_PIN_SIOD CONFIG_CAMERA_PIN_SIOD
+#define CAMERA_PIN_SIOC CONFIG_CAMERA_PIN_SIOC
+
+#define CAMERA_PIN_D7 CONFIG_CAMERA_PIN_Y9
+#define CAMERA_PIN_D6 CONFIG_CAMERA_PIN_Y8
+#define CAMERA_PIN_D5 CONFIG_CAMERA_PIN_Y7
+#define CAMERA_PIN_D4 CONFIG_CAMERA_PIN_Y6
+#define CAMERA_PIN_D3 CONFIG_CAMERA_PIN_Y5
+#define CAMERA_PIN_D2 CONFIG_CAMERA_PIN_Y4
+#define CAMERA_PIN_D1 CONFIG_CAMERA_PIN_Y3
+#define CAMERA_PIN_D0 CONFIG_CAMERA_PIN_Y2
+#define CAMERA_PIN_VSYNC CONFIG_CAMERA_PIN_VSYNC
+#define CAMERA_PIN_HREF CONFIG_CAMERA_PIN_HREF
+#define CAMERA_PIN_PCLK CONFIG_CAMERA_PIN_PCLK
+#endif
+
+#define XCLK_FREQ_HZ 15000000
+
+class AppCamera : public Frame
+{
+public:
+    AppCamera(const pixformat_t pixel_fromat,
+              const framesize_t frame_size,
+              const uint8_t fb_count,
+              QueueHandle_t queue_o = nullptr);
+
+    void run();
+};

examples/esp32-s3-eye/main/include/app_face.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "human_face_detect_msr01.hpp"
+#include "human_face_detect_mnp01.hpp"
+#include "face_recognition_tool.hpp"
+// #if CONFIG_MFN_V1_Q8
+// #include "face_recognition_112_v1_s8.hpp"
+// #elif CONFIG_MFN_V1_Q16
+#include "face_recognition_112_v1_s16.hpp"
+// #endif
+
+#include "__base__.hpp"
+#include "app_camera.hpp"
+#include "app_buttom.hpp"
+#include "app_speech.hpp"
+
+typedef enum
+{
+    IDLE = 0,
+    ENROLL = 1,
+    RECOGNIZE = 2,
+    DELETE = 3,
+} recognizer_state_t;
+
+class AppFace : public Observer, public Frame
+{
+private:
+    AppButtom *key;
+    AppSpeech *speech;
+
+public:
+    HumanFaceDetectMSR01 detector;
+    HumanFaceDetectMNP01 detector2;
+
+    // #if CONFIG_MFN_V1_Q8
+    // FaceRecognition112V1S8 *recognizer;
+    // #elif CONFIG_MFN_V1_Q16
+    FaceRecognition112V1S16 *recognizer;
+    // #endif
+
+    face_info_t recognize_result;
+    recognizer_state_t state;
+    recognizer_state_t state_previous;
+
+    bool switch_on;
+
+    uint8_t frame_count;
+
+    AppFace(AppButtom *key,
+                      AppSpeech *speech,
+                      QueueHandle_t queue_i = nullptr,
+                      QueueHandle_t queue_o = nullptr,
+                      void (*callback)(camera_fb_t *) = esp_camera_fb_return);
+    ~AppFace();
+
+    void update();
+    void run();
+};

examples/esp32-s3-eye/main/include/app_lcd.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <stdint.h>
+#include "esp_log.h"
+#include "screen_driver.h"
+
+#include "__base__.hpp"
+#include "app_camera.hpp"
+#include "app_buttom.hpp"
+#include "app_speech.hpp"
+
+#define BOARD_LCD_MOSI 47
+#define BOARD_LCD_MISO -1
+#define BOARD_LCD_SCK 21
+#define BOARD_LCD_CS 44
+#define BOARD_LCD_DC 43
+#define BOARD_LCD_RST -1
+#define BOARD_LCD_BL 48
+#define BOARD_LCD_PIXEL_CLOCK_HZ (40 * 1000 * 1000)
+#define BOARD_LCD_BK_LIGHT_ON_LEVEL 0
+#define BOARD_LCD_BK_LIGHT_OFF_LEVEL !BOARD_LCD_BK_LIGHT_ON_LEVEL
+#define BOARD_LCD_H_RES 240
+#define BOARD_LCD_V_RES 240
+#define BOARD_LCD_CMD_BITS 8
+#define BOARD_LCD_PARAM_BITS 8
+#define LCD_HOST SPI2_HOST
+
+class AppLCD : public Observer, public Frame
+{
+private:
+    AppButtom *key;
+    AppSpeech *speech;
+
+public:
+    scr_driver_t driver;
+    bool switch_on;
+    bool paper_drawn;
+
+    AppLCD(AppButtom *key,
+           AppSpeech *speech,
+           QueueHandle_t xQueueFrameI = nullptr,
+           QueueHandle_t xQueueFrameO = nullptr,
+           void (*callback)(camera_fb_t *) = esp_camera_fb_return);
+
+    void draw_wallpaper();
+    void draw_color(int color);
+
+    void update();
+
+    void run();
+};

examples/esp32-s3-eye/main/include/app_led.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+#include "driver/gpio.h"
+
+#include "app_buttom.hpp"
+#include "app_speech.hpp"
+
+class LED : public Observer
+{
+private:
+    const gpio_num_t pin;
+    AppButtom *key;
+    AppSpeech *sr;
+
+public:
+    LED(const gpio_num_t pin, AppButtom *key, AppSpeech *sr);
+    ~LED();
+
+    void update();
+};

examples/esp32-s3-eye/main/include/app_motion.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "__base__.hpp"
+#include "app_camera.hpp"
+#include "app_buttom.hpp"
+#include "app_speech.hpp"
+
+class AppMotion : public Observer, public Frame
+{
+private:
+    AppButtom *key;
+    AppSpeech *speech;
+
+public:
+    bool switch_on;
+
+    AppMotion(AppButtom *key,
+                    AppSpeech *speech,
+                    QueueHandle_t queue_i = nullptr,
+                    QueueHandle_t queue_o = nullptr,
+                    void (*callback)(camera_fb_t *) = esp_camera_fb_return);
+
+    void update();
+
+    void run();
+};

examples/esp32-s3-eye/main/include/app_speech.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+
+#include "esp_afe_sr_iface.h"
+
+#include "__base__.hpp"
+
+class AppSpeech : public Subject
+{
+public:
+    const esp_afe_sr_iface_t *afe_handle;
+    esp_afe_sr_data_t *afe_data;
+    bool detected;
+    command_word_t command;
+
+    AppSpeech();
+
+    void run();
+};

examples/esp32-s3-eye/main/src/app_buttom.cpp

@@ -0,0 +1,79 @@
+#include "app_buttom.hpp"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "driver/adc_common.h"
+#include "esp_log.h"
+#include "esp_adc_cal.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+
+// ADC Channels
+#define ADC1_EXAMPLE_CHAN0 ADC1_CHANNEL_0
+// ADC Attenuation
+#define ADC_EXAMPLE_ATTEN ADC_ATTEN_DB_11
+// ADC Calibration
+#if CONFIG_IDF_TARGET_ESP32
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_VREF
+#elif CONFIG_IDF_TARGET_ESP32S2
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP
+#elif CONFIG_IDF_TARGET_ESP32C3
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP
+#elif CONFIG_IDF_TARGET_ESP32S3
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP_FIT
+#endif
+
+#define PRESS_INTERVAL 500000
+
+static const char *TAG = "App/Buttom";
+
+AppButtom::AppButtom() : key_configs({{_MENU, 2800, 3000}, {_PLAY, 2250, 2450}, {_UP, 300, 500}, {_DOWN, 850, 1050}}),
+                         pressed(_IDLE)
+{
+    ESP_ERROR_CHECK(adc1_config_width((adc_bits_width_t)ADC_WIDTH_BIT_DEFAULT));
+    ESP_ERROR_CHECK(adc1_config_channel_atten(ADC1_EXAMPLE_CHAN0, ADC_EXAMPLE_ATTEN));
+}
+
+static void task(AppButtom *self)
+{
+    int64_t backup_time = esp_timer_get_time();
+    int64_t last_time = esp_timer_get_time();
+
+    uint8_t menu_count = 0;
+
+    while (true)
+    {
+        uint32_t voltage = adc1_get_raw(ADC1_EXAMPLE_CHAN0);
+        backup_time = esp_timer_get_time();
+        for (auto key_config : self->key_configs)
+        {
+            if ((voltage >= key_config.min) && (voltage <= key_config.max))
+            {
+                if (((backup_time - last_time) > PRESS_INTERVAL))
+                {
+                    ESP_LOGD(TAG, "Key[%d] is pressed", self->pressed);
+                    self->pressed = key_config.key;
+
+                    if (self->pressed == _MENU)
+                    {
+                        self->menu++;
+                        self->menu %= (MENU_MOTION_DETECTION + 1);
+                    }
+
+                    last_time = backup_time;
+                    self->notify();
+
+                    self->pressed = _IDLE;
+                    break;
+                }
+            }
+        }
+        vTaskDelay(pdMS_TO_TICKS(10));
+    }
+}
+
+void AppButtom::run()
+{
+    xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 3 * 1024, this, 5, NULL, 0);
+}

examples/esp32-s3-eye/main/src/app_camera.cpp

@@ -0,0 +1,98 @@
+#include "app_camera.hpp"
+
+#include "assert.h"
+
+#include "esp_log.h"
+#include "esp_system.h"
+
+const static char TAG[] = "App/Camera";
+
+AppCamera::AppCamera(const pixformat_t pixel_fromat,
+                     const framesize_t frame_size,
+                     const uint8_t fb_count,
+                     QueueHandle_t queue_o) : Frame(nullptr, queue_o, nullptr)
+{
+    ESP_LOGI(TAG, "Camera module is %s", CAMERA_MODULE_NAME);
+
+#if CONFIG_CAMERA_MODEL_ESP_EYE || CONFIG_CAMERA_MODEL_ESP32_CAM_BOARD
+    /* IO13, IO14 is designed for JTAG by default,
+     * to use it as generalized input,
+     * firstly declair it as pullup input */
+    gpio_config_t conf;
+    conf.mode = GPIO_MODE_INPUT;
+    conf.pull_up_en = GPIO_PULLUP_ENABLE;
+    conf.pull_down_en = GPIO_PULLDOWN_DISABLE;
+    conf.intr_type = GPIO_INTR_DISABLE;
+    conf.pin_bit_mask = 1LL << 13;
+    gpio_config(&conf);
+    conf.pin_bit_mask = 1LL << 14;
+    gpio_config(&conf);
+#endif
+
+    camera_config_t config;
+    config.ledc_channel = LEDC_CHANNEL_0;
+    config.ledc_timer = LEDC_TIMER_0;
+    config.pin_d0 = CAMERA_PIN_D0;
+    config.pin_d1 = CAMERA_PIN_D1;
+    config.pin_d2 = CAMERA_PIN_D2;
+    config.pin_d3 = CAMERA_PIN_D3;
+    config.pin_d4 = CAMERA_PIN_D4;
+    config.pin_d5 = CAMERA_PIN_D5;
+    config.pin_d6 = CAMERA_PIN_D6;
+    config.pin_d7 = CAMERA_PIN_D7;
+    config.pin_xclk = CAMERA_PIN_XCLK;
+    config.pin_pclk = CAMERA_PIN_PCLK;
+    config.pin_vsync = CAMERA_PIN_VSYNC;
+    config.pin_href = CAMERA_PIN_HREF;
+    config.pin_sscb_sda = CAMERA_PIN_SIOD;
+    config.pin_sscb_scl = CAMERA_PIN_SIOC;
+    config.pin_pwdn = CAMERA_PIN_PWDN;
+    config.pin_reset = CAMERA_PIN_RESET;
+    config.xclk_freq_hz = XCLK_FREQ_HZ;
+    config.pixel_format = pixel_fromat;
+    config.frame_size = frame_size;
+    config.jpeg_quality = 12;
+    config.fb_count = fb_count;
+    config.fb_location = CAMERA_FB_IN_PSRAM;
+    config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
+
+    // camera init
+    esp_err_t err = esp_camera_init(&config);
+    if (err != ESP_OK)
+    {
+        ESP_LOGE(TAG, "Camera init failed with error 0x%x", err);
+        return;
+    }
+
+    sensor_t *s = esp_camera_sensor_get();
+    s->set_vflip(s, 1); // flip it back
+    // initial sensors are flipped vertically and colors are a bit saturated
+    if (s->id.PID == OV3660_PID)
+    {
+        s->set_brightness(s, 1);  // up the blightness just a bit
+        s->set_saturation(s, -2); // lower the saturation
+    }
+    s->set_sharpness(s, 2);
+    s->set_awb_gain(s, 2);
+}
+
+static void task(AppCamera *self)
+{
+    ESP_LOGD(TAG, "Start");
+    while (true)
+    {
+        if (self->queue_o == nullptr)
+            break;
+
+        camera_fb_t *frame = esp_camera_fb_get();
+        if (frame)
+            xQueueSend(self->queue_o, &frame, portMAX_DELAY);
+    }
+    ESP_LOGD(TAG, "Stop");
+    vTaskDelete(NULL);
+}
+
+void AppCamera::run()
+{
+    xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 2 * 1024, this, 5, NULL, 1);
+}

examples/esp32-s3-eye/main/src/app_face.cpp

@@ -0,0 +1,231 @@
+#include "app_face.hpp"
+
+#include <list>
+
+#include "esp_log.h"
+#include "esp_camera.h"
+
+#include "dl_image.hpp"
+#include "fb_gfx.h"
+
+#include "who_ai_utils.hpp"
+
+static const char TAG[] = "App/Face";
+
+#define RGB565_MASK_RED 0xF800
+#define RGB565_MASK_GREEN 0x07E0
+#define RGB565_MASK_BLUE 0x001F
+
+#define FRAME_DELAY_NUM 16
+
+static void rgb_print(camera_fb_t *fb, uint32_t color, const char *str)
+{
+    fb_gfx_print(fb, (fb->width - (strlen(str) * 14)) / 2, 10, color, str);
+}
+
+static int rgb_printf(camera_fb_t *fb, uint32_t color, const char *format, ...)
+{
+    char loc_buf[64];
+    char *temp = loc_buf;
+    int len;
+    va_list arg;
+    va_list copy;
+    va_start(arg, format);
+    va_copy(copy, arg);
+    len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg);
+    va_end(copy);
+    if (len >= sizeof(loc_buf))
+    {
+        temp = (char *)malloc(len + 1);
+        if (temp == NULL)
+        {
+            return 0;
+        }
+    }
+    vsnprintf(temp, len + 1, format, arg);
+    va_end(arg);
+    rgb_print(fb, color, temp);
+    if (len > 64)
+    {
+        free(temp);
+    }
+    return len;
+}
+
+AppFace::AppFace(AppButtom *key,
+                                     AppSpeech *speech,
+                                     QueueHandle_t queue_i,
+                                     QueueHandle_t queue_o,
+                                     void (*callback)(camera_fb_t *)) : Frame(queue_i, queue_o, callback),
+                                                                        key(key),
+                                                                        speech(speech),
+                                                                        detector(0.3F, 0.3F, 10, 0.3F),
+                                                                        detector2(0.4F, 0.3F, 10),
+                                                                        state(IDLE),
+                                                                        switch_on(false)
+//   recognizer(
+// #if CONFIG_MFN_V1_Q8
+//   new FaceRecognition112V1S8()
+// #elif CONFIG_MFN_V1_Q16
+//   new FaceRecognition112V1S16()
+// #endif
+//   )
+{
+    // this->recognizer = new FaceRecognition112V1S8();
+    this->recognizer = new FaceRecognition112V1S16();
+
+    this->recognizer->set_partition(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, "fr");
+    int partition_result = this->recognizer->set_ids_from_flash();
+}
+
+AppFace::~AppFace()
+{
+}
+
+void AppFace::update()
+{
+    // Parse key
+    if (this->key->pressed > _IDLE)
+    {
+        if (this->key->pressed == _MENU)
+        {
+            this->switch_on = (this->key->menu == MENU_FACE_RECOGNITION) ? true : false;
+            ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF");
+        }
+        else if (this->key->pressed == _PLAY)
+        {
+            this->state = RECOGNIZE;
+        }
+        else if (this->key->pressed == _UP)
+        {
+            this->state = ENROLL;
+        }
+        else if (this->key->pressed == _DOWN)
+        {
+            this->state = DELETE;
+        }
+    }
+
+    // Parse speech recognition
+    if (this->speech->command > COMMAND_NOT_DETECTED)
+    {
+        if (this->speech->command >= MENU_STOP_WORKING && this->speech->command <= MENU_MOTION_DETECTION)
+        {
+            this->switch_on = (this->speech->command == MENU_FACE_RECOGNITION) ? true : false;
+            ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF");
+        }
+        else if (this->speech->command == ACTION_ENROLL)
+        {
+            this->state = ENROLL;
+        }
+        else if (this->speech->command == ACTION_RECOGNIZE)
+        {
+            this->state = RECOGNIZE;
+        }
+        else if (this->speech->command == ACTION_DELETE)
+        {
+            this->state = DELETE;
+        }
+    }
+    ESP_LOGD(TAG, "Human face recognition state = %d", this->state);
+}
+
+static void task(AppFace *self)
+{
+    ESP_LOGD(TAG, "Start");
+    camera_fb_t *frame = nullptr;
+
+    while (true)
+    {
+        if (self->queue_i == nullptr)
+            break;
+
+        if (xQueueReceive(self->queue_i, &frame, portMAX_DELAY))
+        {
+            if (self->switch_on)
+            {
+                std::list<dl::detect::result_t> &detect_candidates = self->detector.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3});
+                std::list<dl::detect::result_t> &detect_results = self->detector2.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_candidates);
+
+                if (detect_results.size())
+                {
+                    // print_detection_result(detect_results);
+                    draw_detection_result((uint16_t *)frame->buf, frame->height, frame->width, detect_results);
+                }
+
+                if (self->state && detect_results.size() == 1)
+                {
+                    switch (self->state)
+                    {
+                    case ENROLL:
+                        self->recognizer->enroll_id((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint, "", true);
+                        ESP_LOGW(TAG, "Enroll ID %d", self->recognizer->get_enrolled_ids().back().id);
+                        break;
+
+                    case RECOGNIZE:
+                        self->recognize_result = self->recognizer->recognize((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint);
+                        // print_detection_result(detect_results);
+                        ESP_LOGD(TAG, "Similarity: %f", self->recognize_result.similarity);
+                        if (self->recognize_result.id > 0)
+                            ESP_LOGI(TAG, "Match ID: %d", self->recognize_result.id);
+                        else
+                            ESP_LOGE(TAG, "Match ID: %d", self->recognize_result.id);
+                        break;
+
+                    case DELETE:
+                        vTaskDelay(10);
+                        self->recognizer->delete_id(true);
+                        ESP_LOGE(TAG, "%d IDs left", self->recognizer->get_enrolled_id_num());
+                        break;
+
+                    default:
+                        break;
+                    }
+
+                    self->state_previous = self->state;
+                    self->state = IDLE;
+                    self->frame_count = FRAME_DELAY_NUM;
+                }
+
+                // Write result on several frames of image
+                if (self->frame_count)
+                {
+                    switch (self->state_previous)
+                    {
+                    case DELETE:
+                        rgb_printf(frame, RGB565_MASK_RED, "%d IDs left", self->recognizer->get_enrolled_id_num());
+                        break;
+
+                    case RECOGNIZE:
+                        if (self->recognize_result.id > 0)
+                            rgb_printf(frame, RGB565_MASK_GREEN, "ID %d", self->recognize_result.id);
+                        else
+                            rgb_print(frame, RGB565_MASK_RED, "who ?");
+                        break;
+
+                    case ENROLL:
+                        rgb_printf(frame, RGB565_MASK_BLUE, "Enroll: ID %d", self->recognizer->get_enrolled_ids().back().id);
+                        break;
+
+                    default:
+                        break;
+                    }
+
+                    self->frame_count--;
+                }
+            }
+
+            if (self->queue_o)
+                xQueueSend(self->queue_o, &frame, portMAX_DELAY);
+            else
+                self->callback(frame);
+        }
+    }
+    ESP_LOGD(TAG, "Stop");
+    vTaskDelete(NULL);
+}
+
+void AppFace::run()
+{
+    xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 5 * 1024, this, 5, NULL, 0);
+}

examples/esp32-s3-eye/main/src/app_lcd.cpp

@@ -0,0 +1,178 @@
+#include "app_lcd.hpp"
+
+#include <string.h>
+
+#include "esp_camera.h"
+#include "esp_lcd_panel_io.h"
+#include "esp_lcd_panel_vendor.h"
+#include "esp_lcd_panel_ops.h"
+#include "driver/spi_master.h"
+#include "driver/gpio.h"
+
+#include "logo_en_240x240_lcd.h"
+
+static const char TAG[] = "App/LCD";
+
+AppLCD::AppLCD(AppButtom *key,
+               AppSpeech *speech,
+               QueueHandle_t queue_i,
+               QueueHandle_t queue_o,
+               void (*callback)(camera_fb_t *)) : Frame(queue_i, queue_o, callback),
+                                                  key(key),
+                                                  speech(speech),
+                                                  switch_on(false)
+{
+    do
+    {
+        spi_config_t bus_conf = {
+            .miso_io_num = (gpio_num_t)BOARD_LCD_MISO,
+            .mosi_io_num = (gpio_num_t)BOARD_LCD_MOSI,
+            .sclk_io_num = (gpio_num_t)BOARD_LCD_SCK,
+            .max_transfer_sz = 2 * 240 * 240 + 10,
+        };
+        spi_bus_handle_t spi_bus = spi_bus_create(SPI2_HOST, &bus_conf);
+
+        scr_interface_spi_config_t spi_lcd_cfg = {
+            .spi_bus = spi_bus,
+            .pin_num_cs = BOARD_LCD_CS,
+            .pin_num_dc = BOARD_LCD_DC,
+            .clk_freq = 40 * 1000000,
+            .swap_data = 0,
+        };
+
+        scr_interface_driver_t *iface_drv;
+        scr_interface_create(SCREEN_IFACE_SPI, &spi_lcd_cfg, &iface_drv);
+        if (ESP_OK != scr_find_driver(SCREEN_CONTROLLER_ST7789, &this->driver))
+        {
+            ESP_LOGE(TAG, "screen find failed");
+            break;
+        }
+
+        scr_controller_config_t lcd_cfg = {
+            .interface_drv = iface_drv,
+            .pin_num_rst = BOARD_LCD_RST,
+            .pin_num_bckl = BOARD_LCD_BL,
+            .rst_active_level = 0,
+            .bckl_active_level = 0,
+            .width = 240,
+            .height = 240,
+            .offset_hor = 0,
+            .offset_ver = 0,
+            .rotate = (scr_dir_t)0,
+        };
+
+        if (ESP_OK != this->driver.init(&lcd_cfg))
+        {
+            ESP_LOGE(TAG, "screen initialize failed");
+            break;
+        }
+
+        scr_info_t lcd_info;
+        this->driver.get_info(&lcd_info);
+        ESP_LOGI(TAG, "Screen name:%s | width:%d | height:%d", lcd_info.name, lcd_info.width, lcd_info.height);
+
+        this->draw_color(0x000000);
+        vTaskDelay(pdMS_TO_TICKS(500));
+        this->draw_wallpaper();
+        vTaskDelay(pdMS_TO_TICKS(1000));
+    } while (0);
+}
+
+void AppLCD::draw_wallpaper()
+{
+    uint16_t *pixels = (uint16_t *)heap_caps_malloc((logo_en_240x240_lcd_width * logo_en_240x240_lcd_height) * sizeof(uint16_t), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
+    if (NULL == pixels)
+    {
+        ESP_LOGE(TAG, "Memory for bitmap is not enough");
+        return;
+    }
+    memcpy(pixels, logo_en_240x240_lcd, (logo_en_240x240_lcd_width * logo_en_240x240_lcd_height) * sizeof(uint16_t));
+    this->driver.draw_bitmap(0, 0, logo_en_240x240_lcd_width, logo_en_240x240_lcd_height, (uint16_t *)pixels);
+    heap_caps_free(pixels);
+
+    this->paper_drawn = true;
+}
+
+void AppLCD::draw_color(int color)
+{
+    scr_info_t lcd_info;
+    this->driver.get_info(&lcd_info);
+    uint16_t *buffer = (uint16_t *)malloc(lcd_info.width * sizeof(uint16_t));
+    if (buffer)
+    {
+        for (size_t i = 0; i < lcd_info.width; i++)
+        {
+            buffer[i] = color;
+        }
+
+        for (int y = 0; y < lcd_info.height; y++)
+        {
+            this->driver.draw_bitmap(0, y, lcd_info.width, 1, buffer);
+        }
+
+        free(buffer);
+    }
+    else
+    {
+        ESP_LOGE(TAG, "Memory for bitmap is not enough");
+    }
+}
+
+void AppLCD::update()
+{
+    if (this->key->pressed > _IDLE)
+    {
+        if (this->key->pressed == _MENU)
+        {
+            this->switch_on = (this->key->menu == MENU_STOP_WORKING) ? false : true;
+            ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF");
+        }
+    }
+
+    if (this->speech->command > COMMAND_NOT_DETECTED)
+    {
+        if (this->speech->command >= MENU_STOP_WORKING && this->speech->command <= MENU_MOTION_DETECTION)
+        {
+            this->switch_on = (this->speech->command == MENU_STOP_WORKING) ? false : true;
+            ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF");
+        }
+    }
+
+    if (this->switch_on == false)
+    {
+        this->paper_drawn = false;
+    }
+}
+
+static void task(AppLCD *self)
+{
+    ESP_LOGD(TAG, "Start");
+
+    camera_fb_t *frame = nullptr;
+    while (true)
+    {
+        if (self->queue_i == nullptr)
+            break;
+
+        if (xQueueReceive(self->queue_i, &frame, portMAX_DELAY))
+        {
+            if (self->switch_on)
+                self->driver.draw_bitmap(0, 0, frame->width, frame->height, (uint16_t *)frame->buf);
+            else if (self->paper_drawn == false)
+                self->draw_wallpaper();
+
+            if (self->queue_o)
+                xQueueSend(self->queue_o, &frame, portMAX_DELAY);
+            else
+                self->callback(frame);
+        }
+    }
+    ESP_LOGD(TAG, "Stop");
+    self->draw_wallpaper();
+    vTaskDelete(NULL);
+}
+
+void AppLCD::run()
+{
+    xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 2 * 1024, this, 5, NULL, 0);
+}

examples/esp32-s3-eye/main/src/app_led.cpp

@@ -0,0 +1,124 @@
+#include "app_led.hpp"
+
+#include "esp_log.h"
+
+const static char TAG[] = "App/LED";
+
+typedef enum
+{
+    LED_ALWAYS_OFF = 0,
+    LED_ALWAYS_ON = 1,
+    LED_OFF_1S = 2,
+    LED_OFF_2S = 3,
+    LED_OFF_4S = 4,
+    LED_ON_1S = 5,
+    LED_ON_2S = 6,
+    LED_ON_4S = 7,
+    LED_BLINK_1S = 8,
+    LED_BLINK_2S = 9,
+    LED_BLINK_4S = 10,
+} led_mode_t;
+
+LED::LED(const gpio_num_t pin, AppButtom *key, AppSpeech *sr) : pin(pin), key(key), sr(sr)
+{
+    // initialize GPIO
+    gpio_config_t gpio_conf;
+    gpio_conf.mode = GPIO_MODE_OUTPUT_OD;
+    gpio_conf.pull_up_en = GPIO_PULLUP_ENABLE;
+
+    gpio_conf.intr_type = GPIO_INTR_DISABLE;
+    gpio_conf.pin_bit_mask = 1LL << this->pin;
+    gpio_config(&gpio_conf);
+
+    gpio_set_level(this->pin, 0);
+}
+
+void LED::update()
+{
+    led_mode_t mode = LED_ALWAYS_OFF;
+
+    // parse key
+    if (this->key->pressed)
+    {
+        mode = LED_BLINK_2S;
+    }
+    // parse speech recognition
+    else if (this->sr->detected)
+    {
+        if (this->sr->command > COMMAND_NOT_DETECTED)
+            mode = LED_BLINK_1S;
+        else
+            mode = LED_ALWAYS_ON;
+    }
+
+    // do
+    switch (mode)
+    {
+    case LED_ALWAYS_OFF:
+        gpio_set_level(this->pin, 0);
+        break;
+    case LED_ALWAYS_ON:
+        gpio_set_level(this->pin, 1);
+        break;
+    case LED_OFF_1S:
+        gpio_set_level(this->pin, 0);
+        vTaskDelay(1000 / portTICK_PERIOD_MS);
+        gpio_set_level(this->pin, 1);
+        break;
+    case LED_OFF_2S:
+        gpio_set_level(this->pin, 0);
+        vTaskDelay(2000 / portTICK_PERIOD_MS);
+        gpio_set_level(this->pin, 1);
+        break;
+    case LED_OFF_4S:
+        gpio_set_level(this->pin, 0);
+        vTaskDelay(4000 / portTICK_PERIOD_MS);
+        gpio_set_level(this->pin, 1);
+        break;
+    case LED_ON_1S:
+        gpio_set_level(this->pin, 1);
+        vTaskDelay(1000 / portTICK_PERIOD_MS);
+        gpio_set_level(this->pin, 0);
+        break;
+    case LED_ON_2S:
+        gpio_set_level(this->pin, 1);
+        vTaskDelay(2000 / portTICK_PERIOD_MS);
+        gpio_set_level(this->pin, 0);
+        break;
+    case LED_ON_4S:
+        gpio_set_level(this->pin, 1);
+        vTaskDelay(4000 / portTICK_PERIOD_MS);
+        gpio_set_level(this->pin, 0);
+        break;
+    case LED_BLINK_1S:
+        for (int i = 0; i < 2; ++i)
+        {
+            gpio_set_level(this->pin, 1);
+            vTaskDelay(250 / portTICK_PERIOD_MS);
+            gpio_set_level(this->pin, 0);
+            vTaskDelay(250 / portTICK_PERIOD_MS);
+        }
+        break;
+    case LED_BLINK_2S:
+        for (int i = 0; i < 4; ++i)
+        {
+            gpio_set_level(this->pin, 1);
+            vTaskDelay(250 / portTICK_PERIOD_MS);
+            gpio_set_level(this->pin, 0);
+            vTaskDelay(250 / portTICK_PERIOD_MS);
+        }
+        break;
+    case LED_BLINK_4S:
+        for (int i = 0; i < 8; ++i)
+        {
+            gpio_set_level(this->pin, 1);
+            vTaskDelay(250 / portTICK_PERIOD_MS);
+            gpio_set_level(this->pin, 0);
+            vTaskDelay(250 / portTICK_PERIOD_MS);
+        }
+        break;
+
+    default:
+        break;
+    }
+}

examples/esp32-s3-eye/main/src/app_motion.cpp

@@ -0,0 +1,82 @@
+#include "app_motion.hpp"
+
+#include "assert.h"
+
+#include "esp_log.h"
+#include "esp_camera.h"
+
+#include "dl_image.hpp"
+
+static const char TAG[] = "App/Motion";
+
+AppMotion::AppMotion(AppButtom *key,
+                                 AppSpeech *speech,
+                                 QueueHandle_t queue_i,
+                                 QueueHandle_t queue_o,
+                                 void (*callback)(camera_fb_t *)) : Frame(queue_i, queue_o, callback),
+                                                                    key(key),
+                                                                    speech(speech),
+                                                                    switch_on(false) {}
+
+void AppMotion::update()
+{
+    if (this->key->pressed > _IDLE)
+    {
+        if (this->key->pressed == _MENU)
+        {
+            this->switch_on = (this->key->menu == MENU_MOTION_DETECTION) ? true : false;
+            ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF");
+        }
+    }
+
+    if (this->speech->command > COMMAND_NOT_DETECTED)
+    {
+        if (this->speech->command >= MENU_STOP_WORKING && this->speech->command <= MENU_MOTION_DETECTION)
+        {
+            this->switch_on = (this->speech->command == MENU_MOTION_DETECTION) ? true : false;
+            ESP_LOGD(TAG, "%s", this->switch_on ? "ON" : "OFF");
+        }
+    }
+}
+
+static void task(AppMotion *self)
+{
+    ESP_LOGD(TAG, "Start");
+    while (true)
+    {
+        if (self->queue_i == nullptr)
+            break;
+
+        camera_fb_t *frame1 = NULL;
+        camera_fb_t *frame2 = NULL;
+        if (xQueueReceive(self->queue_i, &frame1, portMAX_DELAY))
+        {
+            if (self->switch_on)
+            {
+                if (xQueueReceive(self->queue_i, &frame2, portMAX_DELAY))
+                {
+                    uint32_t moving_point_number = dl::image::get_moving_point_number((uint16_t *)frame1->buf, (uint16_t *)frame2->buf, frame1->height, frame1->width, 8, 15);
+                    if (moving_point_number > 50)
+                    {
+                        ESP_LOGI(TAG, "Something moved!");
+                        dl::image::draw_filled_rectangle((uint16_t *)frame1->buf, frame1->height, frame1->width, 0, 0, 20, 20);
+                    }
+
+                    self->callback(frame2);
+                }
+            }
+
+            if (self->queue_o)
+                xQueueSend(self->queue_o, &frame1, portMAX_DELAY);
+            else
+                self->callback(frame1);
+        }
+    }
+    ESP_LOGD(TAG, "Stop");
+    vTaskDelete(NULL);
+}
+
+void AppMotion::run()
+{
+    xTaskCreatePinnedToCore((TaskFunction_t)task, TAG, 3 * 1024, this, 6, NULL, 0);
+}

examples/esp32-s3-eye/main/src/app_speech.cpp

@@ -0,0 +1,214 @@
+#include "app_speech.hpp"
+
+#include "driver/i2s.h"
+#include "esp_log.h"
+
+#include "dl_lib_coefgetter_if.h"
+#include "esp_wn_iface.h"
+#include "esp_wn_models.h"
+#include "esp_afe_sr_models.h"
+#include "esp_mn_iface.h"
+#include "esp_mn_models.h"
+#include "esp_spiffs.h"
+
+#define I2S_CHANNEL_NUM 1
+#define I2S_CH ((i2s_port_t)1)
+
+static const char *TAG = "App/Speech";
+
+static void spiffs_init(void)
+{
+    ESP_LOGI(TAG, "Initializing SPIFFS");
+
+    esp_vfs_spiffs_conf_t conf = {
+        .base_path = "/srmodel",
+        .partition_label = "model",
+        .max_files = 5,
+        .format_if_mount_failed = true};
+    // Use settings defined above to initialize and mount SPIFFS filesystem.
+    // Note: esp_vfs_spiffs_register is an all-in-one convenience function.
+    esp_err_t ret = esp_vfs_spiffs_register(&conf);
+    if (ret != ESP_OK)
+    {
+        if (ret == ESP_FAIL)
+        {
+            ESP_LOGE(TAG, "Failed to mount or format filesystem");
+        }
+        else if (ret == ESP_ERR_NOT_FOUND)
+        {
+            ESP_LOGE(TAG, "Failed to find SPIFFS partition");
+        }
+        else
+        {
+            ESP_LOGE(TAG, "Failed to initialize SPIFFS (%s)", esp_err_to_name(ret));
+        }
+        return;
+    }
+    size_t total = 0, used = 0;
+    ret = esp_spiffs_info("model", &total, &used);
+    if (ret != ESP_OK)
+    {
+        ESP_LOGE(TAG, "Failed to get SPIFFS partition information (%s)", esp_err_to_name(ret));
+    }
+    else
+    {
+        ESP_LOGI(TAG, "Partition size: total: %d, used: %d", total, used);
+    }
+}
+
+static void i2s_init(void)
+{
+    i2s_config_t i2s_config = {
+        .mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX), // the mode must be set according to DSP configuration
+        .sample_rate = 16000,                                // must be the same as DSP configuration
+        .bits_per_sample = (i2s_bits_per_sample_t)32,        // must be the same as DSP configuration
+        .channel_format = I2S_CHANNEL_FMT_ONLY_LEFT,         // must be the same as DSP configuration
+        .communication_format = I2S_COMM_FORMAT_STAND_I2S,
+        .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2,
+        .dma_buf_count = 3,
+        .dma_buf_len = 300,
+    };
+    i2s_pin_config_t pin_config = {
+        .mck_io_num = I2S_PIN_NO_CHANGE,
+        .bck_io_num = 41,                  // IIS_SCLK
+        .ws_io_num = 42,                   // IIS_LCLK
+        .data_out_num = I2S_PIN_NO_CHANGE, // IIS_DSIN
+        .data_in_num = 2                   // IIS_DOUT
+    };
+    i2s_driver_install(I2S_CH, &i2s_config, 0, NULL);
+    i2s_set_pin(I2S_CH, &pin_config);
+    i2s_zero_dma_buffer(I2S_CH);
+}
+
+static void feed_handler(AppSpeech *self)
+{
+    esp_afe_sr_data_t *afe_data = self->afe_data;
+    int audio_chunksize = self->afe_handle->get_feed_chunksize(afe_data);
+    int nch = self->afe_handle->get_channel_num(afe_data);
+    size_t samp_len = audio_chunksize * I2S_CHANNEL_NUM;
+    size_t samp_len_bytes = samp_len * sizeof(int32_t);
+    int32_t *i2s_buff = (int32_t *)malloc(samp_len_bytes);
+    assert(i2s_buff);
+    size_t bytes_read;
+    // FILE *fp = fopen("/sdcard/out", "a+");
+    // if (fp == NULL) ESP_LOGE(TAG,"can not open file\n");
+
+    while (true)
+    {
+        i2s_read(I2S_CH, i2s_buff, samp_len_bytes, &bytes_read, portMAX_DELAY);
+
+        for (int i = 0; i < samp_len; ++i)
+        {
+            i2s_buff[i] = i2s_buff[i] >> 14; // 32:8为有效位， 8:0为低8位， 全为0， AFE的输入为16位语音数据，拿29：13位是为了对语音信号放大。
+        }
+        // FatfsComboWrite(i2s_buff, audio_chunksize * I2S_CHANNEL_NUM * sizeof(int16_t), 1, fp);
+
+        self->afe_handle->feed(afe_data, (int16_t *)i2s_buff);
+    }
+    self->afe_handle->destroy(afe_data);
+    vTaskDelete(NULL);
+}
+
+static void detect_hander(AppSpeech *self)
+{
+    esp_afe_sr_data_t *afe_data = self->afe_data;
+    int afe_chunksize = self->afe_handle->get_fetch_chunksize(afe_data);
+    int nch = self->afe_handle->get_channel_num(afe_data);
+    int16_t *buff = (int16_t *)malloc(afe_chunksize * sizeof(int16_t));
+    assert(buff);
+
+    static const esp_mn_iface_t *multinet = &MULTINET_MODEL;
+    model_iface_data_t *model_data = multinet->create((const model_coeff_getter_t *)&MULTINET_COEFF, 5760);
+    int mu_chunksize = multinet->get_samp_chunksize(model_data);
+    int chunk_num = multinet->get_samp_chunknum(model_data);
+    assert(mu_chunksize == afe_chunksize);
+
+    // FILE *fp = fopen("/sdcard/out", "w");
+    // if (fp == NULL) ESP_LOGE(TAG,"can not open file\n");
+
+    ESP_LOGI(TAG, "Ready");
+
+    self->detected = false;
+
+    while (true)
+    {
+        int res = self->afe_handle->fetch(afe_data, buff);
+
+        if (res == AFE_FETCH_WWE_DETECTED)
+        {
+            ESP_LOGI(TAG, ">>> Say your command <<<");
+            self->detected = true;
+            self->afe_handle->disable_wakenet(afe_data);
+            self->notify();
+        }
+
+        if (self->detected)
+        {
+            // Detect command
+            self->command = (command_word_t)multinet->detect(model_data, buff);
+            // FatfsComboWrite(buff, afe_chunksize * sizeof(int16_t), 1, fp);
+
+            if (self->command == COMMAND_NOT_DETECTED)
+                continue;
+            else if (self->command == COMMAND_TIMEOUT)
+            {
+                self->afe_handle->enable_wakenet(afe_data);
+                // self->afe_handle->enable_aec(afe_data);
+
+                self->detected = false;
+                ESP_LOGI(TAG, ">>> Waiting to be waken up <<<");
+                self->notify();
+            }
+            else
+            {
+                self->notify();
+                ESP_LOGD(TAG, "Command: %d", self->command);
+
+#ifndef CONFIG_SR_MN_CN_MULTINET3_CONTINUOUS_RECOGNITION
+                self->afe_handle->enable_wakenet(afe_data);
+                // self->afe_handle->enable_aec(afe_data);
+                self->detected = false;
+                ESP_LOGI(TAG, ">>> Waiting to be waken up <<<");
+#endif
+                self->command = COMMAND_TIMEOUT;
+                self->notify();
+            }
+        }
+    }
+    self->afe_handle->destroy(afe_data);
+    vTaskDelete(NULL);
+}
+
+AppSpeech::AppSpeech() : afe_handle(&esp_afe_sr_1mic), detected(false), command(COMMAND_TIMEOUT)
+{
+    spiffs_init();
+    i2s_init();
+    // sd_card_mount("/sdcard");
+    afe_config_t afe_config = {
+        .aec_init = true,
+        .se_init = true,
+        .vad_init = true,
+        .wakenet_init = true,
+        .vad_mode = 3,
+        .wakenet_model = &WAKENET_MODEL,
+        .wakenet_coeff = (const model_coeff_getter_t *)&WAKENET_COEFF,
+        .wakenet_mode = DET_MODE_2CH_90,
+        .afe_mode = SR_MODE_LOW_COST,
+        .afe_perferred_core = 0,
+        .afe_perferred_priority = 5,
+        .afe_ringbuf_size = 50,
+        .alloc_from_psram = AFE_PSRAM_MEDIA_COST,
+        .agc_mode = 2,
+    };
+    afe_config.aec_init = false;
+    afe_config.se_init = false;
+    afe_config.vad_init = false;
+    afe_config.afe_ringbuf_size = 10;
+    this->afe_data = this->afe_handle->create_from_config(&afe_config);
+}
+
+void AppSpeech::run()
+{
+    xTaskCreatePinnedToCore((TaskFunction_t)feed_handler, "SR_feed", 4 * 1024, this, 5, NULL, 0);
+    xTaskCreatePinnedToCore((TaskFunction_t)detect_hander, "SR_detect", 5 * 1024, this, 5, NULL, 0);
+}

examples/esp32-s3-eye/partitions.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
+
+factory,  app,  factory,  0x010000,  4000K,
+model,   data,  spiffs,   ,          3900K,
+nvs,     data,  nvs,      ,          16K,
+fr,      data,  ,         ,          128K,

examples/esp32-s3-eye/sdkconfig.defaults

@@ -0,0 +1,59 @@
+# General Configuration
+CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
+
+CONFIG_SPIRAM_SPEED_80M=y
+CONFIG_SPIRAM_USE_CAPS_ALLOC=y
+
+CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
+
+CONFIG_ESPTOOLPY_NO_STUB=y
+CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y
+CONFIG_SPIRAM_MODE_OCT=y
+
+# ESP32-S3 Configuration
+CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S3_SPIRAM_SUPPORT=y
+CONFIG_ESP32S3_DATA_CACHE_64KB=y
+CONFIG_ESP32S3_DATA_CACHE_8WAYS=y
+CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
+
+
+# Image Relation
+CONFIG_CAMERA_MODULE_ESP_S3_EYE=y
+CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y
+CONFIG_MFN_V1_Q8=y
+
+
+# Speech Relation
+
+# Chinese
+CONFIG_SR_WN_MODEL_WN7_QUANT8=y
+CONFIG_SR_WN_WN7_HILEXIN=y
+
+CONFIG_SR_MN_CHINESE=y
+CONFIG_SR_MN_CN_MULTINET4_5_SINGLE_RECOGNITION=y
+CONFIG_CN_SPEECH_COMMAND_ID0="ting zhi gong zuo"
+CONFIG_CN_SPEECH_COMMAND_ID1="jin xian shi"
+CONFIG_CN_SPEECH_COMMAND_ID2="ren lian shi bie"
+CONFIG_CN_SPEECH_COMMAND_ID3="yi dong zhen ce"
+CONFIG_CN_SPEECH_COMMAND_ID4="tian jia ren lian"
+CONFIG_CN_SPEECH_COMMAND_ID5="shan chu ren lian"
+CONFIG_CN_SPEECH_COMMAND_ID6="shi bie yi xia"
+
+# English
+# CONFIG_SR_WN_MODEL_WN8_QUANT8=y
+# CONFIG_SR_WN_WN8_HIESP=y
+# 
+# CONFIG_SR_MN_ENGLISH=y
+# CONFIG_SR_MN_EN_MULTINET5_SINGLE_RECOGNITION_QUANT8=y
+# CONFIG_EN_SPEECH_COMMAND_ID0="STnP WkKgl"
+# CONFIG_EN_SPEECH_COMMAND_ID1="DgSPLd bNLm"
+# CONFIG_EN_SPEECH_COMMAND_ID2="FdS RfKcGNgscN"
+# CONFIG_EN_SPEECH_COMMAND_ID3="MbscN DgTfKscN"
+# CONFIG_EN_SPEECH_COMMAND_ID4="fNTk FdS"
+# CONFIG_EN_SPEECH_COMMAND_ID5="DgLmT FdS"
+# CONFIG_EN_SPEECH_COMMAND_ID6="RfKcGNiZ FdS"