diff --git a/.gitignore b/.gitignore index d09c466..05d6839 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ build sdkconfig sdkconfig.old .DS_Store + +.vscode diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f1f9d9..8149da2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,7 @@ before_script: - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # replace submodule esp-idf to internal repository to speedup cloning - sed -i "s%https://github.com/espressif/esp-idf%${GITLAB_SSH_SERVER}/idf/esp-idf.git%" .gitmodules + - sed -i "s%https://github.com/espressif/esp-face%${GITLAB_SSH_SERVER}/face-recognition-framework/esp-face.git%" .gitmodules - git submodule update --init # (the same regular expressions are used to set these are used in 'only:' sections below - source esp-idf/tools/ci/configure_ci_environment.sh diff --git a/components/esp-face b/components/esp-face index 3b87cf0..2f281e8 160000 --- a/components/esp-face +++ b/components/esp-face @@ -1 +1 @@ -Subproject commit 3b87cf09ba22386fe580b5260bac25bab2f3ecc2 +Subproject commit 2f281e81e8eb2c17e1e5c04cc0f720a78adc598e diff --git a/components/esp32-camera b/components/esp32-camera index f8f26ab..165a47f 160000 --- a/components/esp32-camera +++ b/components/esp32-camera @@ -1 +1 @@ -Subproject commit f8f26ab04492c28be5922ac06994a78440ad8258 +Subproject commit 165a47fe6a7c76daf23fa40512ae4a90a994e938 diff --git a/components/recorder_engine/component.mk b/components/recorder_engine/component.mk new file mode 100644 index 0000000..7ae9250 --- /dev/null +++ b/components/recorder_engine/component.mk @@ -0,0 +1,12 @@ +COMPONENT_ADD_INCLUDEDIRS := . \ + ./include + +COMPONENT_SRCDIRS := . \ + ./include + +LIBS := esp_wakenet nn_model + +COMPONENT_ADD_LDFLAGS := -L$(COMPONENT_PATH)/ $(addprefix -l,$(LIBS)) + +ALL_LIB_FILES += $(patsubst %,$(COMPONENT_PATH)/lib%.a,$(LIBS)) + diff --git a/components/recorder_engine/include/esp_sr_iface.h b/components/recorder_engine/include/esp_sr_iface.h new file mode 100644 index 0000000..b4001b3 --- /dev/null +++ b/components/recorder_engine/include/esp_sr_iface.h @@ -0,0 +1,134 @@ +#pragma once +#include "stdint.h" +#include "esp_err.h" + +//Opaque model data container +typedef struct model_iface_data_t model_iface_data_t; + +//Set wake words recognition operating mode +//The probability of being wake words is increased with increasing mode, +//As a consequence also the false alarm rate goes up +typedef enum { + DET_MODE_90 = 0, //Normal, response accuracy rate about 90% + DET_MODE_95 //Aggressive, response accuracy rate about 95% +} det_mode_t; + +typedef struct { + int wake_word_num; //The number of all wake words + char **wake_word_list; //The name list of wake words +} wake_word_info_t; + +/** + * @brief Easy function type to initialze a model instance with a detection mode + * + * @param det_mode The wake words detection mode to trigger wake words, the range of det_threshold is 0.5~0.9999 + * @returns Handle to the model data + */ +typedef model_iface_data_t* (*esp_sr_iface_op_create_t)(det_mode_t det_mode); + + +/** + * @brief Callback function type to fetch the amount of samples that need to be passed to the detect function + * + * Every speech recognition model processes a certain number of samples at the same time. This function + * can be used to query that amount. Note that the returned amount is in 16-bit samples, not in bytes. + * + * @param model The model object to query + * @return The amount of samples to feed the detect function + */ +typedef int (*esp_sr_iface_op_get_samp_chunksize_t)(model_iface_data_t *model); + + +/** + * @brief Get the sample rate of the samples to feed to the detect function + * + * @param model The model object to query + * @return The sample rate, in hz + */ +typedef int (*esp_sr_iface_op_get_samp_rate_t)(model_iface_data_t *model); + +/** + * @brief Get the number of wake words + * + * @param model The model object to query + * @returns the number of wake words + */ +typedef int (*esp_sr_iface_op_get_word_num_t)(model_iface_data_t *model); + +/** + * @brief Get the name of wake word by index + * + * @Warning The index of wake word start with 1 + + * @param model The model object to query + * @param word_index The index of wake word + * @returns the detection threshold + */ +typedef char* (*esp_sr_iface_op_get_word_name_t)(model_iface_data_t *model, int word_index); + +/** + * @brief Get the structure which contains the information about wake words + * + * @param model The model object to query + * @param word_list The structure which contains the number and name of wake words + * @return + * - ESP_OK Success + * - ESP_FAIL The word_list is NULL. + */ +typedef esp_err_t (*esp_sr_iface_op_get_word_list_t)(model_iface_data_t *model, wake_word_info_t* word_list); + +/** + * @brief Set the detection threshold to manually abjust the probability + * + * @param model The model object to query + * @param det_treshold The threshold to trigger wake words, the range of det_threshold is 0.5~0.9999 + * @param word_index The index of wake word + * @return 0: setting failed, 1: setting success + */ +typedef int (*esp_sr_iface_op_set_det_threshold_t)(model_iface_data_t *model, float det_threshold, int word_index); + +/** + * @brief Get the wake word detection threshold of different modes + * + * @param model The model object to query + * @param det_mode The wake words recognition operating mode + * @param word_index The index of wake word + * @returns the detection threshold + */ +typedef float (*esp_sr_iface_op_get_det_threshold_t)(model_iface_data_t *model, det_mode_t det_mode, int word_index); + +/** + * @brief Feed samples of an audio stream to the speech recognition model and detect if there is a keyword found. + * + * @Warning The index of wake word start with 1, 0 means no wake words is detected. + * + * @param model The model object to query + * @param samples An array of 16-bit signed audio samples. The array size used can be queried by the + * get_samp_chunksize function. + * @return The index of wake words, return 0 if no wake word is detected, else the index of the wake words. + */ +typedef int (*esp_sr_iface_op_detect_t)(model_iface_data_t *model, int16_t *samples); + +/** + * @brief Destroy a speech recognition model + * + * @param model Model object to destroy + */ +typedef void (*esp_sr_iface_op_destroy_t)(model_iface_data_t *model); + + +/** + * This structure contains the functions used to do operations on a speech recognition model. + */ +typedef struct { + esp_sr_iface_op_create_t create; + esp_sr_iface_op_get_samp_chunksize_t get_samp_chunksize; + esp_sr_iface_op_get_samp_rate_t get_samp_rate; + esp_sr_iface_op_get_word_num_t get_word_num; + esp_sr_iface_op_get_word_name_t get_word_name; + esp_sr_iface_op_get_word_list_t get_word_list; + esp_sr_iface_op_set_det_threshold_t set_det_threshold; + esp_sr_iface_op_get_det_threshold_t get_det_threshold_by_mode; + esp_sr_iface_op_detect_t detect; + esp_sr_iface_op_destroy_t destroy; +} esp_sr_iface_t; \ No newline at end of file diff --git a/components/recorder_engine/include/esp_sr_models.h b/components/recorder_engine/include/esp_sr_models.h new file mode 100644 index 0000000..6f3d828 --- /dev/null +++ b/components/recorder_engine/include/esp_sr_models.h @@ -0,0 +1,35 @@ +#pragma once +#include "esp_sr_iface.h" + +//Contains declarations of all available speech recognion models. Pair this up with the right coefficients and you have a model that can recognize +//a specific phrase or word. +extern const esp_sr_iface_t esp_sr_wakenet2_float; +extern const esp_sr_iface_t sr_model_wakenet1_float; +extern const esp_sr_iface_t sr_model_wakenet1_quantized; +extern const esp_sr_iface_t esp_sr_wakenet2_quantized; +extern const esp_sr_iface_t esp_sr_wakenet3_quantized; +extern const esp_sr_iface_t esp_sr_wakenet4_quantized; + + +/* example + +static const sr_model_iface_t *model = &sr_model_wakenet3_quantized; + +//Initialize wakeNet model data +static model_iface_data_t *model_data=model->create(DET_MODE_90); + +//Set parameters of buffer +int audio_chunksize=model->get_samp_chunksize(model_data); +int frequency = model->get_samp_rate(model_data); +int16_t *buffer=malloc(audio_chunksize*sizeof(int16_t)); + +//Detect +int r=model->detect(model_data, buffer); +if (r>0) { + printf("Detection triggered output %d.\n", r); +} + +//Destroy model +model->destroy(model_data) + +*/ diff --git a/components/recorder_engine/libesp_wakenet.a b/components/recorder_engine/libesp_wakenet.a new file mode 100644 index 0000000..5709346 Binary files /dev/null and b/components/recorder_engine/libesp_wakenet.a differ diff --git a/components/recorder_engine/libnn_model.a b/components/recorder_engine/libnn_model.a new file mode 100644 index 0000000..30ebc36 Binary files /dev/null and b/components/recorder_engine/libnn_model.a differ diff --git a/docs/_static/get-started/esp-eye_callout.png b/docs/_static/get-started/esp-eye_callout.png new file mode 100644 index 0000000..a512849 Binary files /dev/null and b/docs/_static/get-started/esp-eye_callout.png differ diff --git a/docs/_static/get-started/face_id_enrollment_cn.jpg b/docs/_static/get-started/face_id_enrollment_cn.jpg new file mode 100644 index 0000000..1440a84 Binary files /dev/null and b/docs/_static/get-started/face_id_enrollment_cn.jpg differ diff --git a/docs/_static/get-started/face_id_enrollment_en.png b/docs/_static/get-started/face_id_enrollment_en.png new file mode 100644 index 0000000..dcf423b Binary files /dev/null and b/docs/_static/get-started/face_id_enrollment_en.png differ diff --git a/docs/_static/get-started/wifi_connection.jpeg b/docs/_static/get-started/wifi_connection.jpeg new file mode 100644 index 0000000..71ca504 Binary files /dev/null and b/docs/_static/get-started/wifi_connection.jpeg differ diff --git a/docs/_static/get-started/work_flow_cn.jpg b/docs/_static/get-started/work_flow_cn.jpg new file mode 100644 index 0000000..9ba9b2e Binary files /dev/null and b/docs/_static/get-started/work_flow_cn.jpg differ diff --git a/docs/_static/get-started/work_flow_en.png b/docs/_static/get-started/work_flow_en.png new file mode 100644 index 0000000..1cd53d6 Binary files /dev/null and b/docs/_static/get-started/work_flow_en.png differ diff --git a/docs/en/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md b/docs/en/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md new file mode 100644 index 0000000..2f0a655 --- /dev/null +++ b/docs/en/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md @@ -0,0 +1,190 @@ +# ESP-EYE Getting Started Guide + +[[中文]](../../en/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md) + +## What You Need + +* 1 × ESP-EYE V2.0 board +* 1 × Micro USB cable +* 1 × PC loaded with Windows, Linux or Mac OS + +## Overview + +ESP-EYE is an ESP32-based development board that integrates a digital microphone, 8 MB PSRAM and 4 MB flash, as well as provides an external 2-million-pixel camera, which makes the board very suitable for applications in the fields of face detection, face recognition, and speech recognition. Besides, the board can also support image transmission over Wi-Fi and debugging using the Micro USB port, which enables the users' development of advanced AI solutions. + +## Hardware Preparation + +The list and figure below describe the key components, interfaces, and controls of the ESP-EYE development board: + +![ESP-EYE image](../../_static/get-started/esp-eye_callout.png) + +* **3D_PIFA Antenna** + + A 3D PIFA antenna. Users can choose the R14 resistor to select external IPEX antenna or choose the R15 resistor to select the 3D antenna. + +* **IPEX Connector** + + Used for connecting external IPEX antenna. Users can choose the R14 resistor to select external IPEX antenna or choose the R15 resistor to select the 3D antenna. + * **ESP32 Chip** + + A 2.4 GHz Wi-Fi and Bluetooth combo chip + +* **Crystal Oscillator** + + Provides an external clock for ESP32. + +* **Flash & PSRAM** + + Provides memory to store applications. + +* **CP2102 USB-to-UART Chip** + + Converts the USB signals to UART signals. + +* **USB Port** + + Provides the power supply to the whole system. + +* **LDO Power Supply** + + Provides the required power supplies to the ESP32 chip, camera and LED indicators. + +* **Side Tactile Button** + + A function key + +* **Top Tactile Button** + + Reset/Boot button. We recommend that you do not configure this button for other uses. + +* **LED Indicators** + + The board has a red indicator and a white indicator. Different combinations of red and white indicators reflect the different status of the board, such as waking up, networking, face detection, face recognition, face enrollment, and face recognition... + +* **Camera** + + An external 2-million-pixel camera module for face detection, face recognition and Face ID enrollment + +* **Camera Connector** + + Used to connect the external camera. + +* **MIC** + + A digital microphone for voice control functions + +* **SPI Port** + + A reserved port for data transmission + + +## Software Development + +ESP-EYE supports firmware downloading with a PC loaded with Linux, MacOs or Windows operating systems. For now, users must set up toolchain in the development environment before starting the software development. + +### Preparation + +- Go through [Get Started](https://docs.espressif.com/projects/esp-idf/en/v3.1.1/get-started/index.html) to set up toolchain in your PC; +- Connect ESP-EYE to your PC with a Micro USB cable; +- Launch your development environment tool, such as Terminal (Linux/MacOS) or MinGW (Windows). + +### Get ESP-WHO + +To obtain a local copy: open your Terminal (such as Terminal in a Linux environment), navigate to the directory you want to put ESP-WHO, and clone the repository using `git clone` command: + +``` +git clone --recursive https://github.com/espressif/esp-who.git +``` + +ESP-WHO will be downloaded into an automatically generated folder named `esp-who`. + +> Do not miss the `--recursive` option. If you have already cloned ESP-WHO without this option, please run another command to get all the submodules: `git submodule update --init --recursive` + +### Set UP Path to ESP-WHO + +Please follow the instruction described in [Setup Path to ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/v3.1.1/get-started/index.html#get-started-setup-path) to configure the `IDF_PATH` variable to `esp-who/esp-idf`. + +### Download Firmware + +This section describes the steps to download firmware to ESP-EYE (taking the Linux operating system as an example): + +- Power up ESP-EYE by connecting it to your PC with a USB cable. +- Run `ls /dev/ttyUSB*` in your Terminal to check if the board has connected itself to your PC successfully. You will see a newly generated item named something like `/dev/ttyUSB0` in your list after running the command if the connection is successful. +- Navigate to an example directory, such as `cd esp-who/examples/single_chip/recognition_solution`. +- Run `make defconfig` to complete default configuration. +- Run `make menuconfig`, and go to `Serial flasher config` -> `Default serial port` to configure the device name to the item you saw in the second step, something like `/dev/ttyUSB0`. Then, save and exit. +- Run `make flash` to download firmware. + +### Check Logs + +This section describes the steps to check logs with your Terminal (taking the Linux operating system as an example). + +- Launch your Terminal; +- Run `make monitor`. + +> Note: The ESP-EYE development board will reboot after this operation. + +### Key Process + +The figure below describes the workflow of ESP-EYE: + +![esp-eye-workflow](../../_static/get-started/work_flow_en.png) + +#### 1. Voice Wake-up + +ESP-EYE awaits to be woken up after powering up (Red LED on and white LED off). The board wakes up after recognizing the wake-up command "Hi Lexin", then awaits for networking (Red LED flashing and white LED off). Now, users can initiate the networking. + +#### 2. Networking + +Users can connect their PCs or mobile phones to ESP-EYE's Wi-Fi with the following information (by default): + +- Username: esp-eye-xxxx (xxxx indicates the board's MAC address) +- Password: no needed + +Alternatively, users can also follow the steps below to configure the username and password of the board's Wi-Fi: + +- Launch your Terminal; +- Run `make menuconfig`, and complete the configuration as instructed in the figure below: + + ![wifi connection](../../_static/get-started/wifi_connection.jpeg) + +> Note: You will have to restart from the step of downloading firmware after you have reconfigured the Wi-Fi username and password. + +#### 3. Face Detection + +ESP-EYE starts the face detection after networking. Users can see the real-time image, captured by the board, with their browser (address: `192.168.4.1/face_stream`). During this, the red LED is off and the white LED is on. + +#### 4. Face Recognition + +After detecting a face, ESP-EYE will start the face +recognition if there are any enrolled Face IDs stored in the board: + +- When there is a match: the red LED on the board flashes once, and the browser displays **HELLO ID XXX**. +- When there isn't any match: the board shows no signs, and the browser displays **WHO?**”. + +If there isn't any enrolled Face ID, the board continues face detecting. Therefore, please at least enroll one Face ID if you want to start the face +recognition. + +#### 5. Add/delete a Face ID + +The users can add/delete a Face ID after the network is successfully established. + +##### 5.1 Add a Face ID + +![Enroll a Face ID](../../_static/get-started/face_id_enrollment_en.png) + +- Single-click the Side Tactile Button to enroll a new Face ID. At this moment, the red LED is on, and the browser displays **START ENROLLING**; +- Stand in front of the camera, and the face sampling starts automatically. The red LED flashes whenever the board gets a face sample, and the browser displays the numerical order of the current face sample, such as **THE 1st SAMPLE**. By default, the board has to take three samples to add one Face ID. Users can configure the number of samples needed for one Face ID. (Please adjust your position/distance from the camera and try again if you don't see the red LED flashing for a long time). +- After the Face ID enrollment, the red LED on the board is off and the browser displays **ENROLLED FACE ID XXX**; +- The board enters Face Detection after the Face ID enrollment. + +Currently, ESP-EYE can enroll up to 10 Face IDs. Please note that the maximum number of enrolled Face IDs can be configured based on how the users allocate the flash memory. However, we recommend a number that is no greater than 30. + +##### 5.2 Delete a Face ID + +- Double-click the Side Tactile Button to delete an existing Face ID; +- After that, the board deletes the earliest record of all the existing enrolled Face ID. The white LED on the board flashes, and the browser displays: **XXX ID(S) LEFT**. + +#### Troubleshooting + +The board returns to the status of "waiting to be woken up" when there are any network anomalies, such as "network disconnection" and "network timeout". \ No newline at end of file diff --git a/docs/zh_CN/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md b/docs/zh_CN/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md new file mode 100644 index 0000000..7a661d6 --- /dev/null +++ b/docs/zh_CN/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md @@ -0,0 +1,198 @@ +# ESP-EYE 入门指南 + +[[EN]](../../en/get-started/ESP-EYE_V2.0_Getting_Started_Guide.md) + +## 准备工作 + +* 1 × ESP-EYE V2.0 开发板 +* 1 × Micro USB B 电缆 +* 1 × PC（Windows、Linux 或 Mac OS） + +## 简介 + +ESP-EYE 是一款面向人脸识别和语音识别市场的开发板，搭载 ESP32 芯片、200 W 像素摄像头、数字麦克风、8 MB PSRAM 和 4 MB Flash，可满足各种 AI 应用开发需求。此外，该开发板还支持 Wi-Fi 图像传输、Micro USB 调试和供电，可以实现语音唤醒、人脸检测与识别等功能，可协助用户开发高度集成的 AI 解决方案。 + +## 硬件组成 + +ESP-EYE 开发板的产品图请见下方： + +![ESP-EYE image](../../_static/get-started/esp-eye_callout.png) + +具体包括以下硬件组成： + +* **3D_PIFA Antenna（3D_PIFA 天线）** + + 3D PIFA 天线。用户可通过选贴 R14/R15 电阻，选用 3D 天线（选贴 R15）或外接 IPEX 天线（选贴 R14）。 + +* **IPEX Connector（IPEX 连接器）** + + 用于外接 IPEX 天线。用户可通过选贴 R14/R15 电阻，选用 3D 天线（选贴 R15）或外接 IPEX 天线（选贴 R14）。 + +* **ESP32 Chip（ESP32 芯片）** + + 集成 2.4 GHz Wi-Fi 和蓝牙双模的单芯片。 + +* **Crystal（晶振）** + + ESP32 的外部晶振时钟源。 + +* **Flash & PSRAM** + + 存储芯片，用于储存程序。 + +* **CP2102 USB-UART Chip（USB 转 UART 芯片）** + + 实现 USB 到 UART 的转换功能。 + +* **USB Port（USB 供电接口）** + + 为整个系统供电。 + +* **LDO Power Supply（LDO 供电芯片）** + + 为 ESP32 芯片、摄像头、LED 指示灯等部件提供各自所需的电压。 + +* **Side Tactile Button（侧面轻触按键）** + + 功能按键。 + +* **Top Tactile Button（正面轻触按键）** + + 用于 ESP32 的 RST（复位）、BOOT（下载），不建议设置他用。 + +* **LED 指示灯** + + 红灯和白灯各一个，可用于指示唤醒、联网、人脸检测、人脸录入、人脸识别等不同过程中的状态。 + +* **Camera（摄像头）** + + 实现检测识别等功能。 + +* **Camera Connector（摄像头连接器）** + + 用于外接摄像头模块。 + +* **MIC（麦克风）** + + 数字型麦克风，实现语音控制功能。 + +* **SPI Port（SPI 接口）** + + 预留数据传输接口。 + + +## 软件开发 + +ESP-EYE 可在 Linux、MacOs、Windows 操作系统中完成软件烧写。目前，必须进行开发环境的工具链配置，详见下方介绍。 + +### 准备工作 + +- 阅读 [ESP-IDF编程指南](https://docs.espressif.com/projects/esp-idf/zh_CN/v3.1.1/get-started/index.html)，参考相应章节，配置工具链； +- 准备 Micro USB 线，用于连接 PC 和 ESP-EYE 开发板； +- 选择一款适合开发环境的工具，例如 Terminal (Linux/MacOS) 或 MinGW (Windows) 等。 + +### 软件获取 + +打开终端（例如 Linux 环境下的 Terminal），将软件代码克隆到本地： + +``` +git clone --recursive https://github.com/espressif/esp-who.git +``` + +执行以上命令会默认生成一个 `esp-who` 的文件夹。 + +> 注意不要忘记 `--recursive` 选项。如果你克隆 ESP-IDF 时没有带这个选项，你还需要运进入相应文件夹中，执行以下命令下载相应的子模块： +``` +git submodule update --init --recursive +``` + +### 设置路径 + +请参考[设置路径](https://docs.espressif.com/projects/esp-idf/zh_CN/v3.1.1/get-started/index.html#get-started-setup-path)章节，将 `IDF_PATH` 设置为 `esp-who/esp-idf`。 + +### 软件烧写 + +下面，我们以 Linux 环境为例，介绍向 ESP-EYE 烧写程序的过程： + +- 首先将 ESP-EYE 接入 PC，接入即上电； +- 通过命令 `ls /dev/ttyUSB*` 查看开发板是否成功接入 PC。成功接入后，列表将新增类似 `/dev/ttyUSB0` 的信息； +- 进入一个 example 工程文件中，例如 `cd esp-who/examples/single_chip/recognition_solution`； +- 执行 `make defconfig` 进行默认配置； +- 执行 `make menuconfig`，在 `Serial flasher config` 中设置 `Default serial port` 设备名称（与第二步查看的设备名称一致，一般设置为 `/dev/ttyUSB0`），保存退出； +- 执行 `make flash`，进行软件烧写。 + +### 终端获取日志 + +下面，我们以 Linux 环境为例，介绍如何查看日志： + +- 打开终端； +- 执行 `make monitor`。 + +> 注意：这个过程会重启开发板。 + +### 交互功能 + +ESP-EYE 开发板的工作流程如下图所示： + +![esp-eye-workflow](../../_static/get-started/work_flow_cn.jpg) + + +#### 1. 语音唤醒 + +开发板上电后，会进入“等待唤醒”状态（红灯常亮、白灯常灭），需要用户通过语音进行唤醒。支持“Hi 乐鑫”唤醒，当用户说出“Hi 乐鑫”的唤醒词后，开发板唤醒并进入“等待联网”状态（红灯闪烁，白灯常灭）。此时，用户可进行联网操作。 + +#### 2. 连接网络 + +用户可通过 PC、手机等设备，连接 ESP-EYE 创建的 Wi-Fi 热点。该热点的默认信息如下： + +- 用户名：esp-eye-xxxx（xxxx 为设备 MAC 地址） +- 密码：无需密码 + +用户也可通过如下方式，自行设置用户名和密码： + +- 打开终端； +- 执行 `make menuconfig`，并按照下图进行设置： + + ![wifi connection](../../_static/get-started/wifi_connection.jpeg) + +> 注：用户重新设置 Wi-Fi 热点的用户名和密码后，需要重新进行软件烧录。 + +#### 3. 人脸检测 + +联网成功后，ESP-EYE 会进行“人脸检测”。用户可以打开浏览器，输入地址 `192.168.4.1/face_stream`，在网页上即可看到实时图像信息。此时，开发板红灯熄灭、白灯常亮。 + +#### 4. 人脸识别 + +当开发板检测到人脸时，如已存在录入的 Face ID，则开发板将进行“人脸识别”： + +- “人脸识别”匹配成功 -- 开发板红灯闪烁 1 次，网页显示 **HELLO ID XXX** +- “人脸识别”匹配失败 -- 开发板无反应，网页显示 **WHO?**” + +否则，开发板仅进行“人脸检测”。此时，若用户希望使用人脸识别功能，则请首先录入至少一个 Face ID。 + +#### 5. 人脸录入与删除 + +在联网成功的前提下，可通过摄像头采集人脸，录入 Face ID。 + +##### 5.1 录入 Face ID + +![录入 Face ID](../../_static/get-started/face_id_enrollment_cn.jpg) + +- 用户单击侧面轻触按键，可进入“录入 Face ID”（红灯常亮），网页显示: **START ENROLLING**； +- 用户面对摄像头，开始采集人像。每次成功一次采集，开发板将红灯闪烁，网页显示对应的采集次数，比如 **THE 1st SAMPLE** 等。默认情况下，用户每录入一个 Face ID 需要采集 3 次人像（可配置）。在人像采集过程中，如果红灯长时间未闪烁，建议用户调整姿态和角度，然后再试； +- 人像采集完成后，开发板红灯常灭，表明已完成录入该 Face ID。此时，网页显示：**ENROLLED FACE ID xxx**； +- Face ID 录入成功后，系统将返回“人脸检测”。 + +目前，ESP-EYE 开发板默认可录入 10 个 Face ID（可配置，具体与用户的 flash 内存分配有关，但建议不要超过 30 个 Face ID）。 + +##### 5.2 删除 Face ID + +- 用户双击侧面轻触按键，进入“删除 FACE ID”； +- 双击后，开发板白灯闪烁，系统将自动删除系统中存在的最早一条 FACE ID，终端显示：**XXX ID(S) LEFT**。 + +#### 异常情况 + +当出现“网络断开”或“联网超时”等异常情况时，开发板会回到“等待唤醒”状态。 + +Positioning of 5Gtowers linked by a “diber backbone” at Millbrook.( Millbrook) +Millbrook 试验场中，通过 “diber backbone” 相互连接的 5G 电信塔。（图片来源：Millbrook） \ No newline at end of file diff --git a/examples/single_chip/camera_web_server/main/app_httpd.c b/examples/single_chip/camera_web_server/main/app_httpd.c index 83517b3..7e2ce3b 100644 --- a/examples/single_chip/camera_web_server/main/app_httpd.c +++ b/examples/single_chip/camera_web_server/main/app_httpd.c @@ -75,8 +75,7 @@ static int8_t detection_enabled = 0; #if CONFIG_ESP_FACE_RECOGNITION_ENABLED static int8_t recognition_enabled = 0; static int8_t is_enrolling = 0; -static int32_t next_enroll_index = 0; -static dl_matrix3d_t *id_list[FACE_ID_SAVE_NUMBER] = {0}; +static face_id_list id_list = {0}; #endif #endif static ra_filter_t ra_filter; @@ -195,31 +194,21 @@ static int run_face_recognition(dl_matrix3du_t *image_matrix, box_array_t *net_b return matched_id; } if (align_face(net_boxes, image_matrix, aligned_face) == ESP_OK){ - if ((is_enrolling == 1) && (next_enroll_index < FACE_ID_SAVE_NUMBER)) { - if (id_list[next_enroll_index] == NULL) { - id_list[next_enroll_index] = dl_matrix3d_alloc(1, 1, 1, FACE_ID_SIZE); - if(!id_list[next_enroll_index]){ - ESP_LOGE(TAG, "Could not allocate id_list"); - dl_matrix3du_free(aligned_face); - return matched_id; - } - } - - int8_t left_sample_face = enroll(aligned_face, id_list[next_enroll_index], ENROLL_CONFIRM_TIMES); + if (is_enrolling == 1){ + int8_t left_sample_face = enroll_face(&id_list, aligned_face); if(left_sample_face == (ENROLL_CONFIRM_TIMES - 1)){ - ESP_LOGD(TAG, "Enrolling Face ID: %d", next_enroll_index+1); + ESP_LOGD(TAG, "Enrolling Face ID: %d", id_list.tail); } - ESP_LOGD(TAG, "Enrolling Face ID: %d sample %d", next_enroll_index+1, ENROLL_CONFIRM_TIMES - left_sample_face); - rgb_printf(image_matrix, FACE_COLOR_CYAN, "ID[%u] Sample[%u]", next_enroll_index+1, ENROLL_CONFIRM_TIMES - left_sample_face); + ESP_LOGD(TAG, "Enrolling Face ID: %d sample %d", id_list.tail, ENROLL_CONFIRM_TIMES - left_sample_face); + rgb_printf(image_matrix, FACE_COLOR_CYAN, "ID[%u] Sample[%u]", id_list.tail, ENROLL_CONFIRM_TIMES - left_sample_face); if (left_sample_face == 0){ - next_enroll_index++; is_enrolling = 0; - ESP_LOGD(TAG, "Enrolled Face ID: %d", next_enroll_index); + ESP_LOGD(TAG, "Enrolled Face ID: %d", id_list.tail); } } else { - matched_id = recognize_face(aligned_face, id_list, FACE_REC_THRESHOLD, next_enroll_index); - if (matched_id) { + matched_id = recognize_face(&id_list, aligned_face); + if (matched_id >= 0) { ESP_LOGW(TAG, "Match Face ID: %u", matched_id); rgb_printf(image_matrix, FACE_COLOR_GREEN, "Hello Subject %u", matched_id); } else { @@ -699,6 +688,9 @@ void app_httpd_main(){ mtmn_config.o_threshold.score = 0.7; mtmn_config.o_threshold.nms = 0.4; mtmn_config.o_threshold.candidate_number = 1; +#if CONFIG_ESP_FACE_RECOGNITION_ENABLED + face_id_init(&id_list, FACE_ID_SAVE_NUMBER, ENROLL_CONFIRM_TIMES); +#endif #endif ESP_LOGI(TAG, "Starting web server on port: '%d'", config.server_port); if (httpd_start(&camera_httpd, &config) == ESP_OK) { diff --git a/examples/single_chip/camera_web_server/main/component.mk b/examples/single_chip/camera_web_server/main/component.mk old mode 100755 new mode 100644 diff --git a/examples/single_chip/camera_web_server/sdkconfig.defaults b/examples/single_chip/camera_web_server/sdkconfig.defaults index aeae367..96ff546 100755 --- a/examples/single_chip/camera_web_server/sdkconfig.defaults +++ b/examples/single_chip/camera_web_server/sdkconfig.defaults @@ -73,22 +73,19 @@ CONFIG_SPIRAM_SUPPORT=y CONFIG_SPIRAM_BOOT_INIT=y CONFIG_SPIRAM_IGNORE_NOTFOUND= CONFIG_SPIRAM_USE_MEMMAP= -CONFIG_SPIRAM_USE_CAPS_ALLOC= -CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y +CONFIG_SPIRAM_USE_MALLOC= CONFIG_SPIRAM_TYPE_AUTO=y CONFIG_SPIRAM_TYPE_ESPPSRAM32= CONFIG_SPIRAM_TYPE_ESPPSRAM64= -CONFIG_SPIRAM_SIZE=4194304 +CONFIG_SPIRAM_SIZE=-1 CONFIG_SPIRAM_SPEED_40M= CONFIG_SPIRAM_SPEED_80M=y CONFIG_SPIRAM_MEMTEST=y CONFIG_SPIRAM_CACHE_WORKAROUND=y CONFIG_SPIRAM_BANKSWITCH_ENABLE=y CONFIG_SPIRAM_BANKSWITCH_RESERVE=8 -CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 CONFIG_WIFI_LWIP_ALLOCATION_FROM_SPIRAM_FIRST= -CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=32768 -CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY= CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY= CONFIG_TASK_WDT= diff --git a/examples/single_chip/detection_with_command_line/main/Kconfig.projbuild b/examples/single_chip/detection_with_command_line/main/Kconfig.projbuild old mode 100644 new mode 100755 diff --git a/examples/single_chip/detection_with_command_line/main/app_camera.cpp b/examples/single_chip/detection_with_command_line/main/app_camera.cpp index b00a171..feca609 100644 --- a/examples/single_chip/detection_with_command_line/main/app_camera.cpp +++ b/examples/single_chip/detection_with_command_line/main/app_camera.cpp @@ -19,13 +19,11 @@ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ + #include "app_camera.h" static const char *TAG = "app_camera"; -QueueHandle_t gpst_input_queue = NULL; -TaskHandle_t gpst_input_task = NULL; - void app_camera_init() { camera_config_t config; @@ -59,6 +57,4 @@ void app_camera_init() ESP_LOGE(TAG, "Camera init failed with error 0x%x", err); return; } - - vTaskDelay(200 / portTICK_PERIOD_MS); } diff --git a/examples/single_chip/detection_with_command_line/main/app_facenet.c b/examples/single_chip/detection_with_command_line/main/app_facenet.c index 880482e..a3f87e9 100644 --- a/examples/single_chip/detection_with_command_line/main/app_facenet.c +++ b/examples/single_chip/detection_with_command_line/main/app_facenet.c @@ -24,7 +24,7 @@ #include "esp_log.h" #include "esp_system.h" #include "freertos/FreeRTOS.h" -#include "freertos/queue.h" +#include "freertos/task.h" #include "app_facenet.h" #include "sdkconfig.h" #include "dl_lib.h" diff --git a/examples/single_chip/detection_with_command_line/main/include/app_camera.h b/examples/single_chip/detection_with_command_line/main/include/app_camera.h index 0df36a2..f1a4d30 100644 --- a/examples/single_chip/detection_with_command_line/main/include/app_camera.h +++ b/examples/single_chip/detection_with_command_line/main/include/app_camera.h @@ -26,9 +26,6 @@ #include "esp_log.h" #include "esp_system.h" -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" -#include "freertos/queue.h" #include "esp_camera.h" /** diff --git a/examples/single_chip/detection_with_command_line/sdkconfig.defaults b/examples/single_chip/detection_with_command_line/sdkconfig.defaults index 21ead3b..96ff546 100644 --- a/examples/single_chip/detection_with_command_line/sdkconfig.defaults +++ b/examples/single_chip/detection_with_command_line/sdkconfig.defaults @@ -73,22 +73,19 @@ CONFIG_SPIRAM_SUPPORT=y CONFIG_SPIRAM_BOOT_INIT=y CONFIG_SPIRAM_IGNORE_NOTFOUND= CONFIG_SPIRAM_USE_MEMMAP= -CONFIG_SPIRAM_USE_CAPS_ALLOC= -CONFIG_SPIRAM_USE_MALLOC=y -CONFIG_SPIRAM_TYPE_AUTO= -CONFIG_SPIRAM_TYPE_ESPPSRAM32=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y +CONFIG_SPIRAM_USE_MALLOC= +CONFIG_SPIRAM_TYPE_AUTO=y +CONFIG_SPIRAM_TYPE_ESPPSRAM32= CONFIG_SPIRAM_TYPE_ESPPSRAM64= -CONFIG_SPIRAM_SIZE=4194304 +CONFIG_SPIRAM_SIZE=-1 CONFIG_SPIRAM_SPEED_40M= CONFIG_SPIRAM_SPEED_80M=y CONFIG_SPIRAM_MEMTEST=y CONFIG_SPIRAM_CACHE_WORKAROUND=y CONFIG_SPIRAM_BANKSWITCH_ENABLE=y CONFIG_SPIRAM_BANKSWITCH_RESERVE=8 -CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 CONFIG_WIFI_LWIP_ALLOCATION_FROM_SPIRAM_FIRST= -CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=32768 -CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY= CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY= CONFIG_TASK_WDT= diff --git a/examples/single_chip/recognition_solution/Makefile b/examples/single_chip/recognition_solution/Makefile new file mode 100644 index 0000000..f7523e8 --- /dev/null +++ b/examples/single_chip/recognition_solution/Makefile @@ -0,0 +1,12 @@ +# +# This is a project Makefile. It is assumed the directory this Makefile resides in is a +# project subdirectory. +# + +PROJECT_NAME := recognition_http + +SOLUTION_PATH ?= $(abspath $(shell pwd))/../../.. + +include $(SOLUTION_PATH)/components/component_conf.mk +include $(IDF_PATH)/make/project.mk + diff --git a/examples/single_chip/recognition_solution/README.md b/examples/single_chip/recognition_solution/README.md new file mode 100644 index 0000000..edb694e --- /dev/null +++ b/examples/single_chip/recognition_solution/README.md @@ -0,0 +1,41 @@ +# Face Detection and Recognition with ESP-EYE + +This example is supposed to demonstrate the elementry function of the board [ESP-EYE](), which is based on ESP32 chip and intergrated with camera, led, and digital microphone modules[detection_with_command_line example](./detection_with_command_line), but displaies the image stream through http server. + +## General Information + +用户可以通过连接esp32建立的softap，在局域网内访问其特定的网址，不管是手机端还是电脑端，都可以在网页上看到图像拍摄和检测的效果。 + +## 安装及使用 + +- 通过`make defconfig`使用默认配置 +- 通过`make menuconfig`设置Wi-Fi的用户名和密码 +- 烧入程序`make flash` +- 找到一个手机或者电脑，连接上建立的softap +- 在浏览器中输入`192.168.4.1/face_stream` +- Please enjoy it + +## 图像格式及尺寸 + +我们建议使用默认的320x240的像素大小，因为更大的像素会带来更多的计算。 + +输入输出的图像格式都使用jpeg，比一般的RGB小很多，适合网络的传输。 + +## 配置(Configuration) + +- 可增加自己定制的handler，例如本例子使用的就是 +``` +httpd_uri_t _face_stream_handler = { + .uri = "/face_stream", + .method = HTTP_GET, + .handler = facenet_stream_handler, + .user_ctx = NULL +}; +``` +在初始化过程中进行注册: +`httpd_register_uri_handler(camera_httpd, &_face_stream_handler);` + +- 可增加camera的缓存队列，提高并行的速度 +`camera_config_t config.fb_count = 2` + + diff --git a/examples/single_chip/recognition_solution/main/Kconfig.projbuild b/examples/single_chip/recognition_solution/main/Kconfig.projbuild new file mode 100644 index 0000000..e0e24d2 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/Kconfig.projbuild @@ -0,0 +1,20 @@ +menu "Example Configuration" + +config ESP_WIFI_SSID + string "WiFi SSID" + default "" + help + SSID (network name) for the example to connect to. + +config ESP_WIFI_PASSWORD + string "WiFi Password" + default "" + help + WiFi password (WPA or WPA2) for the example to use. + +config MAX_STA_CONN + int "Maximal STA connections" + default 1 + help + Max number of the STA connects to AP. +endmenu diff --git a/examples/single_chip/recognition_solution/main/app_camera.c b/examples/single_chip/recognition_solution/main/app_camera.c new file mode 100644 index 0000000..55aa7c1 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/app_camera.c @@ -0,0 +1,73 @@ +/* ESPRESSIF MIT License + * + * Copyright (c) 2018 + * + * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, + * it is free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished + * to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "app_camera.h" + +static const char *TAG = "app_camera"; + +void app_camera_init() +{ + /* IO13, IO14 is designed for JTAG by default, + * to use it as generalized input, + * firstly declair it as pullup input */ + gpio_config_t conf; + conf.mode = GPIO_MODE_INPUT; + conf.pull_up_en = GPIO_PULLUP_ENABLE; + conf.pull_down_en = GPIO_PULLDOWN_DISABLE; + conf.intr_type = GPIO_INTR_DISABLE; + conf.pin_bit_mask = 1LL << 13; + gpio_config(&conf); + conf.pin_bit_mask = 1LL << 14; + gpio_config(&conf); + + camera_config_t config; + config.ledc_channel = LEDC_CHANNEL_0; + config.ledc_timer = LEDC_TIMER_0; + config.pin_d0 = Y2_GPIO_NUM; + config.pin_d1 = Y3_GPIO_NUM; + config.pin_d2 = Y4_GPIO_NUM; + config.pin_d3 = Y5_GPIO_NUM; + config.pin_d4 = Y6_GPIO_NUM; + config.pin_d5 = Y7_GPIO_NUM; + config.pin_d6 = Y8_GPIO_NUM; + config.pin_d7 = Y9_GPIO_NUM; + config.pin_xclk = XCLK_GPIO_NUM; + config.pin_pclk = PCLK_GPIO_NUM; + config.pin_vsync = VSYNC_GPIO_NUM; + config.pin_href = HREF_GPIO_NUM; + config.pin_sscb_sda = SIOD_GPIO_NUM; + config.pin_sscb_scl = SIOC_GPIO_NUM; + config.pin_reset = RESET_GPIO_NUM; + config.pin_pwdn = PWDN_GPIO_NUM; + config.xclk_freq_hz = XCLK_FREQ; + config.pixel_format = CAMERA_PIXEL_FORMAT; + config.frame_size = CAMERA_FRAME_SIZE; + config.jpeg_quality = 10; + config.fb_count = 2; + + // camera init + esp_err_t err = esp_camera_init(&config); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Camera init failed with error 0x%x", err); + return; + } +} diff --git a/examples/single_chip/recognition_solution/main/app_httpserver.c b/examples/single_chip/recognition_solution/main/app_httpserver.c new file mode 100644 index 0000000..728a51f --- /dev/null +++ b/examples/single_chip/recognition_solution/main/app_httpserver.c @@ -0,0 +1,411 @@ +/* ESPRESSIF MIT License + * + * Copyright (c) 2018 + * + * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, + * it is free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished + * to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "app_httpserver.h" +#include "esp_log.h" +#include "image_util.h" +#include "fb_gfx.h" +#include "app_main.h" + +static const char *TAG = "app_httpserver"; + +#define FACE_COLOR_WHITE 0x00FFFFFF +#define FACE_COLOR_BLACK 0x00000000 +#define FACE_COLOR_RED 0x000000FF +#define FACE_COLOR_GREEN 0x0000FF00 +#define FACE_COLOR_BLUE 0x00FF0000 +#define FACE_COLOR_YELLOW (FACE_COLOR_RED | FACE_COLOR_GREEN) +#define FACE_COLOR_CYAN (FACE_COLOR_BLUE | FACE_COLOR_GREEN) +#define FACE_COLOR_PURPLE (FACE_COLOR_BLUE | FACE_COLOR_RED) + +#define ENROLL_CONFIRM_TIMES 3 +#define FACE_ID_SAVE_NUMBER 10 + +#define PART_BOUNDARY "123456789000000000000987654321" +static const char* _STREAM_CONTENT_TYPE = "multipart/x-mixed-replace;boundary=" PART_BOUNDARY; +static const char* _STREAM_BOUNDARY = "\r\n--" PART_BOUNDARY "\r\n"; +static const char* _STREAM_PART = "Content-Type: image/jpeg\r\nContent-Length: %u\r\n\r\n"; + +face_id_list st_face_list = {0}; + +dl_matrix3du_t *aligned_face = NULL; + +static void oneshot_timer_callback(void* arg); + +const char *number_suffix(int32_t number) +{ + uint8_t n = number % 10; + + if (n == 0) + return "zero"; + else if (n == 1) + return "st"; + else if (n == 2) + return "nd"; + else if (n == 3) + return "rd"; + else + return "th"; +} + +static void rgb_print(dl_matrix3du_t *image_matrix, uint32_t color, const char * str){ + fb_data_t fb; + fb.width = image_matrix->w; + fb.height = image_matrix->h; + fb.data = image_matrix->item; + fb.bytes_per_pixel = 3; + fb.format = FB_BGR888; + fb_gfx_print(&fb, (fb.width - (strlen(str) * 14)) / 2, 10, color, str); +} + +static int rgb_printf(dl_matrix3du_t *image_matrix, uint32_t color, const char *format, ...) +{ + char loc_buf[64]; + char * temp = loc_buf; + int len; + va_list arg; + va_list copy; + va_start(arg, format); + va_copy(copy, arg); + len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg); + va_end(copy); + if(len >= sizeof(loc_buf)){ + temp = (char*)malloc(len+1); + if(temp == NULL) { + return 0; + } + } + vsnprintf(temp, len+1, format, arg); + va_end(arg); + rgb_print(image_matrix, color, temp); + if(len > 64){ + free(temp); + } + return len; +} + +static void draw_face_boxes(dl_matrix3du_t *image_matrix, box_array_t *boxes){ + int x, y, w, h, i; + uint32_t color = FACE_COLOR_YELLOW; + fb_data_t fb; + fb.width = image_matrix->w; + fb.height = image_matrix->h; + fb.data = image_matrix->item; + fb.bytes_per_pixel = 3; + fb.format = FB_BGR888; + for (i = 0; i < boxes->len; i++){ + // rectangle box + x = (int)boxes->box[i].box_p[0]; + y = (int)boxes->box[i].box_p[1]; + w = (int)boxes->box[i].box_p[2] - x + 1; + h = (int)boxes->box[i].box_p[3] - y + 1; + fb_gfx_drawFastHLine(&fb, x, y, w, color); + fb_gfx_drawFastHLine(&fb, x, y+h-1, w, color); + fb_gfx_drawFastVLine(&fb, x, y, h, color); + fb_gfx_drawFastVLine(&fb, x+w-1, y, h, color); +#if 0 + // landmark + int x0, y0, j; + for (j = 0; j < 10; j+=2) { + x0 = (int)boxes->landmark[i].landmark_p[j]; + y0 = (int)boxes->landmark[i].landmark_p[j+1]; + fb_gfx_fillRect(&fb, x0, y0, 3, 3, color); + } +#endif + } +} + +esp_err_t facenet_stream_handler(httpd_req_t *req) +{ + esp_err_t res = ESP_OK; + if (g_state != WAIT_FOR_CONNECT) + { + res = httpd_resp_send_404(req); + return res; + } + g_state = START_DETECT; + + camera_fb_t * fb = NULL; + size_t _jpg_buf_len = 0; + uint8_t * _jpg_buf = NULL; + char * part_buf[64]; + dl_matrix3du_t *image_matrix = NULL; + + int face_id = -1; + + int64_t fr_start = 0; + int64_t fr_ready = 0; + int64_t fr_face = 0; + int64_t fr_recognize = 0; + int64_t fr_encode = 0; + + mtmn_config_t mtmn_config = mtmn_init_config(); + + static int64_t last_frame = 0; + if(!last_frame) { + last_frame = esp_timer_get_time(); + } + + res = httpd_resp_set_type(req, _STREAM_CONTENT_TYPE); + if(res != ESP_OK){ + return res; + } + + ESP_LOGI(TAG, "Get count %d\n", st_face_list.count); + + while(true) + { + // update fsm state + if (g_is_enrolling) + { + g_state = START_ENROLL; + } + else if (g_is_deleting) + { + g_is_deleting = 0; + g_state = START_DELETE; + } + else if (g_state != START_ENROLL) + { + if (st_face_list.count == 0) + g_state = START_DETECT; + else + g_state = START_RECOGNITION; + } + + ESP_LOGD(TAG, "State: %d, head:%d, tail:%d, count:%d", g_state, st_face_list.head, st_face_list.tail, st_face_list.count); + // exec event + if (g_state == START_DELETE) + { + uint8_t left = delete_face_id_in_flash(&st_face_list); + ESP_LOGW(TAG, "%d ID Left", left); + g_state = START_DETECT; + continue; + } + + // Detection Start + fb = esp_camera_fb_get(); + if (!fb) + { + ESP_LOGE(TAG, "Camera capture failed"); + res = ESP_FAIL; + break; + } + + fr_start = esp_timer_get_time(); + fr_ready = fr_start; + fr_face = fr_start; + fr_encode = fr_start; + fr_recognize = fr_start; + image_matrix = dl_matrix3du_alloc(1, fb->width, fb->height, 3); + if (!image_matrix) + { + ESP_LOGE(TAG, "dl_matrix3du_alloc failed"); + res = ESP_FAIL; + break; + } + + if(!fmt2rgb888(fb->buf, fb->len, fb->format, image_matrix->item)) + { + ESP_LOGW(TAG, "fmt2rgb888 failed"); + //res = ESP_FAIL; + //dl_matrix3du_free(image_matrix); + //break; + } + + fr_ready = esp_timer_get_time(); + box_array_t *net_boxes = face_detect(image_matrix, &mtmn_config); + fr_face = esp_timer_get_time(); + // Detection End + + fr_recognize = fr_face; + if (net_boxes) + { + if ((g_state == START_ENROLL || g_state == START_RECOGNITION) + && (align_face(net_boxes, image_matrix, aligned_face) == ESP_OK)) + { + if (g_state == START_ENROLL) + { + rgb_print(image_matrix, FACE_COLOR_YELLOW, "START ENROLLING"); + ESP_LOGD(TAG, "START ENROLLING"); + + int left_sample_face = enroll_face_id_to_flash(&st_face_list, aligned_face); + ESP_LOGD(TAG, "Face ID %d Enrollment: Taken the %d%s sample", + st_face_list.tail, + ENROLL_CONFIRM_TIMES - left_sample_face, + number_suffix(ENROLL_CONFIRM_TIMES - left_sample_face)); + gpio_set_level(GPIO_LED_RED, 0); + rgb_printf(image_matrix, FACE_COLOR_CYAN, "\nThe %u%s sample", + ENROLL_CONFIRM_TIMES - left_sample_face, + number_suffix(ENROLL_CONFIRM_TIMES - left_sample_face)); + + if (left_sample_face == 0) + { + ESP_LOGI(TAG, "Enrolled Face ID: %d", st_face_list.tail); + rgb_printf(image_matrix, FACE_COLOR_CYAN, "\n\nEnrolled Face ID: %d", st_face_list.tail); + g_is_enrolling = 0; + g_state = START_RECOGNITION; + } + } + else + { + face_id = recognize_face(&st_face_list, aligned_face); + + if (face_id >= 0) + { + gpio_set_level(GPIO_LED_RED, 1); + rgb_printf(image_matrix, FACE_COLOR_GREEN, "Hello ID %u", face_id); + } + else + { + rgb_print(image_matrix, FACE_COLOR_RED, "\nWHO?"); + } + } + } + + draw_face_boxes(image_matrix, net_boxes); + free(net_boxes->box); + free(net_boxes->landmark); + free(net_boxes); + + fr_recognize = esp_timer_get_time(); + if(!fmt2jpg(image_matrix->item, fb->width*fb->height*3, fb->width, fb->height, PIXFORMAT_RGB888, 90, &_jpg_buf, &_jpg_buf_len)) + { + ESP_LOGE(TAG, "fmt2jpg failed"); + dl_matrix3du_free(image_matrix); + res = ESP_FAIL; + } + esp_camera_fb_return(fb); + fb = NULL; + } + else + { + _jpg_buf = fb->buf; + _jpg_buf_len = fb->len; + } + dl_matrix3du_free(image_matrix); + fr_encode = esp_timer_get_time(); + + + if(res == ESP_OK){ + size_t hlen = snprintf((char *)part_buf, 64, _STREAM_PART, _jpg_buf_len); + res = httpd_resp_send_chunk(req, (const char *)part_buf, hlen); + } + if(res == ESP_OK){ + res = httpd_resp_send_chunk(req, (const char *)_jpg_buf, _jpg_buf_len); + } + if(res == ESP_OK){ + res = httpd_resp_send_chunk(req, _STREAM_BOUNDARY, strlen(_STREAM_BOUNDARY)); + } + if(fb){ + esp_camera_fb_return(fb); + fb = NULL; + _jpg_buf = NULL; + } else if(_jpg_buf){ + free(_jpg_buf); + _jpg_buf = NULL; + } + if(res != ESP_OK){ + break; + } + int64_t fr_end = esp_timer_get_time(); + + int64_t ready_time = (fr_ready - fr_start)/1000; + int64_t face_time = (fr_face - fr_ready)/1000; + int64_t recognize_time = (fr_recognize - fr_face)/1000; + int64_t encode_time = (fr_encode - fr_recognize)/1000; + int64_t process_time = (fr_encode - fr_start)/1000; + + int64_t frame_time = fr_end - last_frame; + last_frame = fr_end; + frame_time /= 1000; + ESP_LOGD(TAG, "MJPG: %uKB %ums (%.1ffps), %u+%u+%u+%u=%u", + (uint32_t)(_jpg_buf_len/1024), + (uint32_t)frame_time, 1000.0 / (uint32_t)frame_time, + (uint32_t)ready_time, (uint32_t)face_time, (uint32_t)recognize_time, (uint32_t)encode_time, (uint32_t)process_time); + } + + last_frame = 0; + g_state = WAIT_FOR_WAKEUP; + return ESP_OK; +} + +httpd_uri_t _face_stream_handler = { + .uri = "/face_stream", + .method = HTTP_GET, + .handler = facenet_stream_handler, + .user_ctx = NULL +}; + +httpd_handle_t camera_httpd = NULL; + +const esp_timer_create_args_t oneshot_timer_args = { + .callback = &oneshot_timer_callback, + /* argument specified here will be passed to timer callback function */ + .name = "one-shot" +}; +esp_timer_handle_t oneshot_timer; + +static void oneshot_timer_callback(void* arg) +{ + if(g_state != START_ENROLL) + g_is_enrolling = 1; +} + + +static void IRAM_ATTR gpio_isr_handler(void* arg) +{ + esp_err_t err = esp_timer_start_once(oneshot_timer, 500000); + if(err == ESP_ERR_INVALID_STATE) + { + ESP_ERROR_CHECK(esp_timer_stop(oneshot_timer)); + g_is_enrolling = 0; + g_is_deleting = 1; + gpio_set_level(GPIO_LED_WHITE, 0); + } +} + +void app_httpserver_init () +{ + httpd_config_t config = HTTPD_DEFAULT_CONFIG(); + config.stack_size = 4096 * 2; + + ESP_ERROR_CHECK(esp_timer_create(&oneshot_timer_args, &oneshot_timer)); + + gpio_config_t io_conf = {0}; + io_conf.mode = GPIO_MODE_INPUT; + io_conf.intr_type = GPIO_PIN_INTR_POSEDGE; + io_conf.pin_bit_mask = 1LL << GPIO_BUTTON; + io_conf.pull_up_en = GPIO_PULLUP_ENABLE; + gpio_config(&io_conf); + gpio_isr_handler_add(GPIO_BUTTON, gpio_isr_handler, NULL); + + face_id_init(&st_face_list, FACE_ID_SAVE_NUMBER, ENROLL_CONFIRM_TIMES); + aligned_face = dl_matrix3du_alloc(1, FACE_WIDTH, FACE_HEIGHT, 3); + read_face_id_from_flash(&st_face_list); + + if (httpd_start(&camera_httpd, &config) == ESP_OK) + { + httpd_register_uri_handler(camera_httpd, &_face_stream_handler); + } +} + diff --git a/examples/single_chip/recognition_solution/main/app_main.c b/examples/single_chip/recognition_solution/main/app_main.c new file mode 100644 index 0000000..c0f4ae6 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/app_main.c @@ -0,0 +1,110 @@ +/* ESPRESSIF MIT License + * + * Copyright (c) 2018 + * + * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, + * it is free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished + * to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "app_main.h" +#include "esp_partition.h" + +void gpio_led_init() +{ + gpio_config_t gpio_conf; + gpio_conf.mode = GPIO_MODE_OUTPUT; + gpio_conf.pull_up_en = GPIO_PULLUP_ENABLE; + gpio_conf.intr_type = GPIO_INTR_DISABLE; + gpio_conf.pin_bit_mask = 1LL << GPIO_LED_RED; + gpio_config(&gpio_conf); + gpio_conf.pin_bit_mask = 1LL << GPIO_LED_WHITE; + gpio_config(&gpio_conf); + +} + +void led_task(void *arg) +{ + while(1) + { + switch (g_state) + { + case WAIT_FOR_WAKEUP: + gpio_set_level(GPIO_LED_RED, 1); + gpio_set_level(GPIO_LED_WHITE, 0); + break; + + case WAIT_FOR_CONNECT: + gpio_set_level(GPIO_LED_WHITE, 0); + gpio_set_level(GPIO_LED_RED, 1); + vTaskDelay(1000 / portTICK_PERIOD_MS); + gpio_set_level(GPIO_LED_RED, 0); + break; + + case START_DETECT: + case START_RECOGNITION: + gpio_set_level(GPIO_LED_WHITE, 1); + gpio_set_level(GPIO_LED_RED, 0); + break; + + case START_ENROLL: + gpio_set_level(GPIO_LED_WHITE, 1); + gpio_set_level(GPIO_LED_RED, 1); + break; + + case START_DELETE: + gpio_set_level(GPIO_LED_WHITE, 1); + for (int i = 0; i < 3; i++) + { + gpio_set_level(GPIO_LED_RED, 1); + vTaskDelay(200 / portTICK_PERIOD_MS); + gpio_set_level(GPIO_LED_RED, 0); + vTaskDelay(100 / portTICK_PERIOD_MS); + } + break; + + default: + gpio_set_level(GPIO_LED_WHITE, 1); + gpio_set_level(GPIO_LED_RED, 0); + break; + } + vTaskDelay(1000 / portTICK_PERIOD_MS); + } +} + +en_fsm_state g_state = WAIT_FOR_WAKEUP; +int g_is_enrolling = 0; +int g_is_deleting = 0; + +void app_main() +{ + gpio_led_init(); + app_speech_wakeup_init(); + + xTaskCreatePinnedToCore(&led_task, "blink_task", configMINIMAL_STACK_SIZE, NULL, 5, NULL, 0); + + g_state = WAIT_FOR_WAKEUP; + + vTaskDelay(30 / portTICK_PERIOD_MS); + ESP_LOGI("esp-eye", "Please say 'Hi LeXin' to the board"); + ESP_LOGI("esp-eye", "Version "VERSION); + while (g_state == WAIT_FOR_WAKEUP) + vTaskDelay(1000 / portTICK_PERIOD_MS); + app_wifi_init(); + app_camera_init(); + app_httpserver_init(); + ESP_LOGI("esp-eye", "Version "VERSION" success"); +} diff --git a/examples/single_chip/recognition_solution/main/app_speech_recsrc.c b/examples/single_chip/recognition_solution/main/app_speech_recsrc.c new file mode 100644 index 0000000..5ad9466 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/app_speech_recsrc.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/queue.h" +#include "esp_system.h" +#include "esp_spi_flash.h" +#include "xtensa/core-macros.h" +#include "esp_partition.h" +#include "app_speech_srcif.h" +#include "driver/i2s.h" +#include "esp_log.h" +#include "esp_spiffs.h" +#include "app_main.h" + +static void i2s_init(void) +{ + i2s_config_t i2s_config = { + .mode = I2S_MODE_MASTER | I2S_MODE_RX,//the mode must be set according to DSP configuration + .sample_rate = 16000, //must be the same as DSP configuration + .channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT, //must be the same as DSP configuration + .bits_per_sample = 32, //must be the same as DSP configuration + .communication_format = I2S_COMM_FORMAT_I2S, + .dma_buf_count = 3, + .dma_buf_len = 300, + .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2, + }; + i2s_pin_config_t pin_config = { + .bck_io_num = 26, // IIS_SCLK + .ws_io_num = 32, // IIS_LCLK + .data_out_num = -1,// IIS_DSIN + .data_in_num = 33 // IIS_DOUT + }; + i2s_driver_install(1, &i2s_config, 0, NULL); + i2s_set_pin(1, &pin_config); + i2s_zero_dma_buffer(1); +} + +void recsrcTask(void *arg) +{ + i2s_init(); + + src_cfg_t *cfg=(src_cfg_t*)arg; + size_t samp_len = cfg->item_size*2*sizeof(int)/sizeof(int16_t); + + int *samp=malloc(samp_len); + + size_t read_len = 0; + + while(1) { + if (g_state != WAIT_FOR_WAKEUP) + { + vTaskDelay(1000 / portTICK_PERIOD_MS); + continue; + } + + i2s_read(1, samp, samp_len, &read_len, portMAX_DELAY); + for (int x=0; xitem_size/4; x++) { + int s1 = ((samp[x * 4] + samp[x * 4 + 1]) >> 13) & 0x0000FFFF; + int s2 = ((samp[x * 4 + 2] + samp[x * 4 + 3]) << 3) & 0xFFFF0000; + samp[x] = s1 | s2; + } + + xQueueSend(*cfg->queue, samp, portMAX_DELAY); + } + + vTaskDelete(NULL); +} + diff --git a/examples/single_chip/recognition_solution/main/app_speech_wakeup.c b/examples/single_chip/recognition_solution/main/app_speech_wakeup.c new file mode 100644 index 0000000..5eade03 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/app_speech_wakeup.c @@ -0,0 +1,75 @@ +#include +#include +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/queue.h" +#include "freertos/event_groups.h" +#include "esp_system.h" +#include "esp_spi_flash.h" +#include "xtensa/core-macros.h" +#include "esp_partition.h" +#include "app_speech_srcif.h" +#include "sdkconfig.h" +#include "esp_sr_iface.h" +#include "esp_sr_models.h" +#include "app_main.h" + +#define SR_MODEL esp_sr_wakenet3_quantized + +static src_cfg_t srcif; +static const esp_sr_iface_t *model = &SR_MODEL; +static model_iface_data_t *model_data; + +QueueHandle_t sndQueue; + +static void event_wakeup_detected(int r) +{ + assert(g_state == WAIT_FOR_WAKEUP); + printf("%s DETECTED.\n", model->get_word_name(model_data, r)); + g_state = WAIT_FOR_CONNECT; +} + +void nnTask(void *arg) +{ + int audio_chunksize = model->get_samp_chunksize(model_data); + int16_t *buffer=malloc(audio_chunksize*sizeof(int16_t)); + assert(buffer); + + while(1) { + xQueueReceive(sndQueue, buffer, portMAX_DELAY); + + int r=model->detect(model_data, buffer); + if (r) + { + event_wakeup_detected(r); + } + } + + free(buffer); + vTaskDelete(NULL); +} + +void app_speech_wakeup_init() +{ + //Initialize NN model + model_data=model->create(DET_MODE_95); + + wake_word_info_t* word_list = malloc(sizeof(wake_word_info_t)); + esp_err_t ret = model->get_word_list(model_data, word_list); + if (ret == ESP_OK) printf("wake word number = %d, word1 name = %s\n", + word_list->wake_word_num, word_list->wake_word_list[0]); + free(word_list); + + int audio_chunksize=model->get_samp_chunksize(model_data); + + //Initialize sound source + sndQueue=xQueueCreate(2, (audio_chunksize*sizeof(int16_t))); + srcif.queue=&sndQueue; + srcif.item_size=audio_chunksize*sizeof(int16_t); + + xTaskCreatePinnedToCore(&recsrcTask, "rec", 3*1024, (void*)&srcif, 5, NULL, 1); + + xTaskCreatePinnedToCore(&nnTask, "nn", 2*1024, NULL, 5, NULL, 1); +} diff --git a/examples/single_chip/recognition_solution/main/app_wifi.c b/examples/single_chip/recognition_solution/main/app_wifi.c new file mode 100644 index 0000000..ff23455 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/app_wifi.c @@ -0,0 +1,167 @@ +/* ESPRESSIF MIT License + * + * Copyright (c) 2018 + * + * Permission is hereby granted for use on all ESPRESSIF SYSTEMS products, in which case, + * it is free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished + * to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/event_groups.h" +#include "esp_system.h" +#include "esp_wifi.h" +#include "esp_event_loop.h" +#include "esp_log.h" +#include "nvs_flash.h" +#include "sdkconfig.h" + +#include "lwip/err.h" +#include "lwip/sys.h" +#include "app_wifi.h" + +static const char *TAG = "app_wifi"; + +#define EXAMPLE_ESP_WIFI_MODE_AP 1 //TRUE:AP FALSE:STA +#define EXAMPLE_ESP_WIFI_SSID CONFIG_ESP_WIFI_SSID +#define EXAMPLE_ESP_WIFI_PASS CONFIG_ESP_WIFI_PASSWORD +#define EXAMPLE_MAX_STA_CONN CONFIG_MAX_STA_CONN + +static esp_err_t event_handler(void *ctx, system_event_t *event) +{/*{{{*/ + switch(event->event_id) { + case SYSTEM_EVENT_STA_START: + esp_wifi_connect(); + break; + case SYSTEM_EVENT_STA_GOT_IP: + ESP_LOGI(TAG, "got ip:%s", + ip4addr_ntoa(&event->event_info.got_ip.ip_info.ip)); + break; + case SYSTEM_EVENT_AP_STACONNECTED: + ESP_LOGI(TAG, "station:" MACSTR " join, AID=%d", + MAC2STR(event->event_info.sta_connected.mac), + event->event_info.sta_connected.aid); + break; + case SYSTEM_EVENT_AP_STADISCONNECTED: + ESP_LOGI(TAG, "station:" MACSTR " leave, AID=%d", + MAC2STR(event->event_info.sta_disconnected.mac), + event->event_info.sta_disconnected.aid); + + break; + case SYSTEM_EVENT_STA_DISCONNECTED: + esp_wifi_connect(); + break; + default: + break; + } + return ESP_OK; +}/*}}}*/ + +#if EXAMPLE_ESP_WIFI_MODE_AP +static void wifi_init_softap() +{ + tcpip_adapter_init(); + ESP_ERROR_CHECK(esp_event_loop_init(event_handler, NULL)); + + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + + uint8_t mac[6]; + ESP_ERROR_CHECK(esp_wifi_get_mac(ESP_IF_WIFI_AP, mac)); + + wifi_config_t wifi_config; + memset(&wifi_config, 0, sizeof(wifi_config_t)); + if (strlen(EXAMPLE_ESP_WIFI_SSID) == 0) + { + snprintf((char *)wifi_config.ap.ssid, 32, "esp-eye-%x%x", mac[4], mac[5]); + } + else + { + memcpy(wifi_config.ap.ssid, EXAMPLE_ESP_WIFI_SSID, sizeof(EXAMPLE_ESP_WIFI_SSID)); + } + memcpy(wifi_config.ap.password, EXAMPLE_ESP_WIFI_PASS, sizeof(EXAMPLE_ESP_WIFI_PASS)); + wifi_config.ap.ssid_len = strlen(EXAMPLE_ESP_WIFI_SSID); + wifi_config.ap.max_connection = EXAMPLE_MAX_STA_CONN; + wifi_config.ap.authmode = WIFI_AUTH_WPA_WPA2_PSK; + if (strlen(EXAMPLE_ESP_WIFI_PASS) == 0) { + wifi_config.ap.authmode = WIFI_AUTH_OPEN; + } + + esp_wifi_set_ps(WIFI_PS_NONE); + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_AP)); + ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_AP, &wifi_config)); + + ESP_ERROR_CHECK(esp_wifi_start()); + + ESP_LOGI(TAG, "wifi_init_softap finished.SSID:%s password:%s", + wifi_config.ap.ssid, EXAMPLE_ESP_WIFI_PASS); + + char buf[80]; + sprintf(buf, "SSID:%s", wifi_config.ap.ssid); + sprintf(buf, "PASSWORD:%s", wifi_config.ap.password); + +} + +#else + +static void wifi_init_sta() +{ + tcpip_adapter_init(); + ESP_ERROR_CHECK(esp_event_loop_init(event_handler, NULL)); + + wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); + ESP_ERROR_CHECK(esp_wifi_init(&cfg)); + + wifi_config_t wifi_config = {0}; + memset(&wifi_config, 0, sizeof(wifi_config_t)); + memcpy(wifi_config.sta.ssid, EXAMPLE_ESP_WIFI_SSID, sizeof(EXAMPLE_ESP_WIFI_SSID)); + memcpy(wifi_config.sta.password, EXAMPLE_ESP_WIFI_PASS, sizeof(EXAMPLE_ESP_WIFI_PASS)); + + ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA)); + ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config)); + + ESP_ERROR_CHECK(esp_wifi_start()); + + ESP_LOGI(TAG, "connect to ap SSID:%s password:%s", wifi_config.sta.ssid, wifi_config.sta.password); + + char buf[80]; + sprintf(buf, "SSID:%s", wifi_config.sta.ssid); + sprintf(buf, "PASSWORD:%s", wifi_config.sta.password); + +} +#endif + +void app_wifi_init () +{ + //Initialize NVS + esp_err_t ret = nvs_flash_init(); + if (ret == ESP_ERR_NVS_NO_FREE_PAGES) { + ESP_ERROR_CHECK(nvs_flash_erase()); + ret = nvs_flash_init(); + } + ESP_ERROR_CHECK(ret); + +#if EXAMPLE_ESP_WIFI_MODE_AP + ESP_LOGI(TAG, "ESP_WIFI_MODE_AP"); + wifi_init_softap(); +#else + ESP_LOGI(TAG, "ESP_WIFI_MODE_STA"); + wifi_init_sta(); +#endif /*EXAMPLE_ESP_WIFI_MODE_AP*/ + +} + diff --git a/examples/single_chip/recognition_solution/main/component.mk b/examples/single_chip/recognition_solution/main/component.mk new file mode 100644 index 0000000..b31d3af --- /dev/null +++ b/examples/single_chip/recognition_solution/main/component.mk @@ -0,0 +1,9 @@ +# +# Main Makefile. This is basically the same as a component makefile. +# +# This Makefile should, at the very least, just include $(SDK_PATH)/make/component.mk. By default, +# this will take the sources in the src/ directory, compile them and link them into +# lib(subdirectory_name).a in the build directory. This behaviour is entirely configurable, +# please read the SDK documents if you need to do this. +# + diff --git a/examples/single_chip/recognition_solution/main/include/app_camera.h b/examples/single_chip/recognition_solution/main/include/app_camera.h new file mode 100644 index 0000000..22075f5 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/include/app_camera.h @@ -0,0 +1,77 @@ +/* + * ESPRESSIF MIT License + * + * Copyright (c) 2017 + * + * Permission is hereby granted for use on ESPRESSIF SYSTEMS products only, in which case, + * it is free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished + * to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef _APP_CAMERA_H_ +#define _APP_CAMERA_H_ + +#include "esp_log.h" +#include "esp_system.h" +#include "esp_camera.h" + +/** + * PIXFORMAT_RGB565, // 2BPP/RGB565 + * PIXFORMAT_YUV422, // 2BPP/YUV422 + * PIXFORMAT_GRAYSCALE, // 1BPP/GRAYSCALE + * PIXFORMAT_JPEG, // JPEG/COMPRESSED + * PIXFORMAT_RGB888, // 3BPP/RGB888 + */ +#define CAMERA_PIXEL_FORMAT PIXFORMAT_JPEG + +/* + * FRAMESIZE_QQVGA, // 160x120 + * FRAMESIZE_QQVGA2, // 128x160 + * FRAMESIZE_QCIF, // 176x144 + * FRAMESIZE_HQVGA, // 240x176 + * FRAMESIZE_QVGA, // 320x240 + * FRAMESIZE_CIF, // 400x296 + * FRAMESIZE_VGA, // 640x480 + * FRAMESIZE_SVGA, // 800x600 + * FRAMESIZE_XGA, // 1024x768 + * FRAMESIZE_SXGA, // 1280x1024 + * FRAMESIZE_UXGA, // 1600x1200 + */ +#define CAMERA_FRAME_SIZE FRAMESIZE_QVGA + +#define PWDN_GPIO_NUM -1 +#define RESET_GPIO_NUM -1 +#define XCLK_GPIO_NUM 4 +#define SIOD_GPIO_NUM 18 +#define SIOC_GPIO_NUM 23 + +#define Y9_GPIO_NUM 36 +#define Y8_GPIO_NUM 37 +#define Y7_GPIO_NUM 38 +#define Y6_GPIO_NUM 39 +#define Y5_GPIO_NUM 35 +#define Y4_GPIO_NUM 14 +#define Y3_GPIO_NUM 13 +#define Y2_GPIO_NUM 34 +#define VSYNC_GPIO_NUM 5 +#define HREF_GPIO_NUM 27 +#define PCLK_GPIO_NUM 25 + +#define XCLK_FREQ 20000000 + +void app_camera_init(); + +#endif diff --git a/examples/single_chip/recognition_solution/main/include/app_httpserver.h b/examples/single_chip/recognition_solution/main/include/app_httpserver.h new file mode 100644 index 0000000..1bcc4e5 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/include/app_httpserver.h @@ -0,0 +1,42 @@ +/* + * ESPRESSIF MIT License + * + * Copyright (c) 2017 + * + * Permission is hereby granted for use on ESPRESSIF SYSTEMS products only, in which case, + * it is free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished + * to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef _APP_FACENET_H_ +#define _APP_FACENET_H_ + +#if __cplusplus +extern "C" { +#endif + +#include "esp_http_server.h" +#include "app_camera.h" +#include "fd_forward.h" +#include "fr_forward.h" +#include "fr_flash.h" + +void app_httpserver_init (); + +#if __cplusplus +} +#endif +#endif diff --git a/examples/single_chip/recognition_solution/main/include/app_main.h b/examples/single_chip/recognition_solution/main/include/app_main.h new file mode 100644 index 0000000..0bf82b7 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/include/app_main.h @@ -0,0 +1,25 @@ +#include "app_camera.h" +#include "app_httpserver.h" +#include "app_wifi.h" +#include "app_speech_srcif.h" + +#define VERSION "0.9.0" + +#define GPIO_LED_RED 21 +#define GPIO_LED_WHITE 22 +#define GPIO_BUTTON 15 + +typedef enum +{ + WAIT_FOR_WAKEUP, + WAIT_FOR_CONNECT, + START_DETECT, + START_RECOGNITION, + START_ENROLL, + START_DELETE, + +} en_fsm_state; + +extern en_fsm_state g_state; +extern int g_is_enrolling; +extern int g_is_deleting; diff --git a/examples/single_chip/recognition_solution/main/include/app_speech_srcif.h b/examples/single_chip/recognition_solution/main/include/app_speech_srcif.h new file mode 100644 index 0000000..4209b83 --- /dev/null +++ b/examples/single_chip/recognition_solution/main/include/app_speech_srcif.h @@ -0,0 +1,30 @@ +#ifndef SRC_IF_H +#define SRC_IF_H + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +/* +Basic, somewhat quick and dirty, interface for an audio source. The main code uses this to grab samples +from whatever input source it instantiates. Here's how to use it: + +- Create a src_cfg_t variable +- Set item_size to the amount of samples per queue read. +- Set queue to a queue with an item size of (item_size * sizeof(int16_t)) +- Create a task for the worker function of the input method of your choice (e.g. wavsrcTask). Pass the + src_cfg_t variable address as the argument. +- The worker task should now start filling the queue. Receive from it and do with the samples as + you please. + +Note that at the moment all source interfaces are expected to return signed 16-bit samples at an 16KHz +sample rate. +*/ + + +typedef struct { + QueueHandle_t *queue; + int item_size; //in bytes +} src_cfg_t; + +void recsrcTask(void *arg); +void app_speech_wakeup_init(); +#endif diff --git a/examples/single_chip/recognition_solution/main/include/app_wifi.h b/examples/single_chip/recognition_solution/main/include/app_wifi.h new file mode 100644 index 0000000..e0cf53d --- /dev/null +++ b/examples/single_chip/recognition_solution/main/include/app_wifi.h @@ -0,0 +1,6 @@ +#include "freertos/FreeRTOS.h" +#include "freertos/event_groups.h" + +extern EventGroupHandle_t g_wifi_event_group; + +void app_wifi_init(); diff --git a/examples/single_chip/recognition_solution/partitions.csv b/examples/single_chip/recognition_solution/partitions.csv new file mode 100644 index 0000000..8c732ea --- /dev/null +++ b/examples/single_chip/recognition_solution/partitions.csv @@ -0,0 +1,6 @@ +# Espressif ESP32 Partition Table +# Name, Type, SubType, Offset, Size +factory, app, factory, 0x010000, 3M +nvs, data, nvs, 0x310000, 16K +fr, 32, 32, 0x320000, 128K + diff --git a/examples/single_chip/recognition_solution/sdkconfig.defaults b/examples/single_chip/recognition_solution/sdkconfig.defaults new file mode 100644 index 0000000..7fe2217 --- /dev/null +++ b/examples/single_chip/recognition_solution/sdkconfig.defaults @@ -0,0 +1,91 @@ + +CONFIG_ESPTOOLPY_PORT="/dev/ttyUSB0" +CONFIG_ESPTOOLPY_BAUD_115200B= +CONFIG_ESPTOOLPY_BAUD_230400B= +CONFIG_ESPTOOLPY_BAUD_921600B=y +CONFIG_ESPTOOLPY_BAUD_2MB= +CONFIG_ESPTOOLPY_BAUD_OTHER= +CONFIG_ESPTOOLPY_BAUD_OTHER_VAL=115200 +CONFIG_ESPTOOLPY_BAUD=921600 +CONFIG_ESPTOOLPY_COMPRESSED=y +CONFIG_FLASHMODE_QIO=y +CONFIG_FLASHMODE_QOUT= +CONFIG_FLASHMODE_DIO= +CONFIG_FLASHMODE_DOUT= +CONFIG_ESPTOOLPY_FLASHMODE="dio" +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_ESPTOOLPY_FLASHFREQ_40M= +CONFIG_ESPTOOLPY_FLASHFREQ_26M= +CONFIG_ESPTOOLPY_FLASHFREQ_20M= +CONFIG_ESPTOOLPY_FLASHFREQ="80m" +CONFIG_ESPTOOLPY_FLASHSIZE_1MB= +CONFIG_ESPTOOLPY_FLASHSIZE_2MB= +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_ESPTOOLPY_FLASHSIZE_8MB= +CONFIG_ESPTOOLPY_FLASHSIZE_16MB= +CONFIG_ESPTOOLPY_FLASHSIZE="4MB" +CONFIG_ESPTOOLPY_FLASHSIZE_DETECT=y +CONFIG_ESPTOOLPY_BEFORE_RESET=y +CONFIG_ESPTOOLPY_BEFORE_NORESET= +CONFIG_ESPTOOLPY_BEFORE="default_reset" +CONFIG_ESPTOOLPY_AFTER_RESET=y +CONFIG_ESPTOOLPY_AFTER_NORESET= +CONFIG_ESPTOOLPY_AFTER="hard_reset" +CONFIG_MONITOR_BAUD_9600B= +CONFIG_MONITOR_BAUD_57600B= +CONFIG_MONITOR_BAUD_115200B=y +CONFIG_MONITOR_BAUD_230400B= +CONFIG_MONITOR_BAUD_921600B= +CONFIG_MONITOR_BAUD_2MB= +CONFIG_MONITOR_BAUD_OTHER= +CONFIG_MONITOR_BAUD_OTHER_VAL=115200 +CONFIG_MONITOR_BAUD=115200 + +# +# Partition Table +# +CONFIG_PARTITION_TABLE_SINGLE_APP= +CONFIG_PARTITION_TABLE_TWO_OTA= +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_OFFSET=0x8000 +CONFIG_PARTITION_TABLE_MD5=y +# +# Camera configuration +# +CONFIG_ENABLE_TEST_PATTERN= +CONFIG_OV2640_SUPPORT=y +CONFIG_OV7725_SUPPORT= + +# +# ESP32-specific +# +CONFIG_ESP32_DEFAULT_CPU_FREQ_80= +CONFIG_ESP32_DEFAULT_CPU_FREQ_160= +CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y +CONFIG_ESP32_DEFAULT_CPU_FREQ_MHZ=240 +CONFIG_SPIRAM_SUPPORT=y + +# +# SPI RAM config +# +CONFIG_SPIRAM_BOOT_INIT=y +CONFIG_SPIRAM_IGNORE_NOTFOUND= +CONFIG_SPIRAM_USE_MEMMAP= +CONFIG_SPIRAM_USE_CAPS_ALLOC=y +CONFIG_SPIRAM_USE_MALLOC= +CONFIG_SPIRAM_TYPE_AUTO=y +CONFIG_SPIRAM_TYPE_ESPPSRAM32= +CONFIG_SPIRAM_TYPE_ESPPSRAM64= +CONFIG_SPIRAM_SIZE=-1 +CONFIG_SPIRAM_SPEED_40M= +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_MEMTEST=y +CONFIG_SPIRAM_CACHE_WORKAROUND=y +CONFIG_SPIRAM_BANKSWITCH_ENABLE=y +CONFIG_SPIRAM_BANKSWITCH_RESERVE=8 +CONFIG_WIFI_LWIP_ALLOCATION_FROM_SPIRAM_FIRST= +CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY= + +CONFIG_TASK_WDT= diff --git a/examples/single_chip/recognition_with_command_line/main/app_facenet.c b/examples/single_chip/recognition_with_command_line/main/app_facenet.c index 9d169bc..aa881d9 100644 --- a/examples/single_chip/recognition_with_command_line/main/app_facenet.c +++ b/examples/single_chip/recognition_with_command_line/main/app_facenet.c @@ -31,6 +31,9 @@ static const char *TAG = "app_process"; +#define ENROLL_CONFIRM_TIMES 3 +#define FACE_ID_SAVE_NUMBER 1 +static face_id_list id_list = {0}; char *number_suffix(int32_t number) { uint8_t n = number % 10; @@ -76,10 +79,6 @@ void task_process(void *arg) FACE_HEIGHT, 3); - - fptp_t thresh = FACE_REC_THRESHOLD; - dl_matrix3d_t *id_list[FACE_ID_SAVE_NUMBER] = {0}; - int8_t count_down_second = 3; //second int8_t is_enrolling = 1; int32_t next_enroll_index = 0; @@ -131,12 +130,9 @@ void task_process(void *arg) } //enroll - if ((is_enrolling == 1) && (next_enroll_index < FACE_ID_SAVE_NUMBER)) + if (is_enrolling == 1) { - if (id_list[next_enroll_index] == NULL) - id_list[next_enroll_index] = dl_matrix3d_alloc(1, 1, 1, FACE_ID_SIZE); - - left_sample_face = enroll(aligned_face, id_list[next_enroll_index], ENROLL_CONFIRM_TIMES); + left_sample_face = enroll_face(&id_list, aligned_face); ESP_LOGE(TAG, "Face ID Enrollment: Take the %d%s sample", ENROLL_CONFIRM_TIMES - left_sample_face, number_suffix(ENROLL_CONFIRM_TIMES - left_sample_face)); @@ -144,9 +140,9 @@ void task_process(void *arg) if (left_sample_face == 0) { next_enroll_index++; - ESP_LOGE(TAG, "Enrolled Face ID: %d", next_enroll_index); + ESP_LOGE(TAG, "Enrolled Face ID: %d", id_list.tail); - if (next_enroll_index == FACE_ID_SAVE_NUMBER) + if (id_list.count == FACE_ID_SAVE_NUMBER) { is_enrolling = 0; ESP_LOGE(TAG, ">>> Face Recognition Starts <<<"); @@ -163,11 +159,8 @@ void task_process(void *arg) { int64_t recog_match_time = esp_timer_get_time(); - uint16_t matched_id = recognize_face(aligned_face, - id_list, - thresh, - next_enroll_index); - if (matched_id) + int matched_id = recognize_face(&id_list, aligned_face); + if (matched_id >= 0) ESP_LOGE(TAG, "Matched Face ID: %d", matched_id); else ESP_LOGE(TAG, "No Matched Face ID"); @@ -193,5 +186,6 @@ void task_process(void *arg) void app_facenet_main() { + face_id_init(&id_list, FACE_ID_SAVE_NUMBER, ENROLL_CONFIRM_TIMES); xTaskCreatePinnedToCore(task_process, "process", 4 * 1024, NULL, 5, NULL, 1); } diff --git a/examples/single_chip/recognition_with_command_line/sdkconfig.defaults b/examples/single_chip/recognition_with_command_line/sdkconfig.defaults index 21ead3b..96ff546 100644 --- a/examples/single_chip/recognition_with_command_line/sdkconfig.defaults +++ b/examples/single_chip/recognition_with_command_line/sdkconfig.defaults @@ -73,22 +73,19 @@ CONFIG_SPIRAM_SUPPORT=y CONFIG_SPIRAM_BOOT_INIT=y CONFIG_SPIRAM_IGNORE_NOTFOUND= CONFIG_SPIRAM_USE_MEMMAP= -CONFIG_SPIRAM_USE_CAPS_ALLOC= -CONFIG_SPIRAM_USE_MALLOC=y -CONFIG_SPIRAM_TYPE_AUTO= -CONFIG_SPIRAM_TYPE_ESPPSRAM32=y +CONFIG_SPIRAM_USE_CAPS_ALLOC=y +CONFIG_SPIRAM_USE_MALLOC= +CONFIG_SPIRAM_TYPE_AUTO=y +CONFIG_SPIRAM_TYPE_ESPPSRAM32= CONFIG_SPIRAM_TYPE_ESPPSRAM64= -CONFIG_SPIRAM_SIZE=4194304 +CONFIG_SPIRAM_SIZE=-1 CONFIG_SPIRAM_SPEED_40M= CONFIG_SPIRAM_SPEED_80M=y CONFIG_SPIRAM_MEMTEST=y CONFIG_SPIRAM_CACHE_WORKAROUND=y CONFIG_SPIRAM_BANKSWITCH_ENABLE=y CONFIG_SPIRAM_BANKSWITCH_RESERVE=8 -CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 CONFIG_WIFI_LWIP_ALLOCATION_FROM_SPIRAM_FIRST= -CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=32768 -CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY= CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY= CONFIG_TASK_WDT=