esp-who/examples/camera_terminal/main/app_dl.cpp

189 lines
5.6 KiB
C++
Raw Normal View History

2021-07-29 20:48:55 +08:00
#include "app_dl.h"
2021-07-16 16:20:24 +08:00
#include "esp_log.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "sdkconfig.h"
2021-07-29 20:48:55 +08:00
#include "app_camera.hpp"
2021-07-16 16:20:24 +08:00
#include <list>
#include "dl_tool.hpp"
#include "dl_image.hpp"
#include "dl_detect_define.hpp"
2021-07-29 20:48:55 +08:00
#include "app_common.hpp"
2021-07-16 16:20:24 +08:00
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_DETECTION_S1_MSR01
2021-07-16 16:20:24 +08:00
#include "human_face_detect_msr01.hpp"
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_DETECTION_S2_MNP01
2021-07-16 16:20:24 +08:00
#include "human_face_detect_mnp01.hpp"
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_CAT_FACE_DETECTION_MN03
2021-07-16 16:20:24 +08:00
#include "cat_face_detect_mn03.hpp"
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_RECOGNITION_XXX
2021-07-16 16:20:24 +08:00
// TODO: recognize human face
#endif
static const char *TAG = "app_dl";
2021-07-29 20:48:55 +08:00
2021-07-16 16:20:24 +08:00
void task_dl(void *arg)
{
dl::tool::Latency latency_total(24);
dl::tool::Latency latency_fetch;
dl::tool::Latency latency_decode;
dl::tool::Latency latency_detect;
2021-07-29 20:48:55 +08:00
dl::tool::Latency latency_recognize;
2021-08-03 16:11:34 +08:00
dl::tool::Latency latency_moving;
2021-07-16 16:20:24 +08:00
/* 1. Load configuration for detection */
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE
#if CONFIG_DL_HUMAN_FACE_DETECTION_S1_MSR01
2021-07-16 16:20:24 +08:00
HumanFaceDetectMSR01 detector(0.3F, 0.3F, 10, 0.3F);
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_DETECTION_S2_MNP01
2021-07-16 16:20:24 +08:00
HumanFaceDetectMNP01 detector2(0.4F, 0.3F, 10);
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_RECOGNITION_XXX
// TODO: recognize human face
#endif
2021-07-16 16:20:24 +08:00
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_CAT_FACE
#if CONFIG_DL_CAT_FACE_DETECTION_MN03
CatFaceDetectMN03 detector(0.4F, 0.3F, 10, 0.3F);
#endif
2021-07-16 16:20:24 +08:00
#endif
while (true)
{
2021-07-29 20:48:55 +08:00
latency_fetch.clear_period();
latency_decode.clear_period();
latency_detect.clear_period();
latency_recognize.clear_period();
2021-08-03 16:11:34 +08:00
latency_moving.clear_period();
2021-07-29 20:48:55 +08:00
2021-07-16 16:20:24 +08:00
latency_total.start();
latency_fetch.start();
2021-08-03 16:11:34 +08:00
camera_fb_t *frame = esp_camera_fb_get();
if (!frame)
2021-07-16 16:20:24 +08:00
{
ESP_LOGE(TAG, "Camera capture failed");
continue;
}
latency_fetch.end();
2021-08-03 16:11:34 +08:00
#if CONFIG_DL_MOVING_TARGET_DETECTION_ENABLED && CONFIG_CAMERA_PIXEL_FORMAT_RGB565
camera_fb_t *frame2 = esp_camera_fb_get();
if (!frame2)
{
ESP_LOGE(TAG, "Camera capture failed");
continue;
}
latency_fetch.end();
latency_moving.start();
uint32_t moving_point_number = dl::image::get_moving_point_number((uint16_t *)frame->buf, (uint16_t *)frame2->buf, frame->height, frame->width, 8, 15);
latency_moving.end();
if (moving_point_number > 50)
{
ESP_LOGI("Moving Target", "Detected.");
dl::image::draw_filled_rectangle((uint16_t *)frame->buf, frame->height, frame->width, 0, 0, 10, 10);
}
esp_camera_fb_return(frame2);
#endif
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_ENABLED
2021-07-16 16:20:24 +08:00
latency_decode.start();
2021-08-03 16:11:34 +08:00
IMAGE_T *image_ptr = (IMAGE_T *)app_camera_decode(frame);
if (!image_ptr)
2021-07-16 16:20:24 +08:00
{
2021-08-03 16:11:34 +08:00
esp_camera_fb_return(frame);
2021-07-16 16:20:24 +08:00
continue;
}
2021-08-03 16:11:34 +08:00
int image_height = frame->height;
int image_width = frame->width;
pixformat_t image_format = frame->format;
if (image_format != PIXFORMAT_RGB565)
2021-08-03 16:11:34 +08:00
esp_camera_fb_return(frame);
2021-07-16 16:20:24 +08:00
latency_decode.end();
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE
2021-07-16 16:20:24 +08:00
latency_detect.start();
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_DETECTION_S2_ENABLED
std::list<dl::detect::result_t> &detect_candidates = detector.infer(image_ptr, {(int)image_height, (int)image_width, 3});
std::list<dl::detect::result_t> &detect_results = detector2.infer(image_ptr, {(int)image_height, (int)image_width, 3}, detect_candidates);
2021-07-16 16:20:24 +08:00
#else
std::list<dl::detect::result_t> &detect_results = detector.infer(image_ptr, {(int)image_height, (int)image_width, 3});
2021-07-16 16:20:24 +08:00
#endif
latency_detect.end();
2021-07-29 20:48:55 +08:00
if (detect_results.size() > 0)
2021-07-16 16:20:24 +08:00
{
2021-07-29 20:48:55 +08:00
print_detection_result(detect_results);
2021-07-16 16:20:24 +08:00
2021-07-29 20:48:55 +08:00
#if CONFIG_DL_HUMAN_FACE_RECOGNITION_ENABLED
2021-07-16 16:20:24 +08:00
latency_recognize.start();
2021-07-29 20:48:55 +08:00
// TODO: recognize human face
latency_recognize.end();
2021-07-16 16:20:24 +08:00
#endif
}
2021-07-29 20:48:55 +08:00
#endif // CONFIG_DL_HUMAN_FACE
#if CONFIG_DL_CAT_FACE
latency_detect.start();
std::list<dl::detect::result_t> &detect_results = detector.infer(image_ptr, {(int)image_height, (int)image_width, 3});
2021-07-29 20:48:55 +08:00
latency_detect.end();
if (detect_results.size() > 0)
{
print_detection_result(detect_results);
2021-07-29 20:48:55 +08:00
}
#endif // CONFIG_DL_CAT_FACE
#if CONFIG_DL_HUMAN_HAND
latency_detect.start();
// TODO:
latency_detect.end();
#endif // CONFIG_DL_HUMAN_HAND
if (image_format == PIXFORMAT_RGB565)
2021-08-03 16:11:34 +08:00
esp_camera_fb_return(frame);
else
free(image_ptr);
2021-07-16 16:20:24 +08:00
#else
2021-08-03 16:11:34 +08:00
esp_camera_fb_return(frame);
2021-07-29 20:48:55 +08:00
#endif // CONFIG_DL_ENABLED
latency_total.end();
2021-07-16 16:20:24 +08:00
uint32_t frame_latency = latency_total.get_period() / 1000;
uint32_t average_frame_latency = latency_total.get_average_period() / 1000;
2021-08-03 16:11:34 +08:00
ESP_LOGI("Frame Latency", "%4ums (%2.1ffps), Average: %4ums (%2.1ffps) | fetch: %4ums, "
#if CONFIG_DL_MOVING_TARGET_DETECTION_ENABLED && CONFIG_CAMERA_PIXEL_FORMAT_RGB565
"moving: %4uus, "
#endif
"decode: %4ums, detect: %4ums, recognize: %5ums",
2021-07-16 16:20:24 +08:00
frame_latency, 1000.0 / frame_latency, average_frame_latency, 1000.0 / average_frame_latency,
latency_fetch.get_period() / 1000,
2021-08-03 16:11:34 +08:00
#if CONFIG_DL_MOVING_TARGET_DETECTION_ENABLED && CONFIG_CAMERA_PIXEL_FORMAT_RGB565
latency_moving.get_period(),
#endif
2021-07-16 16:20:24 +08:00
latency_decode.get_period() / 1000,
2021-07-29 20:48:55 +08:00
latency_detect.get_period() / 1000,
latency_recognize.get_period() / 1000);
2021-07-16 16:20:24 +08:00
}
}
void app_dl_init()
{
xTaskCreatePinnedToCore(task_dl, "dl", 4 * 1024, NULL, 5, NULL, 1);
}