#include "rknn.h" #include #include #include "rknn_api.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "MppDecode.h" #include "cJSON.h" #include "pose_body.h" #include "yolov5.h" #include "common.h" #include "postprocess.h" #include "vertix.h" #include "snapshot_notify.h" #include "myrga.h" #ifdef mkdir #undef mkdir #endif void *upload_info(void *arg); typedef struct _upload_target_info_t { int active; int heartbeat; int number; int left; int top; int right; int bottom; int edge_touch; int left_edge; float prop; char boxes[1024]; char rga[256]; } upload_target_info_t; // #define LABEL_NALE_TXT_PATH "./model/coco_80_labels_list.txt" int pre_change_frames[OBJ_CLASS_MAX]; // static int last_objects[OBJ_NUMB_MAX_SIZE] = {0}; int pre_change_objects[OBJ_CLASS_MAX]; static int size_of_last_objects = 0; static int size_of_pre_changed = 0; char *CameraID = NULL; int TypeID = 0; int UploadPort = 0; static char is_uploading = 0; static time_t last_upload = 0; static time_t last_heartbeat_log = 0; static int last_object_count[OBJ_CLASS_MAX] = {0}; static pthread_mutex_t upload_state_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; // rknn_t *rk = NULL; char *global_model = NULL; float conf_threshold; const float nms_threshold = 0.4; static int init = -1; static pthread_mutex_t detect_log_mutex = PTHREAD_MUTEX_INITIALIZER; static FILE *detect_log_fp = NULL; static int detect_log_day = 0; static int detect_log_cleanup_day = 0; static char detect_log_dir[512] = {0}; static char detect_log_script[512] = {0}; static const float LOW_CONF_SNAPSHOT_MIN = 0.30f; static const int LOW_CONF_SNAPSHOT_GAP = 30; static const time_t DETECT_DETAIL_LOG_INTERVAL_SEC = 30; static time_t detect_detail_log_last = 0; static time_t low_conf_snapshot_last[OBJ_CLASS_MAX] = {0}; static bool low_conf_snapshot_missing_logged = false; static long long inference_frame_seq = 0; static int try_reserve_upload() { int reserved = 0; pthread_mutex_lock(&upload_state_mutex); if (is_uploading == 0) { is_uploading = 1; reserved = 1; } pthread_mutex_unlock(&upload_state_mutex); return reserved; } static void release_upload() { pthread_mutex_lock(&upload_state_mutex); is_uploading = 0; pthread_mutex_unlock(&upload_state_mutex); } static int is_edge_touch(int left, int top, int right, int bottom) { return left <= 2 || top <= 2 || right >= DSTWidth - 3 || bottom >= DSTHeight - 3; } static int is_left_edge(int left) { return left <= 2; } static void format_rga_frame_info(char *out, size_t out_size) { rga_frame_info_t info; memset(&info, 0, sizeof(info)); rga_get_last_frame_info(&info); if (info.seq == 0) { snprintf(out, out_size, "rga=unknown"); return; } snprintf(out, out_size, "rga=seq:%llu src:%dx%d stride:%dx%d dst:%dx%d rect:%d,%d,%dx%d", info.seq, info.src_width, info.src_height, info.src_h_stride, info.src_v_stride, info.dst_width, info.dst_height, info.dst_x, info.dst_y, info.dst_rect_width, info.dst_rect_height); } static void append_upload_box(upload_target_info_t *info, int left, int top, int right, int bottom, float prop, const char *label) { if (!info) { return; } if (info->number == 0) { info->left = left; info->top = top; info->right = right; info->bottom = bottom; info->prop = prop; } else { if (left < info->left) info->left = left; if (top < info->top) info->top = top; if (right > info->right) info->right = right; if (bottom > info->bottom) info->bottom = bottom; if (prop > info->prop) info->prop = prop; } info->active = 1; info->edge_touch = info->edge_touch || is_edge_touch(left, top, right, bottom); info->left_edge = info->left_edge || is_left_edge(left); size_t used = strlen(info->boxes); if (used >= sizeof(info->boxes) - 1) { return; } snprintf(info->boxes + used, sizeof(info->boxes) - used, "%s%s(%d,%d,%d,%d,%.3f)", used > 0 ? ";" : "", label ? label : "", left, top, right, bottom, prop); } static void merge_upload_box_info(upload_target_info_t *dst, const upload_target_info_t *src) { if (!dst || !src || src->number <= 0) { return; } if (dst->number == 0) { dst->left = src->left; dst->top = src->top; dst->right = src->right; dst->bottom = src->bottom; dst->prop = src->prop; } else { if (src->left < dst->left) dst->left = src->left; if (src->top < dst->top) dst->top = src->top; if (src->right > dst->right) dst->right = src->right; if (src->bottom > dst->bottom) dst->bottom = src->bottom; if (src->prop > dst->prop) dst->prop = src->prop; } dst->edge_touch = dst->edge_touch || src->edge_touch; dst->left_edge = dst->left_edge || src->left_edge; if (dst->rga[0] == '\0' && src->rga[0] != '\0') { snprintf(dst->rga, sizeof(dst->rga), "%s", src->rga); } size_t used = strlen(dst->boxes); size_t src_len = strlen(src->boxes); if (src_len > 0 && used < sizeof(dst->boxes) - 1) { snprintf(dst->boxes + used, sizeof(dst->boxes) - used, "%s%s", used > 0 ? ";" : "", src->boxes); } } static void get_executable_dir(char *out, size_t out_size) { ssize_t len = readlink("/proc/self/exe", out, out_size - 1); if (len > 0) { out[len] = '\0'; char *slash = strrchr(out, '/'); if (slash) { *slash = '\0'; return; } } snprintf(out, out_size, "."); } static int get_today_int(struct tm *out_tm) { time_t now = time(NULL); struct tm t; localtime_r(&now, &t); if (out_tm) { *out_tm = t; } return (t.tm_year + 1900) * 10000 + (t.tm_mon + 1) * 100 + t.tm_mday; } static void format_now(char *out, size_t out_size, time_t *out_ts) { time_t now = time(NULL); struct tm t; localtime_r(&now, &t); strftime(out, out_size, "%Y-%m-%d %H:%M:%S", &t); if (out_ts) { *out_ts = now; } } static void cleanup_old_detect_logs() { DIR *dir = opendir(detect_log_dir); if (!dir) { return; } time_t cutoff = time(NULL) - 7 * 24 * 3600; struct dirent *entry; while ((entry = readdir(dir)) != NULL) { int y, m, d; if (sscanf(entry->d_name, "%d-%d-%d-yolov5.log", &y, &m, &d) != 3) { continue; } struct tm t; memset(&t, 0, sizeof(t)); t.tm_year = y - 1900; t.tm_mon = m - 1; t.tm_mday = d; time_t file_time = mktime(&t); if (file_time != (time_t)-1 && file_time < cutoff) { char path[1024]; snprintf(path, sizeof(path), "%s/%s", detect_log_dir, entry->d_name); remove(path); } } closedir(dir); } static int ensure_detect_log_open() { struct tm t; int today = get_today_int(&t); if (detect_log_fp && detect_log_day == today) { return 0; } if (detect_log_fp) { fclose(detect_log_fp); detect_log_fp = NULL; } if (detect_log_dir[0] == '\0') { char exe_dir[512]; get_executable_dir(exe_dir, sizeof(exe_dir)); snprintf(detect_log_dir, sizeof(detect_log_dir), "%s/.log", exe_dir); snprintf(detect_log_script, sizeof(detect_log_script), "%s/low_conf_snapshot.sh", exe_dir); mkdir(detect_log_dir, 0755); } if (detect_log_cleanup_day != today) { cleanup_old_detect_logs(); detect_log_cleanup_day = today; } char path[1024]; snprintf(path, sizeof(path), "%s/%04d-%02d-%02d-yolov5.log", detect_log_dir, t.tm_year + 1900, t.tm_mon + 1, t.tm_mday); detect_log_fp = fopen(path, "a+"); if (!detect_log_fp) { return -1; } detect_log_day = today; return 0; } static void detect_log_line(const char *fmt, ...) { pthread_mutex_lock(&detect_log_mutex); if (ensure_detect_log_open() == 0) { va_list ap; va_start(ap, fmt); vfprintf(detect_log_fp, fmt, ap); va_end(ap); fprintf(detect_log_fp, "\n"); fflush(detect_log_fp); } pthread_mutex_unlock(&detect_log_mutex); } static std::string shell_quote(const char *s) { std::string out = "'"; if (s) { for (const char *p = s; *p; ++p) { if (*p == '\'') { out += "'\\''"; } else { out += *p; } } } out += "'"; return out; } static void maybe_trigger_low_conf_snapshot(const char *label, float prop, int left, int top, int right, int bottom) { if (prop < LOW_CONF_SNAPSHOT_MIN || prop >= conf_threshold) { return; } time_t now = time(NULL); int cls_id = -1; for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { if (labels[i] == label) { cls_id = i; break; } } if (cls_id >= 0 && now - low_conf_snapshot_last[cls_id] < LOW_CONF_SNAPSHOT_GAP) { return; } if (detect_log_script[0] == '\0') { ensure_detect_log_open(); } if (access(detect_log_script, X_OK) != 0) { if (!low_conf_snapshot_missing_logged) { detect_log_line("%ld camera=%s snapshot skipped: script not executable: %s", now, CameraID ? CameraID : "", detect_log_script); low_conf_snapshot_missing_logged = true; } return; } if (cls_id >= 0) { low_conf_snapshot_last[cls_id] = now; } char ts[32], conf[32], l[16], t[16], r[16], b[16]; snprintf(ts, sizeof(ts), "%ld", now); snprintf(conf, sizeof(conf), "%f", prop); snprintf(l, sizeof(l), "%d", left); snprintf(t, sizeof(t), "%d", top); snprintf(r, sizeof(r), "%d", right); snprintf(b, sizeof(b), "%d", bottom); std::string cmd; cmd += shell_quote(detect_log_script); cmd += " "; cmd += shell_quote(server); cmd += " "; cmd += shell_quote(CameraID); cmd += " "; cmd += shell_quote(ts); cmd += " "; cmd += shell_quote(label); cmd += " "; cmd += shell_quote(conf); cmd += " "; cmd += shell_quote(l); cmd += " "; cmd += shell_quote(t); cmd += " "; cmd += shell_quote(r); cmd += " "; cmd += shell_quote(b); cmd += " 2>&1"; FILE *pipe = popen(cmd.c_str(), "r"); if (!pipe) { detect_log_line("%ld camera=%s snapshot failed: popen errno=%d", now, CameraID ? CameraID : "", errno); return; } char line[1024]; while (fgets(line, sizeof(line), pipe) != NULL) { line[strcspn(line, "\r\n")] = '\0'; detect_log_line("%ld camera=%s snapshot %s", now, CameraID ? CameraID : "", line); } int ret = pclose(pipe); if (ret != 0) { detect_log_line("%ld camera=%s snapshot command exit=%d", now, CameraID ? CameraID : "", ret); } } #ifdef DUMP_FIRST_FRAME #include "rga.h" volatile char save_flag = 1; #endif int process_a_frame(char *buf, int sw, int sh) { // pthread_mutex_lock(&frame_mutex); // int ret = pose_body(buf, sw, sh, 0); struct timeval tpend1, tpend2; long usec1 = 0; gettimeofday(&tpend1, NULL); int ret; if (pthread_mutex_trylock(&mutex) == 0) { image_buffer_t src_img = {0}; src_img.format = IMAGE_FORMAT_RGB888; src_img.height = sh; src_img.width = sw; src_img.height_stride = 0; src_img.width_stride = 0; src_img.virt_addr = (unsigned char *)buf; src_img.size = sw * sh * 3; object_detect_result_list od_results; memset(&od_results, 0, sizeof(object_detect_result_list)); ret = inference_yolov5_model(&rknn_app_ctx, &src_img, &od_results); if (rknn_app_ctx.callback != NULL) { rknn_app_ctx.callback(&rknn_app_ctx, &od_results, (unsigned char*)buf); } else { free(buf); } // printf("detect result: %d, %s\n", od_results.count, coco_cls_to_name(0)); /* for (int i = 0; i < od_results.count; i++) { object_detect_result *det_result = &(od_results.results[i]); printf("(%d)[%s] @ (%d %d %d %d) %.3f\n", det_result->cls_id, coco_cls_to_name(det_result->cls_id), det_result->box.left, det_result->box.top, det_result->box.right, det_result->box.bottom, det_result->prop); } */ pthread_mutex_unlock(&mutex); } else { ret = -1; free(buf); printf("skipped judge\n"); } // free(buf); gettimeofday(&tpend2, NULL); usec1 = 1000 * (tpend2.tv_sec - tpend1.tv_sec) + (tpend2.tv_usec - tpend1.tv_usec) / 1000; printf("rknn cost_time=%ld ms\n", usec1); pthread_mutex_unlock(&frame_mutex); return ret; } int print_object_detect_result(rknn_app_context_t *rknn, object_detect_result_list *group, unsigned char *buff) { upload_target_info_t need_upload_info[OBJ_CLASS_MAX]; upload_target_info_t current_objects[OBJ_CLASS_MAX]; char need_upload = 0; char positive_seen = 0; for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { memset(&need_upload_info[i], 0, sizeof(upload_target_info_t)); memset(¤t_objects[i], 0, sizeof(upload_target_info_t)); } long long frame_seq = ++inference_frame_seq; char now_text[32]; time_t now_ts; format_now(now_text, sizeof(now_text), &now_ts); bool should_log_detail = false; if (detect_detail_log_last == 0 || now_ts - detect_detail_log_last >= DETECT_DETAIL_LOG_INTERVAL_SEC) { detect_detail_log_last = now_ts; should_log_detail = true; } if (should_log_detail) { detect_log_line("%s ts=%ld camera=%s frame_seen frame=%lld raw_count=%d", now_text, now_ts, CameraID ? CameraID : "", frame_seq, group->count); } for (int i = 0; i < group->count; i++) { if (group->results[i].cls_id >= OBJ_CLASS_REAL_NUM) { printf("skip clsid > OBJ_CLASS_NUM: %d\n", group->results[i].cls_id); continue; } char *label = labels[group->results[i].cls_id]; if (group->results[i].prop < conf_threshold) { printf("skip prop = %f for class %s\n", group->results[i].prop, label); if (should_log_detail) { int low_left = group->results[i].box.left; int low_top = group->results[i].box.top; int low_right = group->results[i].box.right; int low_bottom = group->results[i].box.bottom; if (group->results[i].prop >= LOW_CONF_SNAPSHOT_MIN && is_point_in_workspace(low_left, low_top, low_right, low_bottom)) { detect_log_line("%s ts=%ld camera=%s weak_present frame=%lld result=%2d code=%d box=(%3d %3d %3d %3d) prop=%f thresh=%f label=%s", now_text, now_ts, CameraID ? CameraID : "", frame_seq, i, codes[group->results[i].cls_id], low_left, low_top, low_right, low_bottom, group->results[i].prop, conf_threshold, label); } else { detect_log_line("%s ts=%ld camera=%s filtered_low_conf frame=%lld result=%2d code=%d box=(%3d %3d %3d %3d) prop=%f min=%f label=%s", now_text, now_ts, CameraID ? CameraID : "", frame_seq, i, codes[group->results[i].cls_id], low_left, low_top, low_right, low_bottom, group->results[i].prop, LOW_CONF_SNAPSHOT_MIN, label); } } continue; } int left = group->results[i].box.left; int top = group->results[i].box.top; int right = group->results[i].box.right; int bottom = group->results[i].box.bottom; if (!is_point_in_workspace(left, top, right, bottom)) { printf("not in "); if (should_log_detail) { detect_log_line("%s ts=%ld camera=%s filtered_workspace frame=%lld result=%2d code=%d box=(%3d %3d %3d %3d) prop=%f label=%s", now_text, now_ts, CameraID ? CameraID : "", frame_seq, i, codes[group->results[i].cls_id], left, top, right, bottom, group->results[i].prop, label); } printf("result %2d: (%3d %3d %3d %3d) %f, %s\n", i, left, top, right, bottom, group->results[i].prop, label); continue; } if (group->results[i].prop >= conf_threshold) { printf("in "); if (should_log_detail) { detect_log_line("%s ts=%ld camera=%s strong_present frame=%lld result=%2d code=%d box=(%3d %3d %3d %3d) prop=%f thresh=%f label=%s", now_text, now_ts, CameraID ? CameraID : "", frame_seq, i, codes[group->results[i].cls_id], left, top, right, bottom, group->results[i].prop, conf_threshold, label); } append_upload_box(¤t_objects[group->results[i].cls_id], left, top, right, bottom, group->results[i].prop, label); current_objects[group->results[i].cls_id].number += 1; } printf("result %2d: (%3d %3d %3d %3d) %f, %s\n", i, left, top, right, bottom, group->results[i].prop, label); } for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { if (current_objects[i].number <= 0 || codes[i] == 0) { continue; } for (int j = 0; j < i; j++) { if (current_objects[j].number > 0 && codes[j] == codes[i]) { detect_log_line("%s ts=%ld camera=%s merge_same_code code=%d from_label=%s from_number=%d from_boxes=%s to_label=%s", now_text, now_ts, CameraID ? CameraID : "", codes[i], labels[i] ? labels[i] : "", current_objects[i].number, current_objects[i].boxes, labels[j] ? labels[j] : ""); current_objects[j].number += current_objects[i].number; merge_upload_box_info(¤t_objects[j], ¤t_objects[i]); memset(¤t_objects[i], 0, sizeof(upload_target_info_t)); break; } } } for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { if (current_objects[i].number <= 0) { continue; } positive_seen = 1; if (codes[i] == 0) { detect_log_line("%s ts=%ld camera=%s upload_skip_code0 frame=%lld label=%s number=%d prop=%.3f box=(%d,%d,%d,%d) boxes=%s", now_text, now_ts, CameraID ? CameraID : "", frame_seq, labels[i] ? labels[i] : "", current_objects[i].number, current_objects[i].prop, current_objects[i].left, current_objects[i].top, current_objects[i].right, current_objects[i].bottom, current_objects[i].boxes); continue; } need_upload = 1; format_rga_frame_info(current_objects[i].rga, sizeof(current_objects[i].rga)); need_upload_info[i] = current_objects[i]; last_object_count[i] = current_objects[i].number; snapshot_notify_event_t snapshot_event; memset(&snapshot_event, 0, sizeof(snapshot_event)); snapshot_event.camera_id = CameraID ? CameraID : ""; snapshot_event.rtsp = server ? server : ""; snapshot_event.ts = now_ts; snapshot_event.frame = frame_seq; snapshot_event.ccid = codes[i]; snapshot_event.label = labels[i] ? labels[i] : ""; snapshot_event.confidence = current_objects[i].prop; snapshot_event.box_left = current_objects[i].left; snapshot_event.box_top = current_objects[i].top; snapshot_event.box_right = current_objects[i].right; snapshot_event.box_bottom = current_objects[i].bottom; char snapshot_reason[128] = {0}; int snapshot_ret = snapshot_notify_send(&snapshot_event, snapshot_reason, sizeof(snapshot_reason)); detect_log_line("%s ts=%ld camera=%s upload_positive frame=%lld code=%d label=%s number=%d prop=%.3f box=(%d,%d,%d,%d) edge_touch=%d left_edge=%d boxes=%s %s snapshot_notify=%s reason=%s", now_text, now_ts, CameraID ? CameraID : "", frame_seq, codes[i], labels[i] ? labels[i] : "", current_objects[i].number, current_objects[i].prop, current_objects[i].left, current_objects[i].top, current_objects[i].right, current_objects[i].bottom, current_objects[i].edge_touch, current_objects[i].left_edge, current_objects[i].boxes, current_objects[i].rga, snapshot_ret == 0 ? "ok" : "failed", snapshot_reason[0] ? snapshot_reason : "unknown"); } time_t now_timestamp; time(&now_timestamp); if (!need_upload && now_timestamp - last_heartbeat_log >= HEARTBEAT_SECS) { last_heartbeat_log = now_timestamp; for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { if (codes[i] != 0) { detect_log_line("%s ts=%ld camera=%s heartbeat_log_only frame=%lld code=%d label=%s number=0 upload=0", now_text, now_ts, CameraID ? CameraID : "", frame_seq, codes[i], labels[i] ? labels[i] : ""); break; } } } if (need_upload && try_reserve_upload()) { time(&last_upload); upload_target_info_t *need_upload_copy = new upload_target_info_t[OBJ_CLASS_REAL_NUM]; for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { need_upload_copy[i] = need_upload_info[i]; } pthread_t uploadth; if (pthread_create(&uploadth, NULL, upload_info, (void *)need_upload_copy) == 0) { pthread_detach(uploadth); } else { detect_log_line("%s ts=%ld camera=%s upload_thread_create_failed frame=%lld", now_text, now_ts, CameraID ? CameraID : "", frame_seq); delete[] need_upload_copy; release_upload(); } } else if (need_upload) { detect_log_line("%s ts=%ld camera=%s upload_skip_busy frame=%lld", now_text, now_ts, CameraID ? CameraID : "", frame_seq); } else if (!positive_seen && should_log_detail) { detect_log_line("%s ts=%ld camera=%s no_target frame=%lld", now_text, now_ts, CameraID ? CameraID : "", frame_seq); } free(buff); } void *upload_info(void *arg) { upload_target_info_t *need_upload_info = (upload_target_info_t *)arg; char need_uploading = 0; cJSON *array = cJSON_CreateArray(); for (int i = 0; i < OBJ_CLASS_REAL_NUM; i++) { if (need_upload_info[i].active && codes[i] != 0) { int total_number = need_upload_info[i].number; upload_target_info_t merged_info = need_upload_info[i]; for (int j = i+1; j < OBJ_CLASS_REAL_NUM; j++) { if (codes[i] == codes[j] && need_upload_info[j].active) { printf("merge %s(No: %d) to %s", labels[j], need_upload_info[j].number, labels[i]); total_number += need_upload_info[j].number; merge_upload_box_info(&merged_info, &need_upload_info[j]); memset(&need_upload_info[j], 0, sizeof(upload_target_info_t)); } } if (total_number <= 0) { detect_log_line("upload_skip_nonpositive camera=%s code=%d label=%s number=%d heartbeat=%d", CameraID ? CameraID : "", codes[i], labels[i] ? labels[i] : "", total_number, merged_info.heartbeat); continue; } need_uploading = 1; printf("xxxxxxx class idx = %d\n", codes[i]); detect_log_line("%s camera=%s code=%d label=%s number=%d prop=%.3f box=(%d,%d,%d,%d) edge_touch=%d left_edge=%d boxes=%s %s", merged_info.heartbeat ? "heartbeat_payload" : "upload_positive_payload", CameraID ? CameraID : "", codes[i], labels[i] ? labels[i] : "", total_number, merged_info.prop, merged_info.left, merged_info.top, merged_info.right, merged_info.bottom, merged_info.edge_touch, merged_info.left_edge, merged_info.boxes, merged_info.rga[0] ? merged_info.rga : "rga=none"); cJSON *obj = cJSON_CreateObject(); // 事件编码,外部事件编�? cJSON_AddNumberToObject(obj, "class_idx", codes[i]); cJSON_AddStringToObject(obj, "name", labels[i]); cJSON_AddNumberToObject(obj, "number", total_number); cJSON_AddItemToArray(array, obj); // do upload } } if (!need_uploading) { cJSON_Delete(array); delete[] need_upload_info; release_upload(); printf("no need to upload\n"); return NULL; } cJSON *uploadinfo = cJSON_CreateObject(); cJSON_AddStringToObject(uploadinfo, "serial", CameraID); // 内部事件编码,因为没法表示多个,所以,即将废弃 cJSON_AddNumberToObject(uploadinfo, "type", TypeID); cJSON_AddNumberToObject(uploadinfo, "at", current_time_stamp()); cJSON_AddItemToObject(uploadinfo, "params", array); time_t now; now = time(NULL); post_json("127.0.0.1", UploadPort, "localhost", "/video/post", uploadinfo); cJSON_Delete(uploadinfo); delete[] need_upload_info; // int resgap = now % TIMEGAP; // printf("sleep for %d seconds", (2 * TIMEGAP - resgap)); // sleep(2 * (TIMEGAP - resgap)); release_upload(); }