yolov5-inferrence/MppDecode.cpp

//
// Created by LX on 2020/4/25.
//

#include "MppDecode.h"
#include "myrga.h"
#include "pose_body.h"
#include "rknn.h"
#include "vertix.h"

extern long max_time_elapsed;

volatile int origin_height, origin_width;
volatile float resize_width_scale;
volatile float resize_height_scale;

pthread_mutex_t frame_mutex = PTHREAD_MUTEX_INITIALIZER;

// rknn_t *rknn = get_rknn("fall_5n.rknn");

/*
int print_messgae(rknn_t *rk, detect_result_group_t *group) {
    printf("got group->count = %d\n", group->count);
    for (int i = 0; i < group->count; i++) {
        printf("%d th result: %s at [%d, %d, %d, %d] with score %f\n", i, group->results[i].name,
            group->results[i].box.left, group->results[i].box.right,
            group->results[i].box.top, group->results[i].box.bottom, group->results[i].prop);
    }

}

int rknn_pose_body(char *out, int width, int height) {
    struct timeval tpend1, tpend2;
    long usec1 = 0;
    gettimeofday(&tpend1, NULL);

    rknn->do_judge(rknn, out, width, height, print_messgae);

    gettimeofday(&tpend2, NULL);
    usec1 = 1000 * (tpend2.tv_sec - tpend1.tv_sec) + (tpend2.tv_usec - tpend1.tv_usec) / 1000;
    // printf("rknn cost_time=%ld ms\n", usec1);

    free(out);
    // printf("in pose_body_thread ok\n");
    pthread_mutex_unlock(&frame_mutex);
}
*/

void set_resize_ratio(int imgwidth, int imgheight)
{
    origin_width = imgwidth;
    origin_height = imgheight;
    resize_height_scale = (float)DSTHeight / (float)imgheight;
    resize_width_scale = (float)DSTWidth / (float)imgwidth;
}

void *pose_body_thread(void *arg)
{
    char *out = (char *)arg;
    // printf("in pose_body_thread\n");

        // process_a_frame(out, DSTWidth, DSTHeight);
    process_a_frame(out, 640, 640);
    // rknn_pose_body(out, DSTWidth, DSTHeight);
}

void convert_a_frame(MppFrame frame)
{
    RK_U32 width = 0;
    RK_U32 height = 0;
    RK_U32 h_stride = 0;
    RK_U32 v_stride = 0;

    MppBuffer buffer = NULL;
    RK_U8 *base = NULL;

    width = mpp_frame_get_width(frame);
    height = mpp_frame_get_height(frame);
    h_stride = mpp_frame_get_hor_stride(frame);
    v_stride = mpp_frame_get_ver_stride(frame);
    buffer = mpp_frame_get_buffer(frame);

    char *out = (char *)malloc(DSTWidth * DSTHeight * 3 * sizeof(char));
    if (out == NULL)
    {
        printf("failed to allocate a frame");
        return;
    }
    base = (RK_U8 *)mpp_buffer_get_ptr(buffer);
    rga_client.convert(base, width, height, h_stride, v_stride, out);
    send_inference_heartbeat_if_needed();
    // printf("convert a frame ok");
    if (pthread_mutex_trylock(&frame_mutex) == 0)
    {
        // lock acquired, should create thread and use pose_body

        if (!should_do_judge_according_time()) {
            if (inited_after_out_of_time_gap == 0) {
                inited_after_out_of_time_gap = 1;
                for (int i = 0; i < OBJ_CLASS_MAX; i++) {
                    pre_change_objects[i] = -1;
                    pre_change_frames[i] = 0;
                }
            }
            pthread_mutex_unlock(&frame_mutex);
            free(out);
            return;
        } else {
            if (inited_after_out_of_time_gap == 1) {
                inited_after_out_of_time_gap = 0;
            }
        }
        pthread_t pose_body_handler;
        pthread_create(&pose_body_handler, NULL, pose_body_thread, (void *)out);
        pthread_detach(pose_body_handler);
    }
    else
    {
        free(out);
    }
    // free(out);
}

int frame_null = 0;
int count = 0;
void dump_mpp_frame_to_file(MppFrame frame, FILE *fp)
{
    RK_U32 width = 0;
    RK_U32 height = 0;
    RK_U32 h_stride = 0;
    RK_U32 v_stride = 0;

    MppBuffer buffer = NULL;
    RK_U8 *base = NULL;

    width = mpp_frame_get_width(frame);
    height = mpp_frame_get_height(frame);
    h_stride = mpp_frame_get_hor_stride(frame);
    v_stride = mpp_frame_get_ver_stride(frame);
    buffer = mpp_frame_get_buffer(frame);

    base = (RK_U8 *)mpp_buffer_get_ptr(buffer);
    RK_U32 buf_size = mpp_frame_get_buf_size(frame);
    size_t base_length = mpp_buffer_get_size(buffer);
    mpp_log("base_length = %d\n", base_length);

    RK_U32 i;
    RK_U8 *base_y = base;
    RK_U8 *base_c = base + h_stride * v_stride;

    // 保存为YUV420sp格式
    /*for (i = 0; i < height; i++, base_y += h_stride)
    {
        fwrite(base_y, 1, width, fp);
    }
    for (i = 0; i < height / 2; i++, base_c += h_stride)
    {
        fwrite(base_c, 1, width, fp);
    }*/

    // 保存为YUV420p格式
    for (i = 0; i < height; i++, base_y += h_stride)
    {
        fwrite(base_y, 1, width, fp);
    }
    for (i = 0; i < height * width / 2; i += 2)
    {
        fwrite((base_c + i), 1, 1, fp);
    }
    for (i = 1; i < height * width / 2; i += 2)
    {
        fwrite((base_c + i), 1, 1, fp);
    }
}

size_t mpp_buffer_group_usage(MppBufferGroup group)
{
    if (NULL == group)
    {
        mpp_err_f("input invalid group %p\n", group);
        return MPP_BUFFER_MODE_BUTT;
    }

    MppBufferGroupImpl *p = (MppBufferGroupImpl *)group;
    return p->usage;
}

void dump_frame_to_file(MppFrame frame)
{
    FILE *out_fp = fopen("single.yuv", "w+b");
    RK_U32 width = 0;
    RK_U32 height = 0;
    RK_U32 h_stride = 0;
    RK_U32 v_stride = 0;
    MppFrameFormat fmt = MPP_FMT_YUV420SP;
    MppBuffer buffer = NULL;
    RK_U8 *base = NULL;

    width = mpp_frame_get_width(frame);
    height = mpp_frame_get_height(frame);
    h_stride = mpp_frame_get_hor_stride(frame);
    v_stride = mpp_frame_get_ver_stride(frame);
    fmt = mpp_frame_get_fmt(frame);
    buffer = mpp_frame_get_buffer(frame);

    RK_U32 buf_size = mpp_frame_get_buf_size(frame);
    printf("w x h: %dx%d hor_stride:%d ver_stride:%d buf_size:%d\n",
           width, height, h_stride, v_stride, buf_size);

    if (NULL == buffer)
    {
        printf("buffer is null\n");
        return;
    }

    base = (RK_U8 *)mpp_buffer_get_ptr(buffer);

    // MPP_FMT_YUV420SP
    if (fmt != MPP_FMT_YUV420SP)
    {
        printf("fmt %d not supported\n", fmt);
        return;
    }

    RK_U32 i;
    RK_U8 *base_y = base;
    RK_U8 *base_c = base + h_stride * v_stride;

    for (i = 0; i < height; i++, base_y += h_stride)
    {
        fwrite(base_y, 1, width, out_fp);
    }
    for (i = 0; i < height / 2; i++, base_c += h_stride)
    {
        fwrite(base_c, 1, width, out_fp);
    }

    fclose(out_fp);
}

int decode_simple(MpiDecLoopData *data, AVPacket *av_packet)
{
    RK_U32 pkt_done = 0;
    RK_U32 pkt_eos = 0;
    RK_U32 err_info = 0;
    MPP_RET ret = MPP_OK;
    MppCtx ctx = data->ctx;
    MppApi *mpi = data->mpi;
    // char   *buf = data->buf;
    MppPacket packet = NULL;
    MppFrame frame = NULL;
    size_t read_size = 0;
    size_t packet_size = data->packet_size;

    /*
    if ((av_packet->data[4] & 0x1f) != 0x01) {
        printf("sending data: %02x %02x[", av_packet->data[4], (av_packet->data[4] & 0x1f));
        for (int i = 0; i < av_packet->size; i++) {
            printf("%02x ", av_packet->data[i]);
        }
        printf("]\n");
    }
    */
    ret = mpp_packet_init(&packet, av_packet->data, av_packet->size);
    mpp_packet_set_pts(packet, av_packet->pts);

    do
    {
        RK_S32 times = 5;
        clock_t start, end;
        // send the packet first if packet is not done
        start = clock();
        if (!pkt_done)
        {
            ret = mpi->decode_put_packet(ctx, packet);
            if (MPP_OK == ret)
                pkt_done = 1;
        }

        // then get all available frame and release
        do
        {
            RK_S32 get_frm = 0;
            RK_U32 frm_eos = 0;

        try_again:
            ret = mpi->decode_get_frame(ctx, &frame);
            if (MPP_ERR_TIMEOUT == ret)
            {
                if (times > 0)
                {
                    times--;
                    msleep(2);
                    goto try_again;
                }
                mpp_err("decode_get_frame failed too much time\n");
            }
            if (MPP_OK != ret)
            {
                mpp_err("decode_get_frame failed ret %d\n", ret);
                break;
            }
            if (frame)
            {
                end = clock();
                if ((end - start) > max_time_elapsed)
                {
                    max_time_elapsed = end - start;
                }
                MppFrameFormat fmt = mpp_frame_get_fmt(frame);

                // printf("vpu cost_time=%f ms\n", (float)(end - start) / CLOCKS_PER_SEC * 1000);
                if (mpp_frame_get_info_change(frame))
                {
                    RK_U32 width = mpp_frame_get_width(frame);
                    RK_U32 height = mpp_frame_get_height(frame);
                    RK_U32 hor_stride = mpp_frame_get_hor_stride(frame);
                    RK_U32 ver_stride = mpp_frame_get_ver_stride(frame);
                    RK_U32 buf_size = mpp_frame_get_buf_size(frame);

                    mpp_log("decode_get_frame get info changed found\n");
                    mpp_log("decoder require buffer w:h [%d:%d] stride [%d:%d] buf_size %d",
                            width, height, hor_stride, ver_stride, buf_size);

                    set_resize_ratio(hor_stride, ver_stride);

                    ret = mpp_buffer_group_get_internal(&data->frm_grp, MPP_BUFFER_TYPE_ION);
                    if (ret)
                    {
                        mpp_err("get mpp buffer group  failed ret %d\n", ret);
                        break;
                    }
                    mpi->control(ctx, MPP_DEC_SET_EXT_BUF_GROUP, data->frm_grp);

                    mpi->control(ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);
                }
                else
                {
                    err_info = mpp_frame_get_errinfo(frame) | mpp_frame_get_discard(frame);
                    if (err_info)
                    {
                        mpp_log("decoder_get_frame get err info:%d discard:%d.\n",
                                mpp_frame_get_errinfo(frame), mpp_frame_get_discard(frame));
                    }
                    data->frame_count++;
                    RK_U32 buf_size = mpp_frame_get_buf_size(frame);
                    // mpp_log("decode_get_frame get frame %d: size: %d\n", data->frame_count, buf_size);
                    if (!err_info)
                    {
                        printf("convert a frame\n");
                        convert_a_frame(frame);
                        // cv::Mat rgbImg;
                        // YUV420SP2Mat(frame, rgbImg);
                        // cv::imwrite("./"+std::to_string(count++)+".jpg", rgbImg);
                        // dump_mpp_frame_to_file(frame, data->fp_output);
                    }
                }
                frm_eos = mpp_frame_get_eos(frame);
                mpp_frame_deinit(&frame);
                frame = NULL;
                get_frm = 1;

                /*
                int locked = pthread_mutex_trylock(&frame_mutex);
                if (locked == 0)
                {
                    // no one is using current frame, replace the global_frame
                    if (global_frame != NULL)
                    {
                        mpp_frame_deinit(&global_frame);
                    }
                    global_frame = frame;
                    pthread_mutex_unlock(&frame_mutex);
                }
                else
                {
                    // destory the frame
                    mpp_frame_deinit(&frame);
                }
                */
            }

            // try get runtime frame memory usage
            if (data->frm_grp)
            {
                size_t usage = mpp_buffer_group_usage(data->frm_grp);
                if (usage > data->max_usage)
                    data->max_usage = usage;
            }

            // if last packet is send but last frame is not found continue
            if (pkt_eos && pkt_done && !frm_eos)
            {
                msleep(10);
                continue;
            }

            if (frm_eos)
            {
                mpp_log("found last frame\n");
                break;
            }

            if (data->frame_num > 0 && data->frame_count >= data->frame_num)
            {
                data->eos = 1;
                break;
            }

            if (get_frm)
                continue;
            break;
        } while (1);

        if (data->frame_num > 0 && data->frame_count >= data->frame_num)
        {
            data->eos = 1;
            mpp_log("reach max frame number %d\n", data->frame_count);
            break;
        }

        if (pkt_done)
            break;

        /*
         * why sleep here:
         * mpi->decode_put_packet will failed when packet in internal queue is
         * full,waiting the package is consumed .Usually hardware decode one
         * frame which resolution is 1080p needs 2 ms,so here we sleep 3ms
         * * is enough.
         */
        msleep(3);
    } while (1);
    mpp_packet_deinit(&packet);
    packet = NULL;

    return ret;
}