Add v4l2m2m sim encoder

Signed-off-by: shizhenghui <shizhenghui@xiaomi.com>
This commit is contained in:
shizhenghui 2024-02-05 21:10:16 +08:00 committed by Xiang Xiao
parent ed1ad1be65
commit 932cdf228b
8 changed files with 818 additions and 0 deletions

View File

@ -343,6 +343,16 @@ config SIM_VIDEO_DECODER_DEV_PATH
depends on SIM_VIDEO_DECODER
default "/dev/video1"
config SIM_VIDEO_ENCODER
bool "Video encoder support on sim"
depends on VIDEO
default n
config SIM_VIDEO_ENCODER_DEV_PATH
string "Video encoder device path"
depends on SIM_VIDEO_ENCODER
default "/dev/video2"
endmenu
menu "Simulated Graphics/Input"

View File

@ -256,6 +256,12 @@ ifeq ($(CONFIG_SIM_VIDEO_DECODER),y)
STDLIBS += -lopenh264
endif
ifeq ($(CONFIG_SIM_VIDEO_ENCODER),y)
HOSTSRCS += sim_hostencoder.c
CSRCS += sim_encoder.c
STDLIBS += -lx264
endif
COBJS = $(CSRCS:.c=$(OBJEXT))
NUTTXOBJS = $(AOBJS) $(COBJS)

View File

@ -161,6 +161,12 @@ if(CONFIG_SIM_VIDEO_DECODER)
list(APPEND STDLIBS openh264)
endif()
if(CONFIG_SIM_VIDEO_ENCODER)
list(APPEND HOSTSRCS sim_hostencoder.c)
list(APPEND SRCS sim_encoder.c)
list(APPEND STDLIBS x264)
endif()
if(CONFIG_SPINLOCK)
list(APPEND HOSTSRCS sim_testset.c)
endif()

View File

@ -0,0 +1,547 @@
/****************************************************************************
* arch/sim/src/sim/sim_encoder.c
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The
* ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
****************************************************************************/
/****************************************************************************
* Included Files
****************************************************************************/
#include <errno.h>
#include <nuttx/kmalloc.h>
#include <nuttx/video/v4l2_m2m.h>
#include "sim_hostencoder.h"
#include "sim_internal.h"
/****************************************************************************
* Pre-processor Definitions
****************************************************************************/
#define SIM_ENCODER_NAME "sim-x264"
/****************************************************************************
* Private Types
****************************************************************************/
struct sim_encoder_s
{
struct host_encoder_s *encoder;
struct v4l2_format output_fmt;
struct v4l2_format capture_fmt;
struct work_s work;
void *cookie;
int bframe;
int fps;
bool capture_on;
bool flushing;
};
typedef struct sim_encoder_s sim_encoder_t;
/****************************************************************************
* Private Function Prototypes
****************************************************************************/
static int sim_encoder_open(void *cookie, void **priv);
static int sim_encoder_close(void *priv);
static int sim_encoder_capture_streamon(void *priv);
static int sim_encoder_output_streamon(void *priv);
static int sim_encoder_capture_streamoff(void *priv);
static int sim_encoder_output_streamoff(void *priv);
static int sim_encoder_capture_available(void *priv);
static int sim_encoder_output_available(void *priv);
static int sim_encoder_querycap(void *priv,
struct v4l2_capability *cap);
static int sim_encoder_capture_enum_fmt(void *priv,
struct v4l2_fmtdesc *fmt);
static int sim_encoder_output_enum_fmt(void *priv,
struct v4l2_fmtdesc *fmt);
static int sim_encoder_capture_g_fmt(void *priv,
struct v4l2_format *fmt);
static int sim_encoder_output_g_fmt(void *priv,
struct v4l2_format *fmt);
static int sim_encoder_capture_s_fmt(void *priv,
struct v4l2_format *fmt);
static int sim_encoder_output_s_fmt(void *priv,
struct v4l2_format *fmt);
static int sim_encoder_capture_try_fmt(void *priv,
struct v4l2_format *fmt);
static int sim_encoder_output_try_fmt(void *priv,
struct v4l2_format *fmt);
static size_t sim_encoder_capture_g_bufsize(void *priv);
static size_t sim_encoder_output_g_bufsize(void *priv);
static int sim_encoder_capture_s_parm(void *priv,
struct v4l2_streamparm *parm);
static int sim_encoder_capture_g_ext_ctrls(void *priv,
struct v4l2_ext_controls *ctrls);
static int sim_encoder_capture_s_ext_ctrls(void *priv,
struct v4l2_ext_controls *ctrls);
static int sim_encoder_subscribe_event(void *priv,
struct v4l2_event_subscription *sub);
static void sim_encoder_work(void *cookie);
/****************************************************************************
* Private Data
****************************************************************************/
static const struct codec_ops_s g_sim_encoder_ops =
{
.open = sim_encoder_open,
.close = sim_encoder_close,
.capture_streamon = sim_encoder_capture_streamon,
.output_streamon = sim_encoder_output_streamon,
.capture_streamoff = sim_encoder_capture_streamoff,
.output_streamoff = sim_encoder_output_streamoff,
.capture_available = sim_encoder_capture_available,
.output_available = sim_encoder_output_available,
.querycap = sim_encoder_querycap,
.capture_enum_fmt = sim_encoder_capture_enum_fmt,
.output_enum_fmt = sim_encoder_output_enum_fmt,
.capture_g_fmt = sim_encoder_capture_g_fmt,
.output_g_fmt = sim_encoder_output_g_fmt,
.capture_s_fmt = sim_encoder_capture_s_fmt,
.output_s_fmt = sim_encoder_output_s_fmt,
.capture_try_fmt = sim_encoder_capture_try_fmt,
.output_try_fmt = sim_encoder_output_try_fmt,
.capture_g_bufsize = sim_encoder_output_g_bufsize,
.output_g_bufsize = sim_encoder_capture_g_bufsize,
.capture_s_parm = sim_encoder_capture_s_parm,
.g_ext_ctrls = sim_encoder_capture_g_ext_ctrls,
.s_ext_ctrls = sim_encoder_capture_s_ext_ctrls,
.subscribe_event = sim_encoder_subscribe_event,
};
static struct codec_s g_sim_codec_encoder =
{
.ops = &g_sim_encoder_ops,
};
/****************************************************************************
* Private Functions
****************************************************************************/
static int sim_encoder_open(void *cookie, void **priv)
{
sim_encoder_t *sim_encoder;
struct host_encoder_s *encoder;
sim_encoder = kmm_zalloc(sizeof(struct sim_encoder_s));
if (sim_encoder == NULL)
{
return -ENOMEM;
}
encoder = host_encoder_open();
if (encoder == NULL)
{
kmm_free(sim_encoder);
return -ENOMEM;
}
sim_encoder->encoder = encoder;
sim_encoder->cookie = cookie;
*priv = sim_encoder;
return 0;
}
static int sim_encoder_close(void *priv)
{
sim_encoder_t *sim_encoder = priv;
host_encoder_close(sim_encoder->encoder);
kmm_free(sim_encoder);
return 0;
}
static int sim_encoder_capture_streamon(void *priv)
{
sim_encoder_t *sim_encoder = priv;
sim_encoder->capture_on = true;
work_queue(HPWORK, &sim_encoder->work,
sim_encoder_work, sim_encoder, 0);
return 0;
}
static int sim_encoder_output_streamon(void *priv)
{
sim_encoder_t *sim_encoder = priv;
return host_encoder_streamon(sim_encoder->encoder,
sim_encoder->output_fmt.fmt.pix.width,
sim_encoder->output_fmt.fmt.pix.height,
sim_encoder->fps ? sim_encoder->fps : 30,
sim_encoder->bframe);
}
static int sim_encoder_capture_available(void *priv)
{
sim_encoder_t *sim_encoder = priv;
if (sim_encoder->capture_on == false)
{
return 0;
}
work_queue(HPWORK, &sim_encoder->work,
sim_encoder_work, sim_encoder, 0);
return 0;
}
static int sim_encoder_output_available(void *priv)
{
sim_encoder_t *sim_encoder = priv;
if (sim_encoder->capture_on == false)
{
return 0;
}
work_queue(HPWORK, &sim_encoder->work,
sim_encoder_work, sim_encoder, 0);
return 0;
}
static int sim_encoder_capture_streamoff(void *priv)
{
sim_encoder_t *sim_encoder = priv;
sim_encoder->capture_on = false;
return host_encoder_streamoff(sim_encoder->encoder);
}
static int sim_encoder_output_streamoff(void *priv)
{
sim_encoder_t *sim_encoder = priv;
if (!sim_encoder->capture_on)
{
return 0;
}
sim_encoder->flushing = true;
work_queue(HPWORK, &sim_encoder->work,
sim_encoder_work, sim_encoder, 0);
return 0;
}
static int sim_encoder_querycap(void *priv,
struct v4l2_capability *cap)
{
strncpy((char *)cap->driver, SIM_ENCODER_NAME, sizeof(cap->driver));
strncpy((char *)cap->card, SIM_ENCODER_NAME, sizeof(cap->card));
cap->capabilities = V4L2_CAP_VIDEO_M2M;
return 0;
}
static int sim_encoder_capture_enum_fmt(void *priv,
struct v4l2_fmtdesc *fmt)
{
if (fmt->index >= 1)
{
return -EINVAL;
}
fmt->pixelformat = V4L2_PIX_FMT_H264;
return 0;
}
static int sim_encoder_output_enum_fmt(void *priv,
struct v4l2_fmtdesc *fmt)
{
if (fmt->index >= 1)
{
return -EINVAL;
}
fmt->pixelformat = V4L2_PIX_FMT_YUV420;
return 0;
}
static int sim_encoder_capture_g_fmt(void *priv,
struct v4l2_format *fmt)
{
fmt->fmt.pix.field = V4L2_FIELD_NONE;
fmt->fmt.pix.pixelformat = V4L2_PIX_FMT_H264;
return 0;
}
static int sim_encoder_output_g_fmt(void *priv,
struct v4l2_format *fmt)
{
fmt->fmt.pix.field = V4L2_FIELD_NONE;
fmt->fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420;
return 0;
}
static int sim_encoder_capture_try_fmt(void *priv,
struct v4l2_format *fmt)
{
if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_H264)
{
return 0;
}
return -EINVAL;
}
static int sim_encoder_output_try_fmt(void *priv,
struct v4l2_format *fmt)
{
if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_YUV420)
{
return 0;
}
return -EINVAL;
}
static int sim_encoder_capture_s_fmt(void *priv,
struct v4l2_format *fmt)
{
sim_encoder_t *sim_encoder = priv;
size_t sizeimage;
if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_H264)
{
sim_encoder->capture_fmt = *fmt;
sizeimage = fmt->fmt.pix.width * fmt->fmt.pix.height;
sizeimage = (sizeimage * 3 / 2) / 2 + 128;
sim_encoder->capture_fmt.fmt.pix.sizeimage = sizeimage;
return 0;
}
return -EINVAL;
}
static int sim_encoder_output_s_fmt(void *priv,
struct v4l2_format *fmt)
{
sim_encoder_t *sim_encoder = priv;
size_t sizeimage;
if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_YUV420)
{
sim_encoder->output_fmt = *fmt;
sizeimage = fmt->fmt.pix.width * fmt->fmt.pix.height * 3 / 2;
sim_encoder->output_fmt.fmt.pix.sizeimage = sizeimage;
return 0;
}
return -EINVAL;
}
static size_t sim_encoder_capture_g_bufsize(void *priv)
{
sim_encoder_t *sim_encoder = priv;
if (sim_encoder->capture_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_H264)
{
return sim_encoder->capture_fmt.fmt.pix.sizeimage;
}
return 0;
}
static size_t sim_encoder_output_g_bufsize(void *priv)
{
sim_encoder_t *sim_encoder = priv;
if (sim_encoder->output_fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_YUV420)
{
return sim_encoder->output_fmt.fmt.pix.sizeimage;
}
return 0;
}
static int sim_encoder_capture_s_parm(void *priv,
struct v4l2_streamparm *parm)
{
sim_encoder_t *sim_encoder = priv;
struct v4l2_fract fract;
fract = parm->parm.capture.timeperframe;
if (fract.numerator > 0 && fract.denominator > 0)
{
sim_encoder->fps = fract.denominator / fract.numerator;
}
return 0;
}
static int sim_encoder_capture_s_ext_ctrls(void *priv,
struct v4l2_ext_controls *ctrls)
{
sim_encoder_t *sim_encoder = priv;
struct v4l2_ext_control *ctrl;
if (ctrls->count != 1)
{
return -EINVAL;
}
ctrl = ctrls->controls;
switch (ctrl->id)
{
case V4L2_CID_MPEG_VIDEO_B_FRAMES:
sim_encoder->bframe = ctrl->value;
return 0;
default:
return -EINVAL;
}
}
static int sim_encoder_capture_g_ext_ctrls(void *priv,
struct v4l2_ext_controls *ctrls)
{
sim_encoder_t *sim_encoder = priv;
struct v4l2_ext_control *ctrl;
if (ctrls->count != 1)
{
return -EINVAL;
}
ctrl = ctrls->controls;
switch (ctrl->id)
{
case V4L2_CID_MPEG_VIDEO_B_FRAMES:
ctrl->value = sim_encoder->bframe;
return 0;
default:
return -EINVAL;
}
}
static int sim_encoder_subscribe_event(void *priv,
struct v4l2_event_subscription *sub)
{
switch (sub->type)
{
case V4L2_EVENT_EOS:
return OK;
default:
return -EINVAL;
}
}
static int sim_encoder_process(sim_encoder_t *sim_encoder,
struct v4l2_buffer *dst_buf,
struct v4l2_buffer *src_buf)
{
struct v4l2_event event;
uint8_t *src_data = NULL;
uint32_t src_size = 0;
int64_t src_pts = 0;
int64_t dst_pts = 0;
int ret;
if (src_buf != NULL)
{
src_data = (uint8_t *)src_buf->m.userptr;
src_size = src_buf->bytesused;
src_pts = src_buf->timestamp.tv_sec * 1000000 +
src_buf->timestamp.tv_usec;
}
ret = host_encoder_enqueue(sim_encoder->encoder,
src_data, src_size, src_pts);
if (ret >= 0 && src_buf != NULL)
{
codec_output_put_buf(sim_encoder->cookie, src_buf);
}
if (ret < 1)
{
return ret;
}
ret = host_encoder_dequeue(sim_encoder->encoder,
(uint8_t *)dst_buf->m.userptr,
&dst_buf->bytesused,
&dst_pts,
&dst_buf->flags);
if (ret == 0 && src_buf == NULL)
{
sim_encoder->flushing = false;
dst_buf->flags |= V4L2_BUF_FLAG_LAST;
memset(&event, 0, sizeof(event));
event.type = V4L2_EVENT_EOS;
codec_queue_event(sim_encoder->cookie, &event);
}
dst_buf->timestamp.tv_usec = dst_pts % 1000000;
dst_buf->timestamp.tv_sec = dst_pts / 1000000;
codec_capture_put_buf(sim_encoder->cookie, dst_buf);
return ret;
}
static void sim_encoder_work(void *encoder)
{
sim_encoder_t *sim_encoder = encoder;
struct v4l2_buffer *src_buf;
struct v4l2_buffer *dst_buf;
int ret;
src_buf = codec_output_get_buf(sim_encoder->cookie);
if (src_buf == NULL && !sim_encoder->flushing)
{
return;
}
dst_buf = codec_capture_get_buf(sim_encoder->cookie);
if (dst_buf == NULL)
{
return;
}
ret = sim_encoder_process(encoder, dst_buf, src_buf);
if (ret > 0)
{
work_queue(HPWORK, &sim_encoder->work,
sim_encoder_work, sim_encoder, 0);
}
}
/****************************************************************************
* Public Functions
****************************************************************************/
int sim_encoder_initialize()
{
return codec_register(CONFIG_SIM_VIDEO_ENCODER_DEV_PATH,
&g_sim_codec_encoder);
}

View File

@ -0,0 +1,190 @@
/****************************************************************************
* arch/sim/src/sim/sim_hostencoder.c
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The
* ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
****************************************************************************/
/****************************************************************************
* Included Files
****************************************************************************/
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <linux/videodev2.h>
#include <x264.h>
#include "sim_hostencoder.h"
#include "sim_internal.h"
/****************************************************************************
* Private Types
****************************************************************************/
struct host_encoder_s
{
x264_t *enc_ctx;
x264_picture_t pic_in;
x264_picture_t pic_out;
x264_nal_t *nal;
x264_param_t param;
int i_nal;
int remaining_frames;
};
/****************************************************************************
* Public Functions
****************************************************************************/
struct host_encoder_s *host_encoder_open(void)
{
return calloc(1, sizeof(struct host_encoder_s));
}
int host_encoder_close(struct host_encoder_s *encoder)
{
free(encoder);
return 0;
}
int host_encoder_streamon(struct host_encoder_s *encoder,
int width, int height, int fps, int bframe)
{
int ret;
memset(&encoder->param, 0, sizeof(x264_param_t));
ret = x264_param_default_preset(&encoder->param,
"fast",
"zerolatency");
if (ret < 0)
{
return ret;
}
encoder->param.i_width = width;
encoder->param.i_height = height;
encoder->param.i_fps_num = fps;
encoder->param.i_fps_den = 1;
encoder->param.b_annexb = 1;
encoder->param.i_csp = X264_CSP_I420;
encoder->param.i_keyint_max = 50;
encoder->param.i_keyint_min = 25;
encoder->param.i_bframe = bframe;
ret = x264_picture_alloc(&encoder->pic_in,
X264_CSP_I420,
width,
height);
if (ret < 0)
{
return ret;
}
encoder->enc_ctx = x264_encoder_open(&encoder->param);
if (!encoder->enc_ctx)
{
x264_picture_clean(&encoder->pic_in);
return -EINVAL;
}
return 0;
}
int host_encoder_streamoff(struct host_encoder_s *encoder)
{
x264_encoder_close(encoder->enc_ctx);
x264_picture_clean(&encoder->pic_in);
encoder->remaining_frames = 0;
return 0;
}
int host_encoder_enqueue(struct host_encoder_s *encoder,
uint8_t *data, uint32_t size, int64_t pts)
{
int ret;
if (data != NULL)
{
int width = encoder->param.i_width;
int height = encoder->param.i_height;
encoder->pic_in.i_pts = pts;
memcpy(encoder->pic_in.img.plane[0], data, width * height);
memcpy(encoder->pic_in.img.plane[1], data + width * height,
width * height / 4);
memcpy(encoder->pic_in.img.plane[2], data + width * height * 5 / 4,
width * height / 4);
}
ret = x264_encoder_encode(encoder->enc_ctx,
&encoder->nal,
&encoder->i_nal,
(data != NULL ? &encoder->pic_in : NULL),
&encoder->pic_out);
if (data == NULL)
{
encoder->remaining_frames =
x264_encoder_delayed_frames(encoder->enc_ctx);
}
if (ret >= 0)
{
return 1;
}
return ret;
}
int host_encoder_dequeue(struct host_encoder_s *encoder,
uint8_t *data, uint32_t *size,
int64_t *pts, uint32_t *flags)
{
int total_size = 0;
int i;
for (i = 0; i < encoder->i_nal; i++)
{
memcpy(data + total_size,
encoder->nal[i].p_payload, encoder->nal[i].i_payload);
total_size += encoder->nal[i].i_payload;
}
*size = total_size;
*pts = encoder->pic_out.i_pts;
switch (encoder->pic_out.i_type)
{
case X264_TYPE_IDR:
case X264_TYPE_I:
*flags = V4L2_BUF_FLAG_KEYFRAME;
break;
case X264_TYPE_P:
*flags = V4L2_BUF_FLAG_PFRAME;
break;
case X264_TYPE_B:
case X264_TYPE_BREF:
*flags = V4L2_BUF_FLAG_BFRAME;
break;
}
return encoder->remaining_frames;
}

View File

@ -0,0 +1,51 @@
/****************************************************************************
* arch/sim/src/sim/sim_hostencoder.h
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The
* ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
****************************************************************************/
#ifndef __ARCH_SIM_SRC_SIM_SIM_HOSTENCODER_H
#define __ARCH_SIM_SRC_SIM_SIM_HOSTENCODER_H
/****************************************************************************
* Included Files
****************************************************************************/
#include <stdint.h>
/****************************************************************************
* Public Types
****************************************************************************/
struct host_encoder_s;
/****************************************************************************
* Public Function Prototypes
****************************************************************************/
struct host_encoder_s *host_encoder_open(void);
int host_encoder_close(struct host_encoder_s *encoder);
int host_encoder_streamon(struct host_encoder_s *encoder,
int width, int height, int fps, int bframe);
int host_encoder_streamoff(struct host_encoder_s *encoder);
int host_encoder_enqueue(struct host_encoder_s *encoder,
uint8_t *data, uint32_t size, int64_t pts);
int host_encoder_dequeue(struct host_encoder_s *encoder,
uint8_t *data, uint32_t *size,
int64_t *pts, uint32_t *flags);
#endif /* __ARCH_SIM_SRC_SIM_SIM_HOSTENCODER_H */

View File

@ -310,6 +310,10 @@ void up_initialize(void)
sim_decoder_initialize();
#endif
#ifdef CONFIG_SIM_VIDEO_ENCODER
sim_encoder_initialize();
#endif
kthread_create("loop_task", CONFIG_SIM_LOOPTASK_PRIORITY,
CONFIG_DEFAULT_TASK_STACKSIZE,
sim_loop_task, NULL);

View File

@ -439,6 +439,10 @@ void sim_camera_loop(void);
int sim_decoder_initialize(void);
#endif
#ifdef CONFIG_SIM_VIDEO_ENCODER
int sim_encoder_initialize(void);
#endif
/* sim_usbdev.c *************************************************************/
#ifdef CONFIG_SIM_USB_DEV