Реализуйте требования, Allwinner IPC, локальный предварительный просмотр видео PJSIP, декодирование и отображение однорангового видео. Сначала разберитесь с процессом локального предварительного просмотра и декодирования PJSIP.
Локальный предварительный просмотр: включена конфигурация по умолчанию vid_preview_enable_native.
PJ_DEF(void) pjsua_media_config_default(pjsua_media_config *cfg)
{
//....
cfg->vid_preview_enable_native = PJ_TRUE;
}
static pj_status_t create_vid_win(pjsua_vid_win_type type,
const pjmedia_format *fmt,
pjmedia_vid_dev_index rend_id,
pjmedia_vid_dev_index cap_id,
pj_bool_t show,
unsigned wnd_flags,
const pjmedia_vid_dev_hwnd *wnd,
pjsua_vid_win_id *id)
{
pj_bool_t enable_native_preview;
pjsua_vid_win_id wid = PJSUA_INVALID_ID;
pjsua_vid_win *w = NULL;
pjmedia_vid_port_param vp_param;
pjmedia_format fmt_;
pj_status_t status;
unsigned i;
enable_native_preview = pjsua_var.media_cfg.vid_preview_enable_native;
//...
//...
/*
* Determine if the device supports native preview.
*/
status = pjmedia_vid_dev_get_info(cap_id, &vdi);
if (status != PJ_SUCCESS)
goto on_error;
if (enable_native_preview &&
(vdi.caps & PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW))
{
/* Device supports native preview! */
w->is_native = PJ_TRUE;
}
//...
}
Процесс локального предварительного просмотра видео:
Видео собирается локально, кодируется и затем передается:
Обратный вызов коллекции камер:
Кодирование драйвера on_clock_tick, отправка по rtp.
Отображать одноранговое видео, получать пакеты, декодировать и затем отображать.
channel->stream->transport
/**
* Media channel.
*/
typedef struct pjmedia_vid_channel
{
pjmedia_vid_stream *stream; /**< Parent stream. */
pjmedia_dir dir; /**< Channel direction. */
pjmedia_port port; /**< Port interface. */
unsigned pt; /**< Payload type. */
pj_bool_t paused; /**< Paused?. */
void *buf; /**< Output buffer. */
unsigned buf_size; /**< Size of output buffer. */
pjmedia_rtp_session rtp; /**< RTP session. */
} pjmedia_vid_channel;
/**
* This structure describes media stream.
* A media stream is bidirectional media transmission between two endpoints.
* It consists of two channels, i.e. encoding and decoding channels.
* A media stream corresponds to a single "m=" line in a SDP session
* description.
*/
struct pjmedia_vid_stream
{
pj_pool_t *own_pool; /**< Internal pool. */
pjmedia_endpt *endpt; /**< Media endpoint. */
pjmedia_vid_codec_mgr *codec_mgr; /**< Codec manager. */
pjmedia_vid_stream_info info; /**< Stream info. */
pj_grp_lock_t *grp_lock; /**< Stream lock. */
pjmedia_vid_channel *enc; /**< Encoding channel. */
pjmedia_vid_channel *dec; /**< Decoding channel. */
pjmedia_dir dir; /**< Stream direction. */
void *user_data; /**< User data. */
pj_str_t name; /**< Stream name */
pj_str_t cname; /**< SDES CNAME */
pjmedia_transport *transport; /**< Stream transport. */
}
Создайте канал кодирования данных: create_channel( pool, stream, PJMEDIA_DIR_ENCODING, info->tx_pt, info, &stream->enc);
Установите канал декодирования данных: create_channel( pool, stream, PJMEDIA_DIR_DECODING, info->rx_pt, info, &stream->dec);
Аудио
Stream.c (pjmedia\src\pjmedia):static pj_status_t create_channel( pj_pool_t *pool,
Stream.c (pjmedia\src\pjmedia): статус = create_channel( пул, поток, PJMEDIA_DIR_DECODING,
Stream.c (pjmedia\src\pjmedia): статус = create_channel( пул, поток, PJMEDIA_DIR_ENCODING,
видео
Vid_stream.c (pjmedia\src\pjmedia):static pj_status_t create_channel( pj_pool_t *pool,
Vid_stream.c (pjmedia\src\pjmedia): статус = create_channel( пул, поток, PJMEDIA_DIR_DECODING,
Vid_stream.c (pjmedia\src\pjmedia): статус = create_channel( пул, поток, PJMEDIA_DIR_ENCODING,
/*
* Create stream.
*/
PJ_DEF(pj_status_t) pjmedia_vid_stream_create(
pjmedia_endpt *endpt,
pj_pool_t *pool,
pjmedia_vid_stream_info *info,
pjmedia_transport *tp,
void *user_data,
pjmedia_vid_stream **p_stream)
/*
* Create media channel.
*/
static pj_status_t create_channel( pj_pool_t *pool,
pjmedia_vid_stream *stream,
pjmedia_dir dir,
unsigned pt,
const pjmedia_vid_stream_info *info,
pjmedia_vid_channel **p_channel)
{
/* Create RTP and RTCP sessions: */
{
pjmedia_rtp_session_setting settings;
settings.flags = (pj_uint8_t)((info->rtp_seq_ts_set << 2) |
(info->has_rem_ssrc << 4) | 3);
settings.default_pt = pt;
settings.sender_ssrc = info->ssrc;
settings.peer_ssrc = info->rem_ssrc;
settings.seq = info->rtp_seq;
settings.ts = info->rtp_ts;
status = pjmedia_rtp_session_init2(&channel->rtp, settings);
}
}
Процесс декодирования:
Вопрос 1:
Драйвер кодирования и отправки управляется on_clock_tick, а как насчет драйвера дисплея декодирования? Что касается звука, воспроизведение звука осуществляется драйвером get_frame порта драйвера play_cb аудиоустройства.
Но отображение пакета декодирования видео управляется on_clock_tick.
/* Call sink->put_frame()
* Note that if transmitter_cnt==0, we should still call put_frame()
* with zero frame size, as sink may need to send keep-alive packets
* and get timestamp update.
*/
pj_bzero(&frame, sizeof(frame));
frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
frame.timestamp = *now;
if (frame_rendered) {
frame.buf = sink->put_buf;
frame.size = sink->put_frm_size;
}
status = pjmedia_port_put_frame(sink->port, &frame);
if (frame_rendered && status != PJ_SUCCESS) {
sink->last_err_cnt++;
if (sink->last_err != status ||
sink->last_err_cnt % MAX_ERR_COUNT == 0)
{
if (sink->last_err != status)
sink->last_err_cnt = 1;
sink->last_err = status;
PJ_PERROR(5, (THIS_FILE, status,
"Failed (%d time(s)) to put frame to port %d"
" [%s]!", sink->last_err_cnt,
sink->idx, sink->port->info.name.ptr));
}
} else {
sink->last_err = status;
sink->last_err_cnt = 0;
}
/**
* Get a frame from the port (and subsequent downstream ports).
*/
PJ_DEF(pj_status_t) pjmedia_port_get_frame( pjmedia_port *port,
pjmedia_frame *frame )
{
PJ_ASSERT_RETURN(port && frame, PJ_EINVAL);
if (port->get_frame)
return port->get_frame(port, frame);
else {
frame->type = PJMEDIA_FRAME_TYPE_NONE;
return PJ_EINVALIDOP;
}
}
Отладка точки останова обнаружила, что драйвером устройства отображения sdl является таймер on_clock_tick.
sink->port = 0x044f70fc {info={name={ptr=0x044f70ec "SDL renderer" slen=12 } signature=1448038479 dir=PJMEDIA_DIR_DECODING (2) ...} ...}
Этот источник:
port = 0x040fb7cc {info={name={ptr=0x040fb860 "vstdec040FAE74" slen=14 } signature=1347834708 dir=PJMEDIA_DIR_DECODING (2) ...} ...}
Оказывается, декодированные данные помещаются вstream->dec_frame.bufв,Метод get_frame обнаруживает, что dec_frame.size больше 0.,Затем скопируйте его в рамку,используется для отображения.
static pj_status_t get_frame(pjmedia_port *port,
pjmedia_frame *frame)
{
pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata;
pjmedia_vid_channel *channel = stream->dec;
/* Return no frame is channel is paused */
if (channel->paused) {
frame->type = PJMEDIA_FRAME_TYPE_NONE;
frame->size = 0;
return PJ_SUCCESS;
}
/* Report pending events. Do not publish the event while holding the
* stream lock as that would lead to deadlock. It should be safe to
* operate on fmt_event without the mutex because format change normally
* would only occur once during the start of the media.
*/
if (stream->fmt_event.type != PJMEDIA_EVENT_NONE) {
pjmedia_event_fmt_changed_data *fmt_chg_data;
fmt_chg_data = &stream->fmt_event.data.fmt_changed;
#if 1 //realloc size
pj_int32_t new_size = fmt_chg_data->new_fmt.det.vid.size.h*fmt_chg_data->new_fmt.det.vid.size.w*1.5;
if (stream->dec_max_size < new_size)
{
PJ_LOG(5, (THIS_FILE, "Reallocating vid_stream dec_buffer %u --> %u",
(unsigned)stream->dec_max_size,
(unsigned)new_size));
pj_mutex_lock(stream->jb_mutex);
stream->dec_max_size = new_size;
stream->dec_frame.buf = pj_pool_alloc(stream->own_pool, stream->dec_max_size);
pj_mutex_unlock(stream->jb_mutex);
}
#endif//realloc size
/* Update stream info and decoding channel port info */
if (fmt_chg_data->dir == PJMEDIA_DIR_DECODING) {
pjmedia_format_copy(&stream->info.codec_param->dec_fmt,
&fmt_chg_data->new_fmt);
pjmedia_format_copy(&stream->dec->port.info.fmt,
&fmt_chg_data->new_fmt);
/* Override the framerate to be 1.5x higher in the event
* for the renderer.
*/
fmt_chg_data->new_fmt.det.vid.fps.num *= 3;
fmt_chg_data->new_fmt.det.vid.fps.num /= 2;
} else {
pjmedia_format_copy(&stream->info.codec_param->enc_fmt,
&fmt_chg_data->new_fmt);
pjmedia_format_copy(&stream->enc->port.info.fmt,
&fmt_chg_data->new_fmt);
}
dump_port_info(fmt_chg_data->dir==PJMEDIA_DIR_DECODING ?
stream->dec : stream->enc,
"changed");
pjmedia_event_publish(NULL, port, &stream->fmt_event,
PJMEDIA_EVENT_PUBLISH_POST_EVENT);
stream->fmt_event.type = PJMEDIA_EVENT_NONE;
}
if (stream->miss_keyframe_event.type != PJMEDIA_EVENT_NONE) {
pjmedia_event_publish(NULL, port, &stream->miss_keyframe_event,
PJMEDIA_EVENT_PUBLISH_POST_EVENT);
stream->miss_keyframe_event.type = PJMEDIA_EVENT_NONE;
}
pj_grp_lock_acquire( stream->grp_lock );
if (stream->dec_frame.size == 0) {
/* Don't have frame in buffer, try to decode one */
if (decode_frame(stream, frame) != PJ_SUCCESS) {
frame->type = PJMEDIA_FRAME_TYPE_NONE;
frame->size = 0;
}
} else {
if (frame->size < stream->dec_frame.size) {
PJ_LOG(4,(stream->dec->port.info.name.ptr,
"Error: not enough buffer for decoded frame "
"(supplied=%d, required=%d)",
(int)frame->size, (int)stream->dec_frame.size));
frame->type = PJMEDIA_FRAME_TYPE_NONE;
frame->size = 0;
} else {
frame->type = stream->dec_frame.type;
frame->timestamp = stream->dec_frame.timestamp;
frame->size = stream->dec_frame.size;
pj_memcpy(frame->buf, stream->dec_frame.buf, frame->size);
}
stream->dec_frame.size = 0;
}
pj_grp_lock_release( stream->grp_lock );
return PJ_SUCCESS;
}
Как порт, отображающий поток, связан с портом, который декодирует выходные данные?
В методе pjmedia_vid_port_create есть следующий код:
vp_param.active = PJ_FALSE;
pjmedia_vid_port_create:
vp->role = prm->active ? ROLE_ACTIVE : ROLE_PASSIVE;
PJ_DEF(pj_status_t) pjmedia_vid_port_create( pj_pool_t *pool,
const pjmedia_vid_port_param *prm,
pjmedia_vid_port **p_vid_port)
{
///......
} else if (vp->role==ROLE_PASSIVE) {
vid_pasv_port *pp;
/* Always need to create media port for passive role */
vp->pasv_port = pp = PJ_POOL_ZALLOC_T(pool, vid_pasv_port);
pp->vp = vp;
if (prm->vidparam.dir & PJMEDIA_DIR_CAPTURE)
pp->base.get_frame = &vid_pasv_port_get_frame;
if (prm->vidparam.dir & PJMEDIA_DIR_RENDER)
pp->base.put_frame = &vid_pasv_port_put_frame;
pp->base.on_destroy = &vid_pasv_port_on_destroy;
pjmedia_port_info_init2(&pp->base.info, &vp->dev_name,
PJMEDIA_SIG_VID_PORT,
prm->vidparam.dir, &prm->vidparam.fmt);
need_frame_buf = PJ_TRUE;
}
//pjsua_vid.c
static pj_status_t create_vid_win(pjsua_vid_win_type type,
const pjmedia_format *fmt,
pjmedia_vid_dev_index rend_id,
pjmedia_vid_dev_index cap_id,
pj_bool_t show,
unsigned wnd_flags,
const pjmedia_vid_dev_hwnd *wnd,
pjsua_vid_win_id *id)
/* Create renderer video port, only if it's not a native preview */
if (!w->is_native) {
status = pjmedia_vid_dev_default_param(w->pool, rend_id,
&vp_param.vidparam);
if (status != PJ_SUCCESS)
goto on_error;
vp_param.active = PJ_FALSE;
vp_param.vidparam.dir = PJMEDIA_DIR_RENDER;
vp_param.vidparam.fmt = *fmt;
vp_param.vidparam.disp_size = fmt->det.vid.size;
vp_param.vidparam.flags |= PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE;
vp_param.vidparam.window_hide = !show;
vp_param.vidparam.flags |= PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW_FLAGS;
vp_param.vidparam.window_flags = wnd_flags;
if (wnd) {
vp_param.vidparam.flags |= PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;
vp_param.vidparam.window = *wnd;
}
status = pjmedia_vid_port_create(w->pool, &vp_param, &w->vp_rend);
if (status != PJ_SUCCESS)
goto on_error;
/* Register renderer to the video conf */
status = pjsua_vid_conf_add_port(
w->pool,
pjmedia_vid_port_get_passive_port(w->vp_rend),
NULL, &w->rend_slot);
if (status != PJ_SUCCESS)
goto on_error;
/* For preview window, connect capturer & renderer (via conf) */
if (w->type == PJSUA_WND_TYPE_PREVIEW && show) {
status = pjsua_vid_conf_connect(w->cap_slot, w->rend_slot, NULL);
if (status != PJ_SUCCESS)
goto on_error;
}
PJ_LOG(4,(THIS_FILE,
"%s window id %d created for cap_dev=%d rend_dev=%d",
pjsua_vid_win_type_name(type), wid, cap_id, rend_id));
} else {
PJ_LOG(4,(THIS_FILE,
"Preview window id %d created for cap_dev %d, "
"using built-in preview!",
wid, cap_id));
}
}
static pj_status_t vid_pasv_port_put_frame(struct pjmedia_port *this_port,
pjmedia_frame *frame)
{
struct vid_pasv_port *vpp = (struct vid_pasv_port*)this_port;
pjmedia_vid_port *vp = vpp->vp;
if (vp->pasv_port->is_destroying)
return PJ_EGONE;
handle_format_change(vp);
if (vp->stream_role==ROLE_PASSIVE) {
/* We are passive and the stream is passive.
* The encoding counterpart is in vid_pasv_port_get_frame().
*/
pj_status_t status;
pjmedia_frame frame_;
if (frame->size != vp->src_size) {
if (frame->size > 0) {
PJ_LOG(4,(THIS_FILE, "Unexpected frame size %lu, expected %lu",
(unsigned long)frame->size,
(unsigned long)vp->src_size));
}
pj_memcpy(&frame_, frame, sizeof(pjmedia_frame));
frame_.buf = NULL;
frame_.size = 0;
/* Send heart beat for updating timestamp or keep-alive. */
return pjmedia_vid_dev_stream_put_frame(vp->strm, &frame_);
}
pj_bzero(&frame_, sizeof(frame_));
status = convert_frame(vp, frame, &frame_);
if (status != PJ_SUCCESS)
return status;
//Это выводится в put_frame устройства отображения.
return pjmedia_vid_dev_stream_put_frame(vp->strm, (vp->conv.conv?
&frame_: frame));
} else {
/* We are passive while the stream is active so we just store the
* frame in the buffer.
* The encoding counterpart is located in vidstream_cap_cb()
*/
if (frame->size == vp->src_size)
copy_frame_to_buffer(vp, frame);
}
return PJ_SUCCESS;
}
В принципе, процесс декодирования и отображения видео с одноранговой камеры прояснён, и у нас есть основная идея реализовать функцию отображения видео с одноранговой камеры.
1. Обратитесь к sdl_dev.c, чтобы реализовать отображаемую разработку, а затем зарегистрируйте ее на заводе.
2. Адаптация дисплея декодирования.