1 /*
2 * MMAL Video Decoder
3 * Copyright (c) 2015 rcombs
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 /**
23 * @file
24 * MMAL Video Decoder
25 */
26
27 #include <bcm_host.h>
28 #include <interface/mmal/mmal.h>
29 #include <interface/mmal/mmal_parameters_video.h>
30 #include <interface/mmal/util/mmal_util.h>
31 #include <interface/mmal/util/mmal_util_params.h>
32 #include <interface/mmal/util/mmal_default_components.h>
33 #include <interface/mmal/vc/mmal_vc_api.h>
34 #include <stdatomic.h>
35
46
55
56 // MMAL_POOL_T destroys all of its MMAL_BUFFER_HEADER_Ts. If we want correct
57 // refcounting for AVFrames, we can free the MMAL_POOL_T only after all AVFrames
58 // have been unreferenced.
63
68
73
78
79 // Waiting input packets. Because the libavcodec API requires decoding and
80 // returning packets in lockstep, it can happen that queue_decoded_frames
81 // contains almost all surfaces - then the decoder input queue can quickly
82 // fill up and won't accept new input either. Without consuming input, the
83 // libavcodec API can't return new frames, and we have a logical deadlock.
84 // This is avoided by queuing such buffers here.
86 /* Packet used to hold received packets temporarily; not owned by us. */
88
98
99 // Assume decoder is guaranteed to produce output after at least this many
100 // packets (where each packet contains 1 frame).
101 #define MAX_DELAYED_FRAMES 16
102
104 {
107 mmal_pool_destroy(
ref->pool);
109 }
110 }
111
113 {
115
116 mmal_buffer_header_release(
ref->buffer);
118
120 }
121
122 // Setup frame with a new reference to buffer. The buffer must have been
123 // allocated from the given pool.
125 MMAL_BUFFER_HEADER_T *
buffer)
126 {
130
133
137 if (!
frame->buf[0]) {
140 }
141
143 mmal_buffer_header_acquire(
buffer);
144
146 frame->data[3] = (uint8_t *)
ref->buffer;
147 return 0;
148 }
149
151 {
154 MMAL_BUFFER_HEADER_T *
buffer;
155
156 mmal_port_disable(
decoder->input[0]);
157 mmal_port_disable(
decoder->output[0]);
158 mmal_port_disable(
decoder->control);
159
160 mmal_port_flush(
decoder->input[0]);
161 mmal_port_flush(
decoder->output[0]);
162 mmal_port_flush(
decoder->control);
163
164 while ((
buffer = mmal_queue_get(
ctx->queue_decoded_frames)))
165 mmal_buffer_header_release(
buffer);
166
167 while (
ctx->waiting_buffers) {
169
171
172 if (
buffer->
flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
174
177 }
178 ctx->waiting_buffers_tail =
NULL;
179
181
182 ctx->frames_output =
ctx->eos_received =
ctx->eos_sent =
ctx->packets_sent =
ctx->extradata_sent = 0;
183 }
184
186 {
188
191
192 mmal_component_destroy(
ctx->decoder);
194 mmal_queue_destroy(
ctx->queue_decoded_frames);
195 mmal_pool_destroy(
ctx->pool_in);
197
198 mmal_vc_deinit();
199
200 return 0;
201 }
202
204 {
207
211 if (entry->
flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
214 }
215 mmal_buffer_header_release(
buffer);
216 }
217
219 {
222
223 mmal_queue_put(
ctx->queue_decoded_frames,
buffer);
224 }
225
227 {
230
231 if (
buffer->cmd == MMAL_EVENT_ERROR) {
234 } else {
237 }
238
239 mmal_buffer_header_release(
buffer);
240 }
241
242 // Feed free output buffers to the decoder.
244 {
246 MMAL_BUFFER_HEADER_T *
buffer;
248
250 return AVERROR_UNKNOWN;
// format change code failed with OOM previously
251
252 while ((
buffer = mmal_queue_get(
ctx->pool_out->pool->queue))) {
253 if ((
status = mmal_port_send_buffer(
ctx->decoder->output[0],
buffer))) {
254 mmal_buffer_header_release(
buffer);
257 }
258 }
259
260 return 0;
261 }
262
264 {
266 case MMAL_COLOR_SPACE_BT470_2_BG:
267 case MMAL_COLOR_SPACE_BT470_2_M:
273 }
274 }
275
277 {
282 MMAL_ES_FORMAT_T *format_out =
decoder->output[0]->format;
283 MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T interlace_type;
284
289 }
291
292 if (!format_out)
294
295 if ((
status = mmal_port_parameter_set_uint32(
decoder->output[0], MMAL_PARAMETER_EXTRA_BUFFERS,
ctx->extra_buffers)))
297
298 if ((
status = mmal_port_parameter_set_boolean(
decoder->output[0], MMAL_PARAMETER_VIDEO_INTERPOLATE_TIMESTAMPS, 0)))
300
302 format_out->encoding = MMAL_ENCODING_OPAQUE;
303 } else {
304 format_out->encoding_variant = format_out->encoding = MMAL_ENCODING_I420;
305 }
306
309
310 interlace_type.hdr.id = MMAL_PARAMETER_VIDEO_INTERLACE_TYPE;
311 interlace_type.hdr.size = sizeof(MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T);
312 status = mmal_port_parameter_get(
decoder->output[0], &interlace_type.hdr);
313 if (
status != MMAL_SUCCESS) {
315 } else {
316 ctx->interlaced_frame = (interlace_type.eMode != MMAL_InterlaceProgressive);
317 ctx->top_field_first = (interlace_type.eMode == MMAL_InterlaceFieldsInterleavedUpperFirst);
318 }
319
321 format_out->es->video.crop.y + format_out->es->video.crop.height)) < 0)
323
324 if (format_out->es->video.par.num && format_out->es->video.par.den) {
327 }
328 if (format_out->es->video.frame_rate.num && format_out->es->video.frame_rate.den) {
329 avctx->
framerate.
num = format_out->es->video.frame_rate.num;
330 avctx->
framerate.
den = format_out->es->video.frame_rate.den;
331 }
332
334
335 decoder->output[0]->buffer_size =
337 decoder->output[0]->buffer_num =
338 FFMAX(
decoder->output[0]->buffer_num_min,
decoder->output[0]->buffer_num_recommended) +
ctx->extra_buffers;
339 ctx->pool_out->pool = mmal_pool_create(
decoder->output[0]->buffer_num,
340 decoder->output[0]->buffer_size);
341 if (!
ctx->pool_out->pool) {
344 }
345
346 return 0;
347
350 }
351
353 {
356 MMAL_ES_FORMAT_T *format_in;
359
361
362 bcm_host_init();
363
364 if (mmal_vc_init()) {
367 }
368
371
373
374 if ((
status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &
ctx->decoder)))
376
378
379 format_in =
decoder->input[0]->format;
380 format_in->type = MMAL_ES_TYPE_VIDEO;
383 format_in->encoding = MMAL_ENCODING_MP2V;
384 break;
386 format_in->encoding = MMAL_ENCODING_MP4V;
387 break;
389 format_in->encoding = MMAL_ENCODING_WVC1;
390 break;
392 default:
393 format_in->encoding = MMAL_ENCODING_H264;
394 break;
395 }
398 format_in->es->video.crop.width = avctx->
width;
399 format_in->es->video.crop.height = avctx->
height;
400 format_in->es->video.frame_rate.num = 24000;
401 format_in->es->video.frame_rate.den = 1001;
404 format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED;
405
408
409 #if HAVE_MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS
410 if (mmal_port_parameter_set_uint32(
decoder->input[0], MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS,
411 -1 -
ctx->extra_decoder_buffers)) {
413 }
414 #endif
415
418
419 decoder->input[0]->buffer_num =
421 decoder->input[0]->buffer_size =
423 ctx->pool_in = mmal_pool_create(
decoder->input[0]->buffer_num, 0);
427 }
428
431
432 ctx->queue_decoded_frames = mmal_queue_create();
433 if (!
ctx->queue_decoded_frames)
435
436 decoder->input[0]->userdata = (
void*)avctx;
437 decoder->output[0]->userdata = (
void*)avctx;
438 decoder->control->userdata = (
void*)avctx;
439
446
449
450 return 0;
451
455 }
456
458 {
462
464
471
472 return;
473
476 }
477
478 // Split packets and add them to the waiting_buffers list. We don't queue them
479 // immediately, because it can happen that the decoder is temporarily blocked
480 // (due to us not reading/returning enough output buffers) and won't accept
481 // new input. (This wouldn't be an issue if MMAL input buffers always were
482 // complete frames - then the input buffer just would have to be big enough.)
483 // If is_extradata is set, send it as MMAL_BUFFER_HEADER_FLAG_CONFIG.
485 int is_extradata)
486 {
490 uint8_t *
data = (uint8_t *)
"";
491 uint8_t *start;
493
497 goto done;
501 if (!is_extradata)
503 } else {
505 goto done;
506 if (!
ctx->packets_sent) {
507 // Short-cut the flush logic to avoid upsetting MMAL.
509 ctx->eos_received = 1;
510 goto done;
511 }
512 }
513
515
516 do {
520 goto done;
521 }
522
525
526 if (is_extradata)
528
530 buffer->
flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_START;
531
534
537
539 buffer->
flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END;
541 }
542
546 }
547
548 if (buf) {
553 goto done;
554 }
555 }
556
557 // Insert at end of the list
558 if (!
ctx->waiting_buffers)
560 if (
ctx->waiting_buffers_tail)
561 ctx->waiting_buffers_tail->next =
buffer;
564
565 done:
568 }
569
570 // Move prepared/split packets from waiting_buffers to the MMAL decoder.
572 {
574
575 while (
ctx->waiting_buffers) {
576 MMAL_BUFFER_HEADER_T *mbuffer;
579
580 mbuffer = mmal_queue_get(
ctx->pool_in->queue);
581 if (!mbuffer)
582 return 0;
583
585
586 mmal_buffer_header_reset(mbuffer);
587 mbuffer->cmd = 0;
589 mbuffer->dts =
buffer->dts;
592 mbuffer->length =
buffer->length;
593 mbuffer->user_data =
buffer;
594 mbuffer->alloc_size =
ctx->decoder->input[0]->buffer_size;
595
596 // Remove from start of the list
599 ctx->waiting_buffers_tail =
NULL;
600
601 if ((
status = mmal_port_send_buffer(
ctx->decoder->input[0], mbuffer))) {
602 mmal_buffer_header_release(mbuffer);
604 if (
buffer->
flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
607 }
608
612 }
613 }
614
615 return 0;
616 }
617
619 MMAL_BUFFER_HEADER_T *
buffer)
620 {
623
624 frame->interlaced_frame =
ctx->interlaced_frame;
625 frame->top_field_first =
ctx->top_field_first;
626
629 return AVERROR_UNKNOWN;
// format change code failed with OOM previously
630
632 goto done;
633
635 goto done;
636 } else {
640 int linesize[4];
641
643 goto done;
644
650 }
651
657
660
661 done:
663 }
664
665 // Fetch a decoded buffer and place it into the frame parameter.
667 {
672
673 if (
ctx->eos_received)
674 goto done;
675
676 while (1) {
677 // To ensure decoding in lockstep with a constant delay between fed packets
678 // and output frames, we always wait until an output buffer is available.
679 // Except during start we don't know after how many input packets the decoder
680 // is going to return the first buffer, and we can't distinguish decoder
681 // being busy from decoder waiting for input. So just poll at the start and
682 // keep feeding new data to the buffer.
683 // We are pretty sure the decoder will produce output if we sent more input
684 // frames than what a H.264 decoder could logically delay. This avoids too
685 // excessive buffering.
686 // We also wait if we sent eos, but didn't receive it yet (think of decoding
687 // stream with a very low number of frames).
689 (
ctx->packets_sent &&
ctx->eos_sent)) {
690 // MMAL will ignore broken input packets, which means the frame we
691 // expect here may never arrive. Dealing with this correctly is
692 // complicated, so here's a hack to avoid that it freezes forever
693 // in this unlikely situation.
694 buffer = mmal_queue_timedwait(
ctx->queue_decoded_frames, 100);
698 goto done;
699 }
700 } else {
701 buffer = mmal_queue_get(
ctx->queue_decoded_frames);
703 goto done;
704 }
705
706 ctx->eos_received |= !!(
buffer->
flags & MMAL_BUFFER_HEADER_FLAG_EOS);
707 if (
ctx->eos_received)
708 goto done;
709
710 if (
buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) {
712 MMAL_EVENT_FORMAT_CHANGED_T *ev = mmal_event_format_changed_get(
buffer);
713 MMAL_BUFFER_HEADER_T *stale_buffer;
714
716
718 goto done;
719
720 while ((stale_buffer = mmal_queue_get(
ctx->queue_decoded_frames)))
721 mmal_buffer_header_release(stale_buffer);
722
723 mmal_format_copy(
decoder->output[0]->format, ev->format);
724
726 goto done;
727
729 goto done;
730
732 goto done;
733
735 goto done;
736
737 mmal_buffer_header_release(
buffer);
738 continue;
742 goto done;
743 }
else if (
buffer->length == 0) {
744 // Unused output buffer that got drained after format change.
745 mmal_buffer_header_release(
buffer);
746 continue;
747 }
748
749 ctx->frames_output++;
750
752 goto done;
753
754 *got_frame = 1;
755 break;
756 }
757
758 done:
760 mmal_buffer_header_release(
buffer);
764 }
765
767 {
771 int got_frame = 0;
772
776 ctx->extradata_sent = 1;
779 }
780
787
790
793
796
797 // ffmmal_read_frame() can block for a while. Since the decoder is
798 // asynchronous, it's a good idea to fill the ports again.
799
802
805
806 if (!got_frame &&
ret == 0)
808 else
810 }
811
815 };
816
819 {
"extra_decoder_buffers",
"extra MMAL internal buffered frames", offsetof(
MMALDecodeContext, extra_decoder_buffers),
AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
821 };
822
828 };
829
830 #define FFMMAL_DEC(NAME, ID) \
831 const AVCodec ff_##NAME##_mmal_decoder = { \
832 .name = #NAME "_mmal", \
833 .long_name = NULL_IF_CONFIG_SMALL(#NAME " (mmal)"), \
834 .type = AVMEDIA_TYPE_VIDEO, \
835 .id = ID, \
836 .priv_data_size = sizeof(MMALDecodeContext), \
837 .init = ffmmal_init_decoder, \
838 .close = ffmmal_close_decoder, \
839 .receive_frame = ffmmal_receive_frame, \
840 .flush = ffmmal_flush, \
841 .priv_class = &ffmmal_dec_class, \
842 .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, \
843 .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
844 .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_MMAL, \
845 AV_PIX_FMT_YUV420P, \
846 AV_PIX_FMT_NONE}, \
847 .hw_configs = mmal_hw_configs, \
848 .wrapper_name = "mmal", \
849 };
850