1 /*
2 * Depacketization for RTP Payload Format For AV1 (v1.0)
3 * https://aomediacodec.github.io/av1-rtp-spec/
4 * Copyright (c) 2024 Axis Communications
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 /**
24 * @file
25 * @brief AV1 / RTP depacketization code (RTP Payload Format For AV1 (v1.0))
26 * @author Chris Hodges <chris.hodges@axis.com>
27 * @note The process will restore TDs and put back size fields into headers.
28 * It will also try to keep complete OBUs and remove partial OBUs
29 * caused by packet drops and thus keep the stream syntactically intact.
30 */
31
35
39
40 // enable tracing of packet data
41 //#define RTPDEC_AV1_VERBOSE_TRACE
42
43 /**
44 * RTP/AV1 specific private data.
45 */
47 uint32_t
timestamp;
///< last received timestamp for frame
48 uint8_t
profile;
///< profile (main/high/professional)
50 uint8_t
tier;
///< main tier or high tier
51 uint16_t
prev_seq;
///< sequence number of previous packet
56 int needs_td;
///< indicates that a TD should be output
60 };
61
65 const char *attr,
const char *
value) {
66 if (!strcmp(attr, "profile")) {
69 } else if (!strcmp(attr, "level-idx")) {
72 } else if (!strcmp(attr, "tier")) {
75 }
76 return 0;
77 }
78
79 // return 0 on complete packet, -1 on partial packet
82 const uint8_t *buf,
int len, uint16_t seq,
84 uint8_t aggr_hdr;
86 int is_frag_cont;
87 int is_last_fragmented;
88 int is_first_pkt;
89 unsigned int num_obus;
90 unsigned int obu_cnt = 1;
91 unsigned int rem_pkt_size =
len;
92 unsigned int pktpos;
93 const uint8_t *buf_ptr = buf;
94 uint16_t expected_seq =
data->prev_seq + 1;
95 int16_t seq_diff = seq - expected_seq;
96
98
102 }
106 }
107
108 /* The payload structure is supposed to be straight-forward, but there are a
109 * couple of edge cases which need to be tackled and make things a bit more
110 * complex.
111 * These are mainly due to:
112 * - To reconstruct the OBU size for fragmented packets and place it the OBU
113 * header, the final size will not be known until the last fragment has
114 * been parsed. However, the number LEBs in the header is variable
115 * depending on the length of the payload.
116 * - We are increasing the out-packet size while we are getting fragmented
117 * OBUs. If an RTP packet gets dropped, we would create corrupted OBUs.
118 * In this case we decide to drop the whole frame.
119 */
120
121 #ifdef RTPDEC_AV1_VERBOSE_TRACE
127 #endif
128
129 /* 8 bit aggregate header: Z Y W W N - - - */
130 aggr_hdr = *buf_ptr++;
131 rem_pkt_size--;
132
133 /* Z: MUST be set to 1 if the first OBU element is an OBU fragment that is a
134 * continuation of an OBU fragment from the previous packet, and MUST be set
135 * to 0 otherwise */
137
138 /* Y: MUST be set to 1 if the last OBU element is an OBU fragment that will
139 * continue in the next packet, and MUST be set to 0 otherwise */
141
142 /* W: two bit field that describes the number of OBU elements in the packet.
143 * This field MUST be set equal to 0 or equal to the number of OBU elements
144 * contained in the packet.
145 * If set to 0, each OBU element MUST be preceded by a length field.
146 * If not set to 0 (i.e., W = 1, 2 or 3) the last OBU element MUST NOT be
147 * preceded by a length field (it's derived from RTP packet size minus other
148 * known lengths). */
150
151 /* N: MUST be set to 1 if the packet is the first packet of a coded video
152 * sequence, and MUST be set to 0 otherwise.*/
154
155 if (is_frag_cont) {
156 if (
data->drop_fragment) {
158 }
159 if (is_first_pkt) {
162 }
163 if (seq_diff) {
165 seq, expected_seq);
166 goto drop_fragment;
167 }
170 goto drop_fragment; // avoid repeated output for the same fragment
171 }
172 } else {
173 if (!is_first_pkt && !
data->keyframe_seen) {
174 if (!
data->wait_for_keyframe) {
175 data->wait_for_keyframe = 1;
177 }
178 goto drop_fragment;
179 }
180 if (seq_diff && !is_first_pkt) {
182 seq, expected_seq);
183 goto drop_fragment;
184 }
185 data->drop_fragment = 0;
186 if (!
data->needs_td && ((
data->timestamp != *timestamp) || is_first_pkt)) {
187 av_log(
ctx,
AV_LOG_TRACE,
"Timestamp changed to %u (or first pkt %d), forcing TD\n", *timestamp, is_first_pkt);
189 data->frag_obu_size = 0;
// new temporal unit might have been caused by dropped packets
190 }
191 if (
data->frag_obu_size) {
192 data->frag_obu_size = 0;
// make sure we recover
195 }
196 // update the timestamp in the frame packet with the one from the RTP packet
197 data->timestamp = *timestamp;
198 }
200
201 #ifdef RTPDEC_AV1_VERBOSE_TRACE
203 len, aggr_hdr, is_frag_cont, is_last_fragmented, num_obus, is_first_pkt);
204 #endif
205
206 if (is_first_pkt) {
208 data->keyframe_seen = 1;
209 data->wait_for_keyframe = 0;
210 }
211
212 // loop over OBU elements
213 while (rem_pkt_size) {
214 uint32_t obu_size;
215 int num_lebs;
216 int needs_size_field;
217 int output_size;
218 unsigned int obu_payload_size;
219 uint8_t obu_hdr;
220
221 obu_size = rem_pkt_size;
222 if (!num_obus || obu_cnt < num_obus) {
223 // read out explicit OBU element size (which almost corresponds to the original OBU size)
224 num_lebs =
parse_leb(
ctx, buf_ptr, rem_pkt_size, &obu_size);
225 if (!num_lebs) {
227 }
228 rem_pkt_size -= num_lebs;
229 buf_ptr += num_lebs;
230 }
231 // read first byte (which is the header byte only for non-fragmented elements)
232 obu_hdr = *buf_ptr;
233 if (obu_size > rem_pkt_size) {
234 av_log(
ctx,
AV_LOG_ERROR,
"AV1 OBU size %u larger than remaining pkt size %d\n", obu_size, rem_pkt_size);
236 }
237
238 if (!obu_size) {
241 }
242
243 if (!is_frag_cont) {
248 }
249 // ignore and remove OBUs according to spec
252 pktpos += obu_size;
253 rem_pkt_size -= obu_size;
254 // TODO: This probably breaks if the OBU_TILE_LIST is fragmented
255 // into the next RTP packet, so at least check and fail here
256 if (rem_pkt_size == 0 && is_last_fragmented) {
259 }
260 obu_cnt++;
261 continue;
262 }
263 }
264
265 // If we need to add a size field, out size will be different
266 output_size = obu_size;
267 // Spec says the OBUs should have their size fields removed,
268 // but this is not mandatory
270 needs_size_field = 0;
271 } else {
272 needs_size_field = 1;
273 // (re)calculate number of LEB bytes needed (if it was implicit, there were no LEBs)
275 }
276
277 if (!is_frag_cont && (obu_cnt == 1)) {
278 if (
data->needs_td) {
279 output_size += 2; // for Temporal Delimiter (TD)
280 }
284 } else {
287 }
288
289 if (
data->needs_td) {
290 // restore TD
293 }
295 } else {
298 }
299
300 obu_payload_size = obu_size;
301 // do we need to restore the OBU size field?
302 if (needs_size_field) {
303 // set obu_has_size_field in header byte
305 data->frag_header_size = 1;
306 obu_payload_size--;
307
308 // copy extension byte, if available
310 /* TODO we cannot handle the edge case where last element is a
311 * fragment of exactly one byte AND the header has the extension
312 * flag set. Note that it would be more efficient to not send a
313 * fragment of one byte and instead drop the size field of the
314 * prior element */
315 if (!obu_payload_size) {
317 obu_hdr);
319 }
320 pkt->
data[pktpos++] = *buf_ptr++;
321 data->frag_header_size = 2;
322 obu_payload_size--;
323 }
324
325 // remember start position of LEB for possibly fragmented packet to
326 // fixup OBU size later
327 data->frag_pkt_leb_pos = pktpos;
328 // write intermediate OBU size field
330 data->frag_lebs_res = num_lebs;
331 pktpos += num_lebs;
332 }
333 // copy verbatim or without above header size patch
334 memcpy(
pkt->
data + pktpos, buf_ptr, obu_payload_size);
335 pktpos += obu_payload_size;
336 buf_ptr += obu_payload_size;
337 rem_pkt_size -= obu_size;
338
339 // if we were handling a fragmented packet and this was the last
340 // fragment, correct OBU size field
341 if (
data->frag_obu_size && (rem_pkt_size || !is_last_fragmented)) {
342 uint32_t final_obu_size =
data->frag_obu_size + obu_size -
data->frag_header_size;
343 uint8_t *lebptr =
pkt->
data +
data->frag_pkt_leb_pos;
345
346 // check if we had allocated enough LEB bytes in header,
347 // otherwise make some extra space
348 if (num_lebs >
data->frag_lebs_res) {
349 int extra_bytes = num_lebs -
data->frag_lebs_res;
352 // update pointer in case buffer address changed
354 // move existing data for OBU back a bit
355 memmove(lebptr + extra_bytes, lebptr,
357 // move pktpos further down for following OBUs in same packet.
358 pktpos += extra_bytes;
359 }
360
361 // update OBU size field
363
364 data->frag_obu_size = 0;
// signal end of fragment
365 } else if (is_last_fragmented && !rem_pkt_size) {
366 // add to total OBU size, so we can fix that in OBU header
367 // (but only if the OBU size was missing!)
368 if (needs_size_field ||
data->frag_obu_size) {
369 data->frag_obu_size += obu_size;
370 }
371 // fragment not yet finished!
373 }
374 is_frag_cont = 0;
375
376 if (!rem_pkt_size && num_obus && (num_obus != obu_cnt)) {
378 num_obus, obu_cnt);
379 }
380 obu_cnt++;
381 }
382
386 } else {
387 // fragment may be complete, but temporal unit is not yet finished
389 }
390
391 if (!is_last_fragmented) {
392 data->frag_obu_size = 0;
393 data->frag_pkt_leb_pos = 0;
394 }
395
396 #ifdef RTPDEC_AV1_VERBOSE_TRACE
402 }
403 #endif
405
407
408 drop_fragment:
409 data->keyframe_seen = 0;
410 data->drop_fragment = 1;
411 data->frag_obu_size = 0;
415 // we can't seem to deallocate the fragmented packet, but we can shrink it to 0
417 }
419 }
420
422 }
423
425 {
426 return !
data->keyframe_seen;
427 }
428
432 const char *
p =
line;
434
435 if (st_index < 0)
436 return 0;
437
438 stream =
s->streams[st_index];
439
440 /* Optional parameters are profile, level-idx, and tier.
441 * See Section 7.2.1 of https://aomediacodec.github.io/av1-rtp-spec/ */
446 }
447
449 }
450
461 };