FFmpeg: libavdevice/avfoundation.m Source File

FFmpeg

[フレーム]

libavdevice

avfoundation.m

Go to the documentation of this file.

1 /*

2 * AVFoundation input device

4 *

5 * This file is part of FFmpeg.

6 *

7 * FFmpeg is free software; you can redistribute it and/or

8 * modify it under the terms of the GNU Lesser General Public

9 * License as published by the Free Software Foundation; either

10 * version 2.1 of the License, or (at your option) any later version.

11 *

12 * FFmpeg is distributed in the hope that it will be useful,

13 * but WITHOUT ANY WARRANTY; without even the implied warranty of

14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

15 * Lesser General Public License for more details.

16 *

17 * You should have received a copy of the GNU Lesser General Public

18 * License along with FFmpeg; if not, write to the Free Software

19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

20 */

22 /**

23 * @file

24 * AVFoundation input device

25 * @author Thilo Borgmann <thilo.borgmann@mail.de>

26 */

28 #import <AVFoundation/AVFoundation.h>

29 #include <pthread.h>

31 #include "libavutil/channel_layout.h"

32 #include "libavutil/pixdesc.h"

33 #include "libavutil/opt.h"

34 #include "libavutil/avstring.h"

35 #include "libavformat/internal.h"

36 #include "libavutil/internal.h"

37 #include "libavutil/parseutils.h"

38 #include "libavutil/time.h"

39 #include "libavutil/imgutils.h"

40 #include "avdevice.h"

42 static const int avf_time_base = 1000000;

44 static const AVRational avf_time_base_q = {

45 .num = 1,

46 .den = avf_time_base

47 };

49 struct AVFPixelFormatSpec {

50 enum AVPixelFormat ff_id;

51 OSType avf_id;

52 };

54 static const struct AVFPixelFormatSpec avf_pixel_formats[] = {

55 { AV_PIX_FMT_MONOBLACK, kCVPixelFormatType_1Monochrome },

56 { AV_PIX_FMT_RGB555BE, kCVPixelFormatType_16BE555 },

57 { AV_PIX_FMT_RGB555LE, kCVPixelFormatType_16LE555 },

58 { AV_PIX_FMT_RGB565BE, kCVPixelFormatType_16BE565 },

59 { AV_PIX_FMT_RGB565LE, kCVPixelFormatType_16LE565 },

60 { AV_PIX_FMT_RGB24, kCVPixelFormatType_24RGB },

61 { AV_PIX_FMT_BGR24, kCVPixelFormatType_24BGR },

62 { AV_PIX_FMT_0RGB, kCVPixelFormatType_32ARGB },

63 { AV_PIX_FMT_BGR0, kCVPixelFormatType_32BGRA },

64 { AV_PIX_FMT_0BGR, kCVPixelFormatType_32ABGR },

65 { AV_PIX_FMT_RGB0, kCVPixelFormatType_32RGBA },

66 { AV_PIX_FMT_BGR48BE, kCVPixelFormatType_48RGB },

67 { AV_PIX_FMT_UYVY422, kCVPixelFormatType_422YpCbCr8 },

68 { AV_PIX_FMT_YUVA444P, kCVPixelFormatType_4444YpCbCrA8R },

69 { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },

70 { AV_PIX_FMT_YUV444P, kCVPixelFormatType_444YpCbCr8 },

71 { AV_PIX_FMT_YUV422P16, kCVPixelFormatType_422YpCbCr16 },

72 { AV_PIX_FMT_YUV422P10, kCVPixelFormatType_422YpCbCr10 },

73 { AV_PIX_FMT_YUV444P10, kCVPixelFormatType_444YpCbCr10 },

74 { AV_PIX_FMT_YUV420P, kCVPixelFormatType_420YpCbCr8Planar },

75 { AV_PIX_FMT_NV12, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },

76 { AV_PIX_FMT_YUYV422, kCVPixelFormatType_422YpCbCr8_yuvs },

77 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080

78 { AV_PIX_FMT_GRAY8, kCVPixelFormatType_OneComponent8 },

79 #endif

80 { AV_PIX_FMT_NONE, 0 }

81 };

83 typedef struct

84 {

85 AVClass* class;

87 int frames_captured;

88 int audio_frames_captured;

89 pthread_mutex_t frame_lock;

90 id avf_delegate;

91 id avf_audio_delegate;

93 AVRational framerate;

94 int width, height;

96 int capture_cursor;

97 int capture_mouse_clicks;

98 int capture_raw_data;

99 int drop_late_frames;

100 int video_is_muxed;

101 int video_is_screen;

102

103 int list_devices;

104 int video_device_index;

105 int video_stream_index;

106 int audio_device_index;

107 int audio_stream_index;

108

109 char *url;

110 char *video_filename;

111 char *audio_filename;

112

113 int num_video_devices;

114

115 int audio_channels;

116 int audio_bits_per_sample;

117 int audio_float;

118 int audio_be;

119 int audio_signed_integer;

120 int audio_packed;

121 int audio_non_interleaved;

122

123 int32_t *audio_buffer;

124 int audio_buffer_size;

125

126 enum AVPixelFormat pixel_format;

127

128 AVCaptureSession *capture_session;

129 AVCaptureVideoDataOutput *video_output;

130 AVCaptureAudioDataOutput *audio_output;

131 CMSampleBufferRef current_frame;

132 CMSampleBufferRef current_audio_frame;

133

134 AVCaptureDevice *observed_device;

135 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

136 AVCaptureDeviceTransportControlsPlaybackMode observed_mode;

137 #endif

138 int observed_quit;

139 } AVFContext;

140

141 static void lock_frames(AVFContext* ctx)

142 {

143 pthread_mutex_lock(&ctx->frame_lock);

144 }

145

146 static void unlock_frames(AVFContext* ctx)

147 {

148 pthread_mutex_unlock(&ctx->frame_lock);

149 }

150

151 /** FrameReciever class - delegate for AVCaptureSession

152 */

153 @interface AVFFrameReceiver : NSObject

154 {

155 AVFContext* _context;

156 }

157

158 - (id)initWithContext:(AVFContext*)context;

159

160 - (void) captureOutput:(AVCaptureOutput *)captureOutput

161 didOutputSampleBuffer:(CMSampleBufferRef)videoFrame

162 fromConnection:(AVCaptureConnection *)connection;

163

164 @end

165

166 @implementation AVFFrameReceiver

167

168 - (id)initWithContext:(AVFContext*)context

169 {

170 if (self = [super init]) {

171 _context = context;

172

173 // start observing if a device is set for it

174 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

175 if (_context->observed_device) {

176 NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));

177 NSKeyValueObservingOptions options = NSKeyValueObservingOptionNew;

178

179 [_context->observed_device addObserver: self

180 forKeyPath: keyPath

181 options: options

182 context: _context];

183 }

184 #endif

185 }

186 return self;

187 }

188

189 - (void)dealloc {

190 // stop observing if a device is set for it

191 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

192 if (_context->observed_device) {

193 NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));

194 [_context->observed_device removeObserver: self forKeyPath: keyPath];

195 }

196 #endif

197 [super dealloc];

198 }

199

200 - (void)observeValueForKeyPath:(NSString *)keyPath

201 ofObject:(id)object

202 change:(NSDictionary *)change

203 context:(void *)context {

204 if (context == _context) {

205 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

206 AVCaptureDeviceTransportControlsPlaybackMode mode =

207 [change[NSKeyValueChangeNewKey] integerValue];

208

209 if (mode != _context->observed_mode) {

210 if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) {

211 _context->observed_quit = 1;

212 }

213 _context->observed_mode = mode;

214 }

215 #endif

216 } else {

217 [super observeValueForKeyPath: keyPath

218 ofObject: object

219 change: change

220 context: context];

221 }

222 }

223

224 - (void) captureOutput:(AVCaptureOutput *)captureOutput

225 didOutputSampleBuffer:(CMSampleBufferRef)videoFrame

226 fromConnection:(AVCaptureConnection *)connection

227 {

228 lock_frames(_context);

229

230 if (_context->current_frame != nil) {

231 CFRelease(_context->current_frame);

232 }

233

234 _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);

235

236 unlock_frames(_context);

237

238 ++_context->frames_captured;

239 }

240

241 @end

242

243 /** AudioReciever class - delegate for AVCaptureSession

244 */

245 @interface AVFAudioReceiver : NSObject

246 {

247 AVFContext* _context;

248 }

249

250 - (id)initWithContext:(AVFContext*)context;

251

252 - (void) captureOutput:(AVCaptureOutput *)captureOutput

253 didOutputSampleBuffer:(CMSampleBufferRef)audioFrame

254 fromConnection:(AVCaptureConnection *)connection;

255

256 @end

257

258 @implementation AVFAudioReceiver

259

260 - (id)initWithContext:(AVFContext*)context

261 {

262 if (self = [super init]) {

263 _context = context;

264 }

265 return self;

266 }

267

268 - (void) captureOutput:(AVCaptureOutput *)captureOutput

269 didOutputSampleBuffer:(CMSampleBufferRef)audioFrame

270 fromConnection:(AVCaptureConnection *)connection

271 {

272 lock_frames(_context);

273

274 if (_context->current_audio_frame != nil) {

275 CFRelease(_context->current_audio_frame);

276 }

277

278 _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);

279

280 unlock_frames(_context);

281

282 ++_context->audio_frames_captured;

283 }

284

285 @end

286

287 static void destroy_context(AVFContext* ctx)

288 {

289 [ctx->capture_session stopRunning];

290

291 [ctx->capture_session release];

292 [ctx->video_output release];

293 [ctx->audio_output release];

294 [ctx->avf_delegate release];

295 [ctx->avf_audio_delegate release];

296

297 ctx->capture_session = NULL;

298 ctx->video_output = NULL;

299 ctx->audio_output = NULL;

300 ctx->avf_delegate = NULL;

301 ctx->avf_audio_delegate = NULL;

302

303 av_freep(&ctx->url);

304 av_freep(&ctx->audio_buffer);

305

306 pthread_mutex_destroy(&ctx->frame_lock);

307

308 if (ctx->current_frame) {

309 CFRelease(ctx->current_frame);

310 }

311 }

312

313 static int parse_device_name(AVFormatContext *s)

314 {

315 AVFContext *ctx = (AVFContext*)s->priv_data;

316 char *save;

317

318 ctx->url = av_strdup(s->url);

319

320 if (!ctx->url)

321 return AVERROR(ENOMEM);

322 if (ctx->url[0] != ':') {

323 ctx->video_filename = av_strtok(ctx->url, ":", &save);

324 ctx->audio_filename = av_strtok(NULL, ":", &save);

325 } else {

326 ctx->audio_filename = av_strtok(ctx->url, ":", &save);

327 }

328 return 0;

329 }

330

331 /**

332 * Configure the video device.

333 *

334 * Configure the video device using a run-time approach to access properties

335 * since formats, activeFormat are available since iOS >= 7.0 or OSX >= 10.7

336 * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.

337 *

338 * The NSUndefinedKeyException must be handled by the caller of this function.

339 *

340 */

341 static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)

342 {

343 AVFContext *ctx = (AVFContext*)s->priv_data;

344

345 double framerate = av_q2d(ctx->framerate);

346 NSObject *range = nil;

347 NSObject *format = nil;

348 NSObject *selected_range = nil;

349 NSObject *selected_format = nil;

350

351 // try to configure format by formats list

352 // might raise an exception if no format list is given

353 // (then fallback to default, no configuration)

354 @try {

355 for (format in [video_device valueForKey:@"formats"]) {

356 CMFormatDescriptionRef formatDescription;

357 CMVideoDimensions dimensions;

358

359 formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];

360 dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);

361

362 if ((ctx->width == 0 && ctx->height == 0) ||

363 (dimensions.width == ctx->width && dimensions.height == ctx->height)) {

364

365 selected_format = format;

366

367 for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {

368 double max_framerate;

369

370 [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];

371 if (fabs (framerate - max_framerate) < 0.01) {

372 selected_range = range;

373 break;

374 }

375 }

376 }

377 }

378

379 if (!selected_format) {

380 av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device.\n",

381 ctx->width, ctx->height);

382 goto unsupported_format;

383 }

384

385 if (!selected_range) {

386 av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",

387 framerate);

388 if (ctx->video_is_muxed) {

389 av_log(s, AV_LOG_ERROR, "Falling back to default.\n");

390 } else {

391 goto unsupported_format;

392 }

393 }

394

395 if ([video_device lockForConfiguration:NULL] == YES) {

396 if (selected_format) {

397 [video_device setValue:selected_format forKey:@"activeFormat"];

398 }

399 if (selected_range) {

400 NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];

401 [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];

402 [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];

403 }

404 } else {

405 av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n");

406 return AVERROR(EINVAL);

407 }

408 } @catch(NSException *e) {

409 av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n");

410 }

411

412 return 0;

413

414 unsupported_format:

415

416 av_log(s, AV_LOG_ERROR, "Supported modes:\n");

417 for (format in [video_device valueForKey:@"formats"]) {

418 CMFormatDescriptionRef formatDescription;

419 CMVideoDimensions dimensions;

420

421 formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];

422 dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);

423

424 for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {

425 double min_framerate;

426 double max_framerate;

427

428 [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];

429 [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];

430 av_log(s, AV_LOG_ERROR, " %dx%d@[%f %f]fps\n",

431 dimensions.width, dimensions.height,

432 min_framerate, max_framerate);

433 }

434 }

435 return AVERROR(EINVAL);

436 }

437

438 static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)

439 {

440 AVFContext *ctx = (AVFContext*)s->priv_data;

441 int ret;

442 NSError *error = nil;

443 AVCaptureInput* capture_input = nil;

444 struct AVFPixelFormatSpec pxl_fmt_spec;

445 NSNumber *pixel_format;

446 NSDictionary *capture_dict;

447 dispatch_queue_t queue;

448

449 if (ctx->video_device_index < ctx->num_video_devices) {

450 capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];

451 } else {

452 capture_input = (AVCaptureInput*) video_device;

453 }

454

455 if (!capture_input) {

456 av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",

457 [[error localizedDescription] UTF8String]);

458 return 1;

459 }

460

461 if ([ctx->capture_session canAddInput:capture_input]) {

462 [ctx->capture_session addInput:capture_input];

463 } else {

464 av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");

465 return 1;

466 }

467

468 // Attaching output

469 ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];

470

471 if (!ctx->video_output) {

472 av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");

473 return 1;

474 }

475

476 // Configure device framerate and video size

477 @try {

478 if ((ret = configure_video_device(s, video_device)) < 0) {

479 return ret;

480 }

481 } @catch (NSException *exception) {

482 if (![[exception name] isEqualToString:NSUndefinedKeyException]) {

483 av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);

484 return AVERROR_EXTERNAL;

485 }

486 }

487

488 // select pixel format

489 pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;

490

491 for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {

492 if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {

493 pxl_fmt_spec = avf_pixel_formats[i];

494 break;

495 }

496 }

497

498 // check if selected pixel format is supported by AVFoundation

499 if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {

500 av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",

501 av_get_pix_fmt_name(pxl_fmt_spec.ff_id));

502 return 1;

503 }

504

505 // check if the pixel format is available for this device

506 if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {

507 av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",

508 av_get_pix_fmt_name(pxl_fmt_spec.ff_id));

509

510 pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;

511

512 av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");

513 for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {

514 struct AVFPixelFormatSpec pxl_fmt_dummy;

515 pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;

516 for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {

517 if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {

518 pxl_fmt_dummy = avf_pixel_formats[i];

519 break;

520 }

521 }

522

523 if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {

524 av_log(s, AV_LOG_ERROR, " %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));

525

526 // select first supported pixel format instead of user selected (or default) pixel format

527 if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {

528 pxl_fmt_spec = pxl_fmt_dummy;

529 }

530 }

531 }

532

533 // fail if there is no appropriate pixel format or print a warning about overriding the pixel format

534 if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {

535 return 1;

536 } else {

537 av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",

538 av_get_pix_fmt_name(pxl_fmt_spec.ff_id));

539 }

540 }

541

542 // set videoSettings to an empty dict for receiving raw data of muxed devices

543 if (ctx->capture_raw_data) {

544 ctx->pixel_format = pxl_fmt_spec.ff_id;

545 ctx->video_output.videoSettings = @{ };

546 } else {

547 ctx->pixel_format = pxl_fmt_spec.ff_id;

548 pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];

549 capture_dict = [NSDictionary dictionaryWithObject:pixel_format

550 forKey:(id)kCVPixelBufferPixelFormatTypeKey];

551

552 [ctx->video_output setVideoSettings:capture_dict];

553 }

554 [ctx->video_output setAlwaysDiscardsLateVideoFrames:ctx->drop_late_frames];

555

556 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

557 // check for transport control support and set observer device if supported

558 if (!ctx->video_is_screen) {

559 int trans_ctrl = [video_device transportControlsSupported];

560 AVCaptureDeviceTransportControlsPlaybackMode trans_mode = [video_device transportControlsPlaybackMode];

561

562 if (trans_ctrl) {

563 ctx->observed_mode = trans_mode;

564 ctx->observed_device = video_device;

565 }

566 }

567 #endif

568

569 ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];

570

571 queue = dispatch_queue_create("avf_queue", NULL);

572 [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];

573 dispatch_release(queue);

574

575 if ([ctx->capture_session canAddOutput:ctx->video_output]) {

576 [ctx->capture_session addOutput:ctx->video_output];

577 } else {

578 av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");

579 return 1;

580 }

581

582 return 0;

583 }

584

585 static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)

586 {

587 AVFContext *ctx = (AVFContext*)s->priv_data;

588 NSError *error = nil;

589 AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];

590 dispatch_queue_t queue;

591

592 if (!audio_dev_input) {

593 av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",

594 [[error localizedDescription] UTF8String]);

595 return 1;

596 }

597

598 if ([ctx->capture_session canAddInput:audio_dev_input]) {

599 [ctx->capture_session addInput:audio_dev_input];

600 } else {

601 av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");

602 return 1;

603 }

604

605 // Attaching output

606 ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];

607

608 if (!ctx->audio_output) {

609 av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");

610 return 1;

611 }

612

613 ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];

614

615 queue = dispatch_queue_create("avf_audio_queue", NULL);

616 [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];

617 dispatch_release(queue);

618

619 if ([ctx->capture_session canAddOutput:ctx->audio_output]) {

620 [ctx->capture_session addOutput:ctx->audio_output];

621 } else {

622 av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");

623 return 1;

624 }

625

626 return 0;

627 }

628

629 static int get_video_config(AVFormatContext *s)

630 {

631 AVFContext *ctx = (AVFContext*)s->priv_data;

632 CVImageBufferRef image_buffer;

633 CMBlockBufferRef block_buffer;

634 CGSize image_buffer_size;

635 AVStream* stream = avformat_new_stream(s, NULL);

636

637 if (!stream) {

638 return 1;

639 }

640

641 // Take stream info from the first frame.

642 while (ctx->frames_captured < 1) {

643 CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);

644 }

645

646 lock_frames(ctx);

647

648 ctx->video_stream_index = stream->index;

649

650 avpriv_set_pts_info(stream, 64, 1, avf_time_base);

651

652 image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);

653 block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);

654

655 if (image_buffer) {

656 image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);

657

658 stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;

659 stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;

660 stream->codecpar->width = (int)image_buffer_size.width;

661 stream->codecpar->height = (int)image_buffer_size.height;

662 stream->codecpar->format = ctx->pixel_format;

663 } else {

664 stream->codecpar->codec_id = AV_CODEC_ID_DVVIDEO;

665 stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;

666 stream->codecpar->format = ctx->pixel_format;

667 }

668

669 CFRelease(ctx->current_frame);

670 ctx->current_frame = nil;

671

672 unlock_frames(ctx);

673

674 return 0;

675 }

676

677 static int get_audio_config(AVFormatContext *s)

678 {

679 AVFContext *ctx = (AVFContext*)s->priv_data;

680 CMFormatDescriptionRef format_desc;

681 AVStream* stream = avformat_new_stream(s, NULL);

682

683 if (!stream) {

684 return 1;

685 }

686

687 // Take stream info from the first frame.

688 while (ctx->audio_frames_captured < 1) {

689 CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);

690 }

691

692 lock_frames(ctx);

693

694 ctx->audio_stream_index = stream->index;

695

696 avpriv_set_pts_info(stream, 64, 1, avf_time_base);

697

698 format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);

699 const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

700

701 if (!basic_desc) {

702 unlock_frames(ctx);

703 av_log(s, AV_LOG_ERROR, "audio format not available\n");

704 return 1;

705 }

706

707 stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;

708 stream->codecpar->sample_rate = basic_desc->mSampleRate;

709 stream->codecpar->channels = basic_desc->mChannelsPerFrame;

710 stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);

711

712 ctx->audio_channels = basic_desc->mChannelsPerFrame;

713 ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;

714 ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;

715 ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;

716 ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;

717 ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;

718 ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;

719

720 if (basic_desc->mFormatID == kAudioFormatLinearPCM &&

721 ctx->audio_float &&

722 ctx->audio_bits_per_sample == 32 &&

723 ctx->audio_packed) {

724 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;

725 } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&

726 ctx->audio_signed_integer &&

727 ctx->audio_bits_per_sample == 16 &&

728 ctx->audio_packed) {

729 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;

730 } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&

731 ctx->audio_signed_integer &&

732 ctx->audio_bits_per_sample == 24 &&

733 ctx->audio_packed) {

734 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;

735 } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&

736 ctx->audio_signed_integer &&

737 ctx->audio_bits_per_sample == 32 &&

738 ctx->audio_packed) {

739 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;

740 } else {

741 unlock_frames(ctx);

742 av_log(s, AV_LOG_ERROR, "audio format is not supported\n");

743 return 1;

744 }

745

746 if (ctx->audio_non_interleaved) {

747 CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);

748 ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);

749 ctx->audio_buffer = av_malloc(ctx->audio_buffer_size);

750 if (!ctx->audio_buffer) {

751 unlock_frames(ctx);

752 av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");

753 return 1;

754 }

755 }

756

757 CFRelease(ctx->current_audio_frame);

758 ctx->current_audio_frame = nil;

759

760 unlock_frames(ctx);

761

762 return 0;

763 }

764

765 static int avf_read_header(AVFormatContext *s)

766 {

767 int ret = 0;

768 NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];

769 uint32_t num_screens = 0;

770 AVFContext *ctx = (AVFContext*)s->priv_data;

771 AVCaptureDevice *video_device = nil;

772 AVCaptureDevice *audio_device = nil;

773 // Find capture device

774 NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];

775 NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];

776

777 ctx->num_video_devices = [devices count] + [devices_muxed count];

778

779 pthread_mutex_init(&ctx->frame_lock, NULL);

780

781 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

782 CGGetActiveDisplayList(0, NULL, &num_screens);

783 #endif

784

785 // List devices if requested

786 if (ctx->list_devices) {

787 int index = 0;

788 av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");

789 for (AVCaptureDevice *device in devices) {

790 const char *name = [[device localizedName] UTF8String];

791 index = [devices indexOfObject:device];

792 av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);

793 }

794 for (AVCaptureDevice *device in devices_muxed) {

795 const char *name = [[device localizedName] UTF8String];

796 index = [devices count] + [devices_muxed indexOfObject:device];

797 av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);

798 }

799 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

800 if (num_screens > 0) {

801 CGDirectDisplayID screens[num_screens];

802 CGGetActiveDisplayList(num_screens, screens, &num_screens);

803 for (int i = 0; i < num_screens; i++) {

804 av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);

805 }

806 }

807 #endif

808

809 av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");

810 devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];

811 for (AVCaptureDevice *device in devices) {

812 const char *name = [[device localizedName] UTF8String];

813 int index = [devices indexOfObject:device];

814 av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);

815 }

816 goto fail;

817 }

818

819 // parse input filename for video and audio device

820 ret = parse_device_name(s);

821 if (ret)

822 goto fail;

823

824 // check for device index given in filename

825 if (ctx->video_device_index == -1 && ctx->video_filename) {

826 sscanf(ctx->video_filename, "%d", &ctx->video_device_index);

827 }

828 if (ctx->audio_device_index == -1 && ctx->audio_filename) {

829 sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);

830 }

831

832 if (ctx->video_device_index >= 0) {

833 if (ctx->video_device_index < ctx->num_video_devices) {

834 if (ctx->video_device_index < [devices count]) {

835 video_device = [devices objectAtIndex:ctx->video_device_index];

836 } else {

837 video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];

838 ctx->video_is_muxed = 1;

839 }

840 } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {

841 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

842 CGDirectDisplayID screens[num_screens];

843 CGGetActiveDisplayList(num_screens, screens, &num_screens);

844 AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];

845

846 if (ctx->framerate.num > 0) {

847 capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);

848 }

849

850 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080

851 if (ctx->capture_cursor) {

852 capture_screen_input.capturesCursor = YES;

853 } else {

854 capture_screen_input.capturesCursor = NO;

855 }

856 #endif

857

858 if (ctx->capture_mouse_clicks) {

859 capture_screen_input.capturesMouseClicks = YES;

860 } else {

861 capture_screen_input.capturesMouseClicks = NO;

862 }

863

864 video_device = (AVCaptureDevice*) capture_screen_input;

865 ctx->video_is_screen = 1;

866 #endif

867 } else {

868 av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");

869 goto fail;

870 }

871 } else if (ctx->video_filename &&

872 strncmp(ctx->video_filename, "none", 4)) {

873 if (!strncmp(ctx->video_filename, "default", 7)) {

874 video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];

875 } else {

876 // looking for video inputs

877 for (AVCaptureDevice *device in devices) {

878 if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {

879 video_device = device;

880 break;

881 }

882 }

883 // looking for muxed inputs

884 for (AVCaptureDevice *device in devices_muxed) {

885 if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {

886 video_device = device;

887 ctx->video_is_muxed = 1;

888 break;

889 }

890 }

891

892 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070

893 // looking for screen inputs

894 if (!video_device) {

895 int idx;

896 if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {

897 CGDirectDisplayID screens[num_screens];

898 CGGetActiveDisplayList(num_screens, screens, &num_screens);

899 AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];

900 video_device = (AVCaptureDevice*) capture_screen_input;

901 ctx->video_device_index = ctx->num_video_devices + idx;

902 ctx->video_is_screen = 1;

903

904 if (ctx->framerate.num > 0) {

905 capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);

906 }

907

908 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080

909 if (ctx->capture_cursor) {

910 capture_screen_input.capturesCursor = YES;

911 } else {

912 capture_screen_input.capturesCursor = NO;

913 }

914 #endif

915

916 if (ctx->capture_mouse_clicks) {

917 capture_screen_input.capturesMouseClicks = YES;

918 } else {

919 capture_screen_input.capturesMouseClicks = NO;

920 }

921 }

922 }

923 #endif

924 }

925

926 if (!video_device) {

927 av_log(ctx, AV_LOG_ERROR, "Video device not found\n");

928 goto fail;

929 }

930 }

931

932 // get audio device

933 if (ctx->audio_device_index >= 0) {

934 NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];

935

936 if (ctx->audio_device_index >= [devices count]) {

937 av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");

938 goto fail;

939 }

940

941 audio_device = [devices objectAtIndex:ctx->audio_device_index];

942 } else if (ctx->audio_filename &&

943 strncmp(ctx->audio_filename, "none", 4)) {

944 if (!strncmp(ctx->audio_filename, "default", 7)) {

945 audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];

946 } else {

947 NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];

948

949 for (AVCaptureDevice *device in devices) {

950 if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {

951 audio_device = device;

952 break;

953 }

954 }

955 }

956

957 if (!audio_device) {

958 av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");

959 goto fail;

960 }

961 }

962

963 // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio

964 if (!video_device && !audio_device) {

965 av_log(s, AV_LOG_ERROR, "No AV capture device found\n");

966 goto fail;

967 }

968

969 if (video_device) {

970 if (ctx->video_device_index < ctx->num_video_devices) {

971 av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);

972 } else {

973 av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);

974 }

975 }

976 if (audio_device) {

977 av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);

978 }

979

980 // Initialize capture session

981 ctx->capture_session = [[AVCaptureSession alloc] init];

982

983 if (video_device && add_video_device(s, video_device)) {

984 goto fail;

985 }

986 if (audio_device && add_audio_device(s, audio_device)) {

987 }

988

989 [ctx->capture_session startRunning];

990

991 /* Unlock device configuration only after the session is started so it

992 * does not reset the capture formats */

993 if (!ctx->video_is_screen) {

994 [video_device unlockForConfiguration];

995 }

996

997 if (video_device && get_video_config(s)) {

998 goto fail;

999 }

1000

1001 // set audio stream

1002 if (audio_device && get_audio_config(s)) {

1003 goto fail;

1004 }

1005

1006 [pool release];

1007 return 0;

1008

1009 fail:

1010 [pool release];

1011 destroy_context(ctx);

1012 if (ret)

1013 return ret;

1014 return AVERROR(EIO);

1015 }

1016

1017 static int copy_cvpixelbuffer(AVFormatContext *s,

1018 CVPixelBufferRef image_buffer,

1019 AVPacket *pkt)

1020 {

1021 AVFContext *ctx = s->priv_data;

1022 int src_linesize[4];

1023 const uint8_t *src_data[4];

1024 int width = CVPixelBufferGetWidth(image_buffer);

1025 int height = CVPixelBufferGetHeight(image_buffer);

1026 int status;

1027

1028 memset(src_linesize, 0, sizeof(src_linesize));

1029 memset(src_data, 0, sizeof(src_data));

1030

1031 status = CVPixelBufferLockBaseAddress(image_buffer, 0);

1032 if (status != kCVReturnSuccess) {

1033 av_log(s, AV_LOG_ERROR, "Could not lock base address: %d (%dx%d)\n", status, width, height);

1034 return AVERROR_EXTERNAL;

1035 }

1036

1037 if (CVPixelBufferIsPlanar(image_buffer)) {

1038 size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);

1039 int i;

1040 for(i = 0; i < plane_count; i++){

1041 src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);

1042 src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);

1043 }

1044 } else {

1045 src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);

1046 src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);

1047 }

1048

1049 status = av_image_copy_to_buffer(pkt->data, pkt->size,

1050 src_data, src_linesize,

1051 ctx->pixel_format, width, height, 1);

1052

1053

1054

1055 CVPixelBufferUnlockBaseAddress(image_buffer, 0);

1056

1057 return status;

1058 }

1059

1060 static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)

1061 {

1062 AVFContext* ctx = (AVFContext*)s->priv_data;

1063

1064 do {

1065 CVImageBufferRef image_buffer;

1066 CMBlockBufferRef block_buffer;

1067 lock_frames(ctx);

1068

1069 if (ctx->current_frame != nil) {

1070 int status;

1071 int length = 0;

1072

1073 image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);

1074 block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);

1075

1076 if (image_buffer != nil) {

1077 length = (int)CVPixelBufferGetDataSize(image_buffer);

1078 } else if (block_buffer != nil) {

1079 length = (int)CMBlockBufferGetDataLength(block_buffer);

1080 } else {

1081 unlock_frames(ctx);

1082 return AVERROR(EINVAL);

1083 }

1084

1085 if (av_new_packet(pkt, length) < 0) {

1086 unlock_frames(ctx);

1087 return AVERROR(EIO);

1088 }

1089

1090 CMItemCount count;

1091 CMSampleTimingInfo timing_info;

1092

1093 if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {

1094 AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);

1095 pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);

1096 }

1097

1098 pkt->stream_index = ctx->video_stream_index;

1099 pkt->flags |= AV_PKT_FLAG_KEY;

1100

1101 if (image_buffer) {

1102 status = copy_cvpixelbuffer(s, image_buffer, pkt);

1103 } else {

1104 status = 0;

1105 OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);

1106 if (ret != kCMBlockBufferNoErr) {

1107 status = AVERROR(EIO);

1108 }

1109 }

1110 CFRelease(ctx->current_frame);

1111 ctx->current_frame = nil;

1112

1113 if (status < 0) {

1114 unlock_frames(ctx);

1115 return status;

1116 }

1117 } else if (ctx->current_audio_frame != nil) {

1118 CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);

1119 int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);

1120

1121 if (!block_buffer || !block_buffer_size) {

1122 unlock_frames(ctx);

1123 return AVERROR(EIO);

1124 }

1125

1126 if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {

1127 unlock_frames(ctx);

1128 return AVERROR_BUFFER_TOO_SMALL;

1129 }

1130

1131 if (av_new_packet(pkt, block_buffer_size) < 0) {

1132 unlock_frames(ctx);

1133 return AVERROR(EIO);

1134 }

1135

1136 CMItemCount count;

1137 CMSampleTimingInfo timing_info;

1138

1139 if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {

1140 AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);

1141 pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);

1142 }

1143

1144 pkt->stream_index = ctx->audio_stream_index;

1145 pkt->flags |= AV_PKT_FLAG_KEY;

1146

1147 if (ctx->audio_non_interleaved) {

1148 int sample, c, shift, num_samples;

1149

1150 OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);

1151 if (ret != kCMBlockBufferNoErr) {

1152 unlock_frames(ctx);

1153 return AVERROR(EIO);

1154 }

1155

1156 num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));

1157

1158 // transform decoded frame into output format

1159 #define INTERLEAVE_OUTPUT(bps) \

1160 { \

1161 int##bps##_t **src; \

1162 int##bps##_t *dest; \

1163 src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*)); \

1164 if (!src) { \

1165 unlock_frames(ctx); \

1166 return AVERROR(EIO); \

1167 } \

1168 \

1169 for (c = 0; c < ctx->audio_channels; c++) { \

1170 src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \

1171 } \

1172 dest = (int##bps##_t*)pkt->data; \

1173 shift = bps - ctx->audio_bits_per_sample; \

1174 for (sample = 0; sample < num_samples; sample++) \

1175 for (c = 0; c < ctx->audio_channels; c++) \

1176 *dest++ = src[c][sample] << shift; \

1177 av_freep(&src); \

1178 }

1179

1180 if (ctx->audio_bits_per_sample <= 16) {

1181 INTERLEAVE_OUTPUT(16)

1182 } else {

1183 INTERLEAVE_OUTPUT(32)

1184 }

1185 } else {

1186 OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);

1187 if (ret != kCMBlockBufferNoErr) {

1188 unlock_frames(ctx);

1189 return AVERROR(EIO);

1190 }

1191 }

1192

1193 CFRelease(ctx->current_audio_frame);

1194 ctx->current_audio_frame = nil;

1195 } else {

1196 pkt->data = NULL;

1197 unlock_frames(ctx);

1198 if (ctx->observed_quit) {

1199 return AVERROR_EOF;

1200 } else {

1201 return AVERROR(EAGAIN);

1202 }

1203 }

1204

1205 unlock_frames(ctx);

1206 } while (!pkt->data);

1207

1208 return 0;

1209 }

1210

1211 static int avf_close(AVFormatContext *s)

1212 {

1213 AVFContext* ctx = (AVFContext*)s->priv_data;

1214 destroy_context(ctx);

1215 return 0;

1216 }

1217

1218 static const AVOption options[] = {

1219 { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },

1220 { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },

1221 { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },

1222 { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},

1223 { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },

1224 { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },

1225 { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },

1226 { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },

1227 { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },

1228 { "drop_late_frames", "drop frames that are available later than expected", offsetof(AVFContext, drop_late_frames), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },

1229

1230 { NULL },

1231 };

1232

1233 static const AVClass avf_class = {

1234 .class_name = "AVFoundation indev",

1235 .item_name = av_default_item_name,

1236 .option = options,

1237 .version = LIBAVUTIL_VERSION_INT,

1238 .category = AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT,

1239 };

1240

1241 const AVInputFormat ff_avfoundation_demuxer = {

1242 .name = "avfoundation",

1243 .long_name = NULL_IF_CONFIG_SMALL("AVFoundation input device"),

1244 .priv_data_size = sizeof(AVFContext),

1245 .read_header = avf_read_header,

1246 .read_packet = avf_read_packet,

1247 .read_close = avf_close,

1248 .flags = AVFMT_NOFILE,

1249 .priv_class = &avf_class,

1250 };

error

static void error(const char *err)

Definition: target_bsf_fuzzer.c:31

AV_CODEC_ID_PCM_S16LE

@ AV_CODEC_ID_PCM_S16LE

Definition: codec_id.h:318

pthread_mutex_t

_fmutex pthread_mutex_t

Definition: os2threads.h:53

AV_LOG_WARNING

#define AV_LOG_WARNING

Something somehow does not look correct.

Definition: log.h:186

AV_CODEC_ID_PCM_F32BE

@ AV_CODEC_ID_PCM_F32BE

Definition: codec_id.h:338

AVPixelFormat

Pixel format.

Definition: pixfmt.h:64

name

it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name

Definition: writing_filters.txt:88

status

they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link

AVFContext::audio_buffer_size

int audio_buffer_size

Definition: avfoundation.m:124

AVERROR

Filter the word "frame" indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions

opt.h

avformat_new_stream

AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)

Add a new stream to a media file.

Definition: options.c:237

AVFContext::audio_float

int audio_float

Definition: avfoundation.m:117

AVFContext::observed_quit

int observed_quit

Definition: avfoundation.m:138

unlock_frames

static void unlock_frames(AVFContext *ctx)

Definition: avfoundation.m:146

AVERROR_EOF

#define AVERROR_EOF

End of file.

Definition: error.h:57

AV_OPT_TYPE_VIDEO_RATE

@ AV_OPT_TYPE_VIDEO_RATE

offset must point to AVRational

Definition: opt.h:238

pthread_mutex_init

static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)

Definition: os2threads.h:104

AV_CODEC_ID_RAWVIDEO

@ AV_CODEC_ID_RAWVIDEO

Definition: codec_id.h:63

AVFContext::current_audio_frame

CMSampleBufferRef current_audio_frame

Definition: avfoundation.m:132

av_get_default_channel_layout

int64_t av_get_default_channel_layout(int nb_channels)

Return default channel layout for a given number of channels.

Definition: channel_layout.c:323

pixdesc.h

AVFContext::audio_frames_captured

int audio_frames_captured

Definition: avfoundation.m:88

AVPacket::data

uint8_t * data

Definition: packet.h:374

AVOption

AVOption.

Definition: opt.h:251

AV_PIX_FMT_BGR24

@ AV_PIX_FMT_BGR24

packed RGB 8:8:8, 24bpp, BGRBGR...

Definition: pixfmt.h:69

AV_PKT_FLAG_KEY

#define AV_PKT_FLAG_KEY

The packet contains a keyframe.

Definition: packet.h:429

parse_device_name

static int parse_device_name(AVFormatContext *s)

Definition: avfoundation.m:313

AV_PIX_FMT_RGB555BE

@ AV_PIX_FMT_RGB555BE

packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), big-endian , X=unused/undefined

Definition: pixfmt.h:107

AVFContext::audio_channels

int audio_channels

Definition: avfoundation.m:115

av_malloc

#define av_malloc(s)

Definition: tableprint_vlc.h:30

AVFContext::video_filename

char * video_filename

Definition: avfoundation.m:110

AVFPixelFormatSpec::avf_id

OSType avf_id

Definition: avfoundation.m:51

init

static int init

Definition: av_tx.c:47

AVFContext::audio_be

int audio_be

Definition: avfoundation.m:118

framerate

int framerate

Definition: h264_levels.c:65

AVFContext::capture_cursor

int capture_cursor

Definition: avfoundation.m:96

avpriv_set_pts_info

void avpriv_set_pts_info(AVStream *st, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)

Set the time base and wrapping info for a given stream.

Definition: avformat.c:697

AV_CODEC_ID_PCM_S16BE

@ AV_CODEC_ID_PCM_S16BE

Definition: codec_id.h:319

fail

#define fail()

Definition: checkasm.h:131

avf_close

static int avf_close(AVFormatContext *s)

Definition: avfoundation.m:1211

avf_time_base

static const int avf_time_base

Definition: avfoundation.m:42

read_close

static av_cold int read_close(AVFormatContext *ctx)

Definition: libcdio.c:143

AVFContext::current_frame

CMSampleBufferRef current_frame

Definition: avfoundation.m:131

AVFPixelFormatSpec::ff_id

enum AVPixelFormat ff_id

Definition: avfoundation.m:50

AVFContext::observed_device

AVCaptureDevice * observed_device

Definition: avfoundation.m:134

AVERROR_BUFFER_TOO_SMALL

#define AVERROR_BUFFER_TOO_SMALL

Buffer too small.

Definition: error.h:53

AVRational::num

int num

Numerator.

Definition: rational.h:59

AVFContext::framerate

AVRational framerate

Definition: avfoundation.m:93

AV_PIX_FMT_YUV444P10

#define AV_PIX_FMT_YUV444P10

Definition: pixfmt.h:409

description

Tag description

Definition: snow.txt:206

pkt

AVPacket * pkt

Definition: movenc.c:59

AV_LOG_ERROR

#define AV_LOG_ERROR

Something went wrong and cannot losslessly be recovered.

Definition: log.h:180

avf_time_base_q

static const AVRational avf_time_base_q

Definition: avfoundation.m:44

AVInputFormat

Definition: avformat.h:656

AV_PIX_FMT_YUV422P16

#define AV_PIX_FMT_YUV422P16

Definition: pixfmt.h:418

AVFContext::num_video_devices

int num_video_devices

Definition: avfoundation.m:113

INTERLEAVE_OUTPUT

#define INTERLEAVE_OUTPUT(bps)

width

#define width

#define s(width, name)

Definition: cbs_vp9.c:256

av_new_packet

int av_new_packet(AVPacket *pkt, int size)

Allocate the payload of a packet and initialize its fields with default values.

Definition: avpacket.c:97

format

AVInputFormat::name

const char * name

A comma separated list of short names for the format.

Definition: avformat.h:661

AVFAudioReceiver::_context

AVFContext * _context

Definition: avfoundation.m:247

options

static const AVOption options[]

Definition: avfoundation.m:1218

add_audio_device

static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)

Definition: avfoundation.m:585

AVMEDIA_TYPE_AUDIO

@ AVMEDIA_TYPE_AUDIO

Definition: avutil.h:202

av_q2d

static double av_q2d(AVRational a)

Convert an AVRational to a double.

Definition: rational.h:104

av_strtok

char * av_strtok(char *s, const char *delim, char **saveptr)

Split the string into several tokens which can be accessed by successive calls to av_strtok().

Definition: avstring.c:189

AVFContext::capture_mouse_clicks

int capture_mouse_clicks

Definition: avfoundation.m:97

AV_LOG_DEBUG

#define AV_LOG_DEBUG

Stuff which is only useful for libav* developers.

Definition: log.h:201

ctx

AVFormatContext * ctx

Definition: movenc.c:48

AVFContext::frame_lock

pthread_mutex_t frame_lock

Definition: avfoundation.m:89

av_rescale_q

int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)

Rescale a 64-bit integer by 2 rational numbers.

Definition: mathematics.c:142

AVFContext::capture_raw_data

int capture_raw_data

Definition: avfoundation.m:98

AVFContext::list_devices

int list_devices

Definition: avfoundation.m:103

AV_PIX_FMT_YUV420P

@ AV_PIX_FMT_YUV420P

planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)

Definition: pixfmt.h:66

AVFPixelFormatSpec

Definition: avfoundation.m:49

get_video_config

static int get_video_config(AVFormatContext *s)

Definition: avfoundation.m:629

if(ret)

Definition: filter_design.txt:179

AVFContext::audio_packed

int audio_packed

Definition: avfoundation.m:120

AVFFrameReceiver::_context

AVFContext * _context

Definition: avfoundation.m:155

context

it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your context

Definition: writing_filters.txt:91

AVFormatContext

Format I/O context.

Definition: avformat.h:1213

internal.h

AVFContext::video_output

AVCaptureVideoDataOutput * video_output

Definition: avfoundation.m:129

AVFContext::audio_signed_integer

int audio_signed_integer

Definition: avfoundation.m:119

AV_PIX_FMT_RGB565LE

@ AV_PIX_FMT_RGB565LE

packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), little-endian

Definition: pixfmt.h:106

LIBAVUTIL_VERSION_INT

#define LIBAVUTIL_VERSION_INT

Definition: version.h:85

read_header

static int read_header(FFV1Context *f)

Definition: ffv1dec.c:532

AVClass

Describe the class of an AVClass context structure.

Definition: log.h:66

fabs

static __device__ float fabs(float a)

Definition: cuda_runtime.h:182

NULL

#define NULL

Definition: coverity.c:32

AVFContext::drop_late_frames

int drop_late_frames

Definition: avfoundation.m:99

AV_PIX_FMT_YUYV422

@ AV_PIX_FMT_YUYV422

packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr

Definition: pixfmt.h:67

add_video_device

static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)

Definition: avfoundation.m:438

AVRational

Rational number (pair of numerator and denominator).

Definition: rational.h:58

AVFFrameReceiver

FrameReciever class - delegate for AVCaptureSession.

Definition: avfoundation.m:153

AV_PIX_FMT_MONOBLACK

@ AV_PIX_FMT_MONOBLACK

Y , 1bpp, 0 is black, 1 is white, in each byte pixels are ordered from the msb to the lsb.

Definition: pixfmt.h:76

AV_OPT_TYPE_IMAGE_SIZE

@ AV_OPT_TYPE_IMAGE_SIZE

offset must point to two consecutive integers

Definition: opt.h:235

av_default_item_name

const char * av_default_item_name(void *ptr)

Return the context name.

Definition: log.c:237

parseutils.h

AV_PIX_FMT_BGR0

@ AV_PIX_FMT_BGR0

packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined

Definition: pixfmt.h:230

time.h

AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT

@ AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT

Definition: log.h:41

AV_PIX_FMT_YUV422P10

#define AV_PIX_FMT_YUV422P10

Definition: pixfmt.h:407

AV_PIX_FMT_GRAY8

@ AV_PIX_FMT_GRAY8

Y , 8bpp.

Definition: pixfmt.h:74

avf_read_packet

static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)

Definition: avfoundation.m:1060

AVFContext::width

int width

Definition: avfoundation.m:94

configure_video_device

static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)

Configure the video device.

Definition: avfoundation.m:341

index

int index

Definition: gxfenc.c:89

Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c

Definition: undefined.txt:32

pthread_mutex_unlock

#define pthread_mutex_unlock(a)

Definition: ffprobe.c:77

AVFContext::audio_buffer

int32_t * audio_buffer

Definition: avfoundation.m:123

AVFContext::video_stream_index

int video_stream_index

Definition: avfoundation.m:105

AV_CODEC_ID_PCM_S24LE

@ AV_CODEC_ID_PCM_S24LE

Definition: codec_id.h:330

AV_PIX_FMT_RGB24

@ AV_PIX_FMT_RGB24

packed RGB 8:8:8, 24bpp, RGBRGB...

Definition: pixfmt.h:68

AVPacket::size

int size

Definition: packet.h:375

NULL_IF_CONFIG_SMALL

#define NULL_IF_CONFIG_SMALL(x)

Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.

Definition: internal.h:117

enum AVCodecID id

Definition: extract_extradata_bsf.c:324

destroy_context

static void destroy_context(AVFContext *ctx)

Definition: avfoundation.m:287

AVFContext::url

char * url

Definition: avfoundation.m:109

AVFormatContext::url

char * url

input or output URL.

Definition: avformat.h:1296

sample

#define sample

Definition: flacdsp_template.c:44

av_make_q

static AVRational av_make_q(int num, int den)

Create an AVRational.

Definition: rational.h:71

ff_avfoundation_demuxer

const AVInputFormat ff_avfoundation_demuxer

Definition: avfoundation.m:1241

AVFMT_NOFILE

#define AVFMT_NOFILE

Demuxer will use avio_open, no opened file should be provided by the caller.

Definition: avformat.h:470

AVFContext::audio_non_interleaved

int audio_non_interleaved

Definition: avfoundation.m:121

avdevice.h

AVPacket::dts

int64_t dts

Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed.

Definition: packet.h:373

height

#define height

AV_PIX_FMT_YUVA444P

@ AV_PIX_FMT_YUVA444P

planar YUV 4:4:4 32bpp, (1 Cr & Cb sample per 1x1 Y & A samples)

Definition: pixfmt.h:167

AVERROR_EXTERNAL

#define AVERROR_EXTERNAL

Generic error in an external library.

Definition: error.h:59

AVPacket::flags

int flags

A combination of AV_PKT_FLAG values.

Definition: packet.h:380

AV_PIX_FMT_RGB0

@ AV_PIX_FMT_RGB0

packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined

Definition: pixfmt.h:228

AV_LOG_INFO

#define AV_LOG_INFO

Standard information.

Definition: log.h:191

pthread_mutex_destroy

static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)

Definition: os2threads.h:112

lock_frames

static void lock_frames(AVFContext *ctx)

Definition: avfoundation.m:141

AVFContext::audio_stream_index

int audio_stream_index

Definition: avfoundation.m:107

copy_cvpixelbuffer

static int copy_cvpixelbuffer(AVFormatContext *s, CVPixelBufferRef image_buffer, AVPacket *pkt)

Definition: avfoundation.m:1017

AV_PIX_FMT_RGB555LE

@ AV_PIX_FMT_RGB555LE

packed RGB 5:5:5, 16bpp, (msb)1X 5R 5G 5B(lsb), little-endian, X=unused/undefined

Definition: pixfmt.h:108

AVFContext::audio_bits_per_sample

int audio_bits_per_sample

Definition: avfoundation.m:116

#define i(width, name, range_min, range_max)

Definition: cbs_h2645.c:269

AVPacket::pts

int64_t pts

Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...

Definition: packet.h:367

avf_read_header

static int avf_read_header(AVFormatContext *s)

Definition: avfoundation.m:765

internal.h

AV_OPT_FLAG_DECODING_PARAM

#define AV_OPT_FLAG_DECODING_PARAM

a generic parameter which can be set by the user for demuxing or decoding

Definition: opt.h:282

AV_CODEC_ID_DVVIDEO

@ AV_CODEC_ID_DVVIDEO

Definition: codec_id.h:74

AV_CODEC_ID_PCM_S32BE

@ AV_CODEC_ID_PCM_S32BE

Definition: codec_id.h:327

AVFContext::frames_captured

int frames_captured

Definition: avfoundation.m:87

AVFContext::video_is_muxed

int video_is_muxed

Definition: avfoundation.m:100

ret

Definition: filter_design.txt:187

read_packet

static int read_packet(void *opaque, uint8_t *buf, int buf_size)

Definition: avio_reading.c:42

AVStream

Stream structure.

Definition: avformat.h:948

AV_PIX_FMT_0BGR

@ AV_PIX_FMT_0BGR

packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined

Definition: pixfmt.h:229

AV_PIX_FMT_NV12

@ AV_PIX_FMT_NV12

planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...

Definition: pixfmt.h:89

AVFContext::audio_device_index

int audio_device_index

Definition: avfoundation.m:106

AVClass::class_name

const char * class_name

The name of the class; usually it is the same name as the context structure type to which the AVClass...

Definition: log.h:71

avf_pixel_formats

static const struct AVFPixelFormatSpec avf_pixel_formats[]

Definition: avfoundation.m:54

AVFContext::audio_output

AVCaptureAudioDataOutput * audio_output

Definition: avfoundation.m:130

AV_PIX_FMT_UYVY422

@ AV_PIX_FMT_UYVY422

packed YUV 4:2:2, 16bpp, Cb Y0 Cr Y1

Definition: pixfmt.h:81

AVFContext::avf_audio_delegate

id avf_audio_delegate

Definition: avfoundation.m:91

channel_layout.h

AVFContext::video_is_screen

int video_is_screen

Definition: avfoundation.m:101

mode

Definition: ebur128.h:83

AV_PIX_FMT_NONE

@ AV_PIX_FMT_NONE

Definition: pixfmt.h:65

AV_OPT_TYPE_INT

@ AV_OPT_TYPE_INT

Definition: opt.h:225

AV_OPT_TYPE_PIXEL_FMT

@ AV_OPT_TYPE_PIXEL_FMT

Definition: opt.h:236

AVPacket::stream_index

int stream_index

Definition: packet.h:376

AV_PIX_FMT_YUV444P

@ AV_PIX_FMT_YUV444P

planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)

Definition: pixfmt.h:71

AVFContext::audio_filename

char * audio_filename

Definition: avfoundation.m:111

AV_PIX_FMT_RGB565BE

@ AV_PIX_FMT_RGB565BE

packed RGB 5:6:5, 16bpp, (msb) 5R 6G 5B(lsb), big-endian

Definition: pixfmt.h:105

shift

static int shift(int a, int b)

Definition: sonic.c:88

av_strdup

char * av_strdup(const char *s)

Duplicate a string.

Definition: mem.c:280

AVMEDIA_TYPE_VIDEO

@ AVMEDIA_TYPE_VIDEO

Definition: avutil.h:201

AV_CODEC_ID_PCM_S32LE

@ AV_CODEC_ID_PCM_S32LE

Definition: codec_id.h:326

get_audio_config

static int get_audio_config(AVFormatContext *s)

Definition: avfoundation.m:677

AVFContext

Definition: avfoundation.m:83

timing_info

static int FUNC() timing_info(CodedBitstreamContext *ctx, RWContext *rw, AV1RawTimingInfo *current)

Definition: cbs_av1_syntax_template.c:158

av_image_copy_to_buffer

int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)

Copy image data from an image into a buffer.

Definition: imgutils.c:501

AVPacket

This structure stores compressed data.

Definition: packet.h:351

AV_OPT_TYPE_BOOL

@ AV_OPT_TYPE_BOOL

Definition: opt.h:244

av_freep

#define av_freep(p)

Definition: tableprint_vlc.h:34

int32_t