FFmpeg: libavfilter/dnn/dnn_backend_torch.cpp Source File

FFmpeg

[フレーム]

dnn_backend_torch.cpp

Go to the documentation of this file.

1 /*

3 *

4 * This file is part of FFmpeg.

5 *

6 * FFmpeg is free software; you can redistribute it and/or

7 * modify it under the terms of the GNU Lesser General Public

8 * License as published by the Free Software Foundation; either

9 * version 2.1 of the License, or (at your option) any later version.

10 *

11 * FFmpeg is distributed in the hope that it will be useful,

12 * but WITHOUT ANY WARRANTY; without even the implied warranty of

13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

14 * Lesser General Public License for more details.

15 *

16 * You should have received a copy of the GNU Lesser General Public

17 * License along with FFmpeg; if not, write to the Free Software

18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

19 */

21 /**

22 * @file

23 * DNN Torch backend implementation.

24 */

26 #include <torch/torch.h>

27 #include <torch/script.h>

29 extern "C" {

30 #include "dnn_io_proc.h"

31 #include "dnn_backend_common.h"

32 #include "libavutil/opt.h"

33 #include "libavutil/mem.h"

34 #include "queue.h"

35 #include "safe_queue.h"

36 }

38 typedef struct THModel {

39 DNNModel model;

40 DnnContext *ctx;

41 torch::jit::Module *jit_model;

42 SafeQueue *request_queue;

43 Queue *task_queue;

44 Queue *lltask_queue;

45 } THModel;

47 typedef struct THInferRequest {

48 torch::Tensor *output;

49 torch::Tensor *input_tensor;

50 } THInferRequest;

52 typedef struct THRequestItem {

53 THInferRequest *infer_request;

54 LastLevelTaskItem *lltask;

55 DNNAsyncExecModule exec_module;

56 } THRequestItem;

59 #define OFFSET(x) offsetof(THOptions, x)

60 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM

61 static const AVOption dnn_th_options[] = {

62 { "optimize", "turn on graph executor optimization", OFFSET(optimize), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS},

63 { NULL }

64 };

66 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)

67 {

68 THModel *th_model = (THModel *)task->model;

69 DnnContext *ctx = th_model->ctx;

70 LastLevelTaskItem *lltask = (LastLevelTaskItem *)av_malloc(sizeof(*lltask));

71 if (!lltask) {

72 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n");

73 return AVERROR(ENOMEM);

74 }

75 task->inference_todo = 1;

76 task->inference_done = 0;

77 lltask->task = task;

78 if (ff_queue_push_back(lltask_queue, lltask) < 0) {

79 av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");

80 av_freep(&lltask);

81 return AVERROR(ENOMEM);

82 }

83 return 0;

84 }

86 static void th_free_request(THInferRequest *request)

87 {

88 if (!request)

89 return;

90 if (request->output) {

91 delete(request->output);

92 request->output = NULL;

93 }

94 if (request->input_tensor) {

95 delete(request->input_tensor);

96 request->input_tensor = NULL;

97 }

98 return;

99 }

100

101 static inline void destroy_request_item(THRequestItem **arg)

102 {

103 THRequestItem *item;

104 if (!arg || !*arg) {

105 return;

106 }

107 item = *arg;

108 th_free_request(item->infer_request);

109 av_freep(&item->infer_request);

110 av_freep(&item->lltask);

111 ff_dnn_async_module_cleanup(&item->exec_module);

112 av_freep(arg);

113 }

114

115 static void dnn_free_model_th(DNNModel **model)

116 {

117 THModel *th_model;

118 if (!model || !*model)

119 return;

120

121 th_model = (THModel *) (*model);

122 while (ff_safe_queue_size(th_model->request_queue) != 0) {

123 THRequestItem *item = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);

124 destroy_request_item(&item);

125 }

126 ff_safe_queue_destroy(th_model->request_queue);

127

128 while (ff_queue_size(th_model->lltask_queue) != 0) {

129 LastLevelTaskItem *item = (LastLevelTaskItem *)ff_queue_pop_front(th_model->lltask_queue);

130 av_freep(&item);

131 }

132 ff_queue_destroy(th_model->lltask_queue);

133

134 while (ff_queue_size(th_model->task_queue) != 0) {

135 TaskItem *item = (TaskItem *)ff_queue_pop_front(th_model->task_queue);

136 av_frame_free(&item->in_frame);

137 av_frame_free(&item->out_frame);

138 av_freep(&item);

139 }

140 ff_queue_destroy(th_model->task_queue);

141 delete th_model->jit_model;

142 av_freep(&th_model);

143 *model = NULL;

144 }

145

146 static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)

147 {

148 input->dt = DNN_FLOAT;

149 input->order = DCO_RGB;

150 input->layout = DL_NCHW;

151 input->dims[0] = 1;

152 input->dims[1] = 3;

153 input->dims[2] = -1;

154 input->dims[3] = -1;

155 return 0;

156 }

157

158 static void deleter(void *arg)

159 {

160 av_freep(&arg);

161 }

162

163 static int fill_model_input_th(THModel *th_model, THRequestItem *request)

164 {

165 LastLevelTaskItem *lltask = NULL;

166 TaskItem *task = NULL;

167 THInferRequest *infer_request = NULL;

168 DNNData input = { 0 };

169 DnnContext *ctx = th_model->ctx;

170 int ret, width_idx, height_idx, channel_idx;

171

172 lltask = (LastLevelTaskItem *)ff_queue_pop_front(th_model->lltask_queue);

173 if (!lltask) {

174 ret = AVERROR(EINVAL);

175 goto err;

176 }

177 request->lltask = lltask;

178 task = lltask->task;

179 infer_request = request->infer_request;

180

181 ret = get_input_th(&th_model->model, &input, NULL);

182 if ( ret != 0) {

183 goto err;

184 }

185 width_idx = dnn_get_width_idx_by_layout(input.layout);

186 height_idx = dnn_get_height_idx_by_layout(input.layout);

187 channel_idx = dnn_get_channel_idx_by_layout(input.layout);

188 input.dims[height_idx] = task->in_frame->height;

189 input.dims[width_idx] = task->in_frame->width;

190 input.data = av_malloc(input.dims[height_idx] * input.dims[width_idx] *

191 input.dims[channel_idx] * sizeof(float));

192 if (!input.data)

193 return AVERROR(ENOMEM);

194 infer_request->input_tensor = new torch::Tensor();

195 infer_request->output = new torch::Tensor();

196

197 switch (th_model->model.func_type) {

198 case DFT_PROCESS_FRAME:

199 input.scale = 255;

200 if (task->do_ioproc) {

201 if (th_model->model.frame_pre_proc != NULL) {

202 th_model->model.frame_pre_proc(task->in_frame, &input, th_model->model.filter_ctx);

203 } else {

204 ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);

205 }

206 }

207 break;

208 default:

209 avpriv_report_missing_feature(NULL, "model function type %d", th_model->model.func_type);

210 break;

211 }

212 *infer_request->input_tensor = torch::from_blob(input.data,

213 {1, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]},

214 deleter, torch::kFloat32);

215 return 0;

216

217 err:

218 th_free_request(infer_request);

219 return ret;

220 }

221

222 static int th_start_inference(void *args)

223 {

224 THRequestItem *request = (THRequestItem *)args;

225 THInferRequest *infer_request = NULL;

226 LastLevelTaskItem *lltask = NULL;

227 TaskItem *task = NULL;

228 THModel *th_model = NULL;

229 DnnContext *ctx = NULL;

230 std::vector<torch::jit::IValue> inputs;

231 torch::NoGradGuard no_grad;

232

233 if (!request) {

234 av_log(NULL, AV_LOG_ERROR, "THRequestItem is NULL\n");

235 return AVERROR(EINVAL);

236 }

237 infer_request = request->infer_request;

238 lltask = request->lltask;

239 task = lltask->task;

240 th_model = (THModel *)task->model;

241 ctx = th_model->ctx;

242

243 if (ctx->torch_option.optimize)

244 torch::jit::setGraphExecutorOptimize(true);

245 else

246 torch::jit::setGraphExecutorOptimize(false);

247

248 if (!infer_request->input_tensor || !infer_request->output) {

249 av_log(ctx, AV_LOG_ERROR, "input or output tensor is NULL\n");

250 return DNN_GENERIC_ERROR;

251 }

252 // Transfer tensor to the same device as model

253 c10::Device device = (*th_model->jit_model->parameters().begin()).device();

254 if (infer_request->input_tensor->device() != device)

255 *infer_request->input_tensor = infer_request->input_tensor->to(device);

256 inputs.push_back(*infer_request->input_tensor);

257

258 *infer_request->output = th_model->jit_model->forward(inputs).toTensor();

259

260 return 0;

261 }

262

263 static void infer_completion_callback(void *args) {

264 THRequestItem *request = (THRequestItem*)args;

265 LastLevelTaskItem *lltask = request->lltask;

266 TaskItem *task = lltask->task;

267 DNNData outputs = { 0 };

268 THInferRequest *infer_request = request->infer_request;

269 THModel *th_model = (THModel *)task->model;

270 torch::Tensor *output = infer_request->output;

271

272 c10::IntArrayRef sizes = output->sizes();

273 outputs.order = DCO_RGB;

274 outputs.layout = DL_NCHW;

275 outputs.dt = DNN_FLOAT;

276 if (sizes.size() == 4) {

277 // 4 dimensions: [batch_size, channel, height, width]

278 // this format of data is normally used for video frame SR

279 outputs.dims[0] = sizes.at(0); // N

280 outputs.dims[1] = sizes.at(1); // C

281 outputs.dims[2] = sizes.at(2); // H

282 outputs.dims[3] = sizes.at(3); // W

283 } else {

284 avpriv_report_missing_feature(th_model->ctx, "Support of this kind of model");

285 goto err;

286 }

287

288 switch (th_model->model.func_type) {

289 case DFT_PROCESS_FRAME:

290 if (task->do_ioproc) {

291 // Post process can only deal with CPU memory.

292 if (output->device() != torch::kCPU)

293 *output = output->to(torch::kCPU);

294 outputs.scale = 255;

295 outputs.data = output->data_ptr();

296 if (th_model->model.frame_post_proc != NULL) {

297 th_model->model.frame_post_proc(task->out_frame, &outputs, th_model->model.filter_ctx);

298 } else {

299 ff_proc_from_dnn_to_frame(task->out_frame, &outputs, th_model->ctx);

300 }

301 } else {

302 task->out_frame->width = outputs.dims[dnn_get_width_idx_by_layout(outputs.layout)];

303 task->out_frame->height = outputs.dims[dnn_get_height_idx_by_layout(outputs.layout)];

304 }

305 break;

306 default:

307 avpriv_report_missing_feature(th_model->ctx, "model function type %d", th_model->model.func_type);

308 goto err;

309 }

310 task->inference_done++;

311 av_freep(&request->lltask);

312 err:

313 th_free_request(infer_request);

314

315 if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {

316 destroy_request_item(&request);

317 av_log(th_model->ctx, AV_LOG_ERROR, "Unable to push back request_queue when failed to start inference.\n");

318 }

319 }

320

321 static int execute_model_th(THRequestItem *request, Queue *lltask_queue)

322 {

323 THModel *th_model = NULL;

324 LastLevelTaskItem *lltask;

325 TaskItem *task = NULL;

326 int ret = 0;

327

328 if (ff_queue_size(lltask_queue) == 0) {

329 destroy_request_item(&request);

330 return 0;

331 }

332

333 lltask = (LastLevelTaskItem *)ff_queue_peek_front(lltask_queue);

334 if (lltask == NULL) {

335 av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");

336 ret = AVERROR(EINVAL);

337 goto err;

338 }

339 task = lltask->task;

340 th_model = (THModel *)task->model;

341

342 ret = fill_model_input_th(th_model, request);

343 if ( ret != 0) {

344 goto err;

345 }

346 if (task->async) {

347 avpriv_report_missing_feature(th_model->ctx, "LibTorch async");

348 } else {

349 ret = th_start_inference((void *)(request));

350 if (ret != 0) {

351 goto err;

352 }

353 infer_completion_callback(request);

354 return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;

355 }

356

357 err:

358 th_free_request(request->infer_request);

359 if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {

360 destroy_request_item(&request);

361 }

362 return ret;

363 }

364

365 static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height,

366 const char *output_name, int *output_width, int *output_height)

367 {

368 int ret = 0;

369 THModel *th_model = (THModel*) model;

370 DnnContext *ctx = th_model->ctx;

371 TaskItem task = { 0 };

372 THRequestItem *request = NULL;

373 DNNExecBaseParams exec_params = {

374 .input_name = input_name,

375 .output_names = &output_name,

376 .nb_output = 1,

377 .in_frame = NULL,

378 .out_frame = NULL,

379 };

380 ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, th_model, input_height, input_width, ctx);

381 if ( ret != 0) {

382 goto err;

383 }

384

385 ret = extract_lltask_from_task(&task, th_model->lltask_queue);

386 if ( ret != 0) {

387 av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");

388 goto err;

389 }

390

391 request = (THRequestItem*) ff_safe_queue_pop_front(th_model->request_queue);

392 if (!request) {

393 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");

394 ret = AVERROR(EINVAL);

395 goto err;

396 }

397

398 ret = execute_model_th(request, th_model->lltask_queue);

399 *output_width = task.out_frame->width;

400 *output_height = task.out_frame->height;

401

402 err:

403 av_frame_free(&task.out_frame);

404 av_frame_free(&task.in_frame);

405 return ret;

406 }

407

408 static THInferRequest *th_create_inference_request(void)

409 {

410 THInferRequest *request = (THInferRequest *)av_malloc(sizeof(THInferRequest));

411 if (!request) {

412 return NULL;

413 }

414 request->input_tensor = NULL;

415 request->output = NULL;

416 return request;

417 }

418

419 static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)

420 {

421 DNNModel *model = NULL;

422 THModel *th_model = NULL;

423 THRequestItem *item = NULL;

424 const char *device_name = ctx->device ? ctx->device : "cpu";

425

426 th_model = (THModel *)av_mallocz(sizeof(THModel));

427 if (!th_model)

428 return NULL;

429 model = &th_model->model;

430 th_model->ctx = ctx;

431

432 c10::Device device = c10::Device(device_name);

433 if (device.is_xpu()) {

434 if (!at::hasXPU()) {

435 av_log(ctx, AV_LOG_ERROR, "No XPU device found\n");

436 goto fail;

437 }

438 at::detail::getXPUHooks().initXPU();

439 } else if (!device.is_cpu()) {

440 av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name);

441 goto fail;

442 }

443

444 try {

445 th_model->jit_model = new torch::jit::Module;

446 (*th_model->jit_model) = torch::jit::load(ctx->model_filename);

447 th_model->jit_model->to(device);

448 } catch (const c10::Error& e) {

449 av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n");

450 goto fail;

451 }

452

453 th_model->request_queue = ff_safe_queue_create();

454 if (!th_model->request_queue) {

455 goto fail;

456 }

457

458 item = (THRequestItem *)av_mallocz(sizeof(THRequestItem));

459 if (!item) {

460 goto fail;

461 }

462 item->lltask = NULL;

463 item->infer_request = th_create_inference_request();

464 if (!item->infer_request) {

465 av_log(NULL, AV_LOG_ERROR, "Failed to allocate memory for Torch inference request\n");

466 goto fail;

467 }

468 item->exec_module.start_inference = &th_start_inference;

469 item->exec_module.callback = &infer_completion_callback;

470 item->exec_module.args = item;

471

472 if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) {

473 goto fail;

474 }

475 item = NULL;

476

477 th_model->task_queue = ff_queue_create();

478 if (!th_model->task_queue) {

479 goto fail;

480 }

481

482 th_model->lltask_queue = ff_queue_create();

483 if (!th_model->lltask_queue) {

484 goto fail;

485 }

486

487 model->get_input = &get_input_th;

488 model->get_output = &get_output_th;

489 model->filter_ctx = filter_ctx;

490 model->func_type = func_type;

491 return model;

492

493 fail:

494 if (item) {

495 destroy_request_item(&item);

496 av_freep(&item);

497 }

498 dnn_free_model_th(&model);

499 return NULL;

500 }

501

502 static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)

503 {

504 THModel *th_model = (THModel *)model;

505 DnnContext *ctx = th_model->ctx;

506 TaskItem *task;

507 THRequestItem *request;

508 int ret = 0;

509

510 ret = ff_check_exec_params(ctx, DNN_TH, model->func_type, exec_params);

511 if (ret != 0) {

512 av_log(ctx, AV_LOG_ERROR, "exec parameter checking fail.\n");

513 return ret;

514 }

515

516 task = (TaskItem *)av_malloc(sizeof(TaskItem));

517 if (!task) {

518 av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");

519 return AVERROR(ENOMEM);

520 }

521

522 ret = ff_dnn_fill_task(task, exec_params, th_model, 0, 1);

523 if (ret != 0) {

524 av_freep(&task);

525 av_log(ctx, AV_LOG_ERROR, "unable to fill task.\n");

526 return ret;

527 }

528

529 ret = ff_queue_push_back(th_model->task_queue, task);

530 if (ret < 0) {

531 av_freep(&task);

532 av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");

533 return ret;

534 }

535

536 ret = extract_lltask_from_task(task, th_model->lltask_queue);

537 if (ret != 0) {

538 av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");

539 return ret;

540 }

541

542 request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);

543 if (!request) {

544 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");

545 return AVERROR(EINVAL);

546 }

547

548 return execute_model_th(request, th_model->lltask_queue);

549 }

550

551 static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)

552 {

553 THModel *th_model = (THModel *)model;

554 return ff_dnn_get_result_common(th_model->task_queue, in, out);

555 }

556

557 static int dnn_flush_th(const DNNModel *model)

558 {

559 THModel *th_model = (THModel *)model;

560 THRequestItem *request;

561

562 if (ff_queue_size(th_model->lltask_queue) == 0)

563 // no pending task need to flush

564 return 0;

565

566 request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);

567 if (!request) {

568 av_log(th_model->ctx, AV_LOG_ERROR, "unable to get infer request.\n");

569 return AVERROR(EINVAL);

570 }

571

572 return execute_model_th(request, th_model->lltask_queue);

573 }

574

575 extern const DNNModule ff_dnn_backend_torch = {

576 .clazz = DNN_DEFINE_CLASS(dnn_th),

577 .type = DNN_TH,

578 .load_model = dnn_load_model_th,

579 .execute_model = dnn_execute_model_th,

580 .get_result = dnn_get_result_th,

581 .flush = dnn_flush_th,

582 .free_model = dnn_free_model_th,

583 };

THRequestItem::lltask

LastLevelTaskItem * lltask

Definition: dnn_backend_torch.cpp:54

THModel::lltask_queue

Queue * lltask_queue

Definition: dnn_backend_torch.cpp:44

THRequestItem::infer_request

THInferRequest * infer_request

Definition: dnn_backend_torch.cpp:53

THModel::ctx

DnnContext * ctx

Definition: dnn_backend_torch.cpp:40

AVERROR

Filter the word "frame" indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions

opt.h

ff_safe_queue_pop_front

void * ff_safe_queue_pop_front(SafeQueue *sq)

Remove and free first element from the queue in SafeQueue.

Definition: safe_queue.c:105

out

FILE * out

Definition: movenc.c:55

deleter

static void deleter(void *arg)

Definition: dnn_backend_torch.cpp:158

FLAGS

#define FLAGS

Definition: dnn_backend_torch.cpp:60

THModel

Definition: dnn_backend_torch.cpp:38

DNNAsyncExecModule

Common Async Execution Mechanism for the DNN Backends.

Definition: dnn_backend_common.h:65

DNNFunctionType

Definition: dnn_interface.h:56

output

filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output

Definition: filter_design.txt:225

ff_queue_pop_front

void * ff_queue_pop_front(Queue *q)

Remove and free first element from the Queue.

Definition: queue.c:151

ff_check_exec_params

int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)

Definition: dnn_backend_common.c:30

ff_queue_size

size_t ff_queue_size(Queue *q)

Return the length of the Queue.

Definition: queue.c:88

DNN_GENERIC_ERROR

#define DNN_GENERIC_ERROR

Definition: dnn_interface.h:33

av_frame_free

void av_frame_free(AVFrame **frame)

Free the frame and any dynamically allocated objects in it, e.g.

Definition: frame.c:162

LastLevelTaskItem

Definition: dnn_backend_common.h:57

ff_dnn_backend_torch

const DNNModule ff_dnn_backend_torch

AVFrame

This structure describes decoded (raw) audio or video data.

Definition: frame.h:389

AVFrame::width

int width

Definition: frame.h:461

SafeQueue

Double-ended queue with mutex locks ensuring data consistency while multithreading.

Definition: safe_queue.c:46

dnn_execute_model_th

static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)

Definition: dnn_backend_torch.cpp:502

AVOption

AVOption.

Definition: opt.h:429

DNNModel::frame_pre_proc

FramePrePostProc frame_pre_proc

Definition: dnn_interface.h:110

DNNExecBaseParams::input_name

const char * input_name

Definition: dnn_interface.h:81

dnn_io_proc.h

TaskItem

Definition: dnn_backend_common.h:43

DNNAsyncExecModule::callback

void(* callback)(void *args)

Completion Callback for the backend.

Definition: dnn_backend_common.h:77

av_malloc

#define av_malloc(s)

Definition: tableprint_vlc.h:30

DNNModel::filter_ctx

AVFilterContext * filter_ctx

Definition: dnn_interface.h:99

ff_queue_create

Queue * ff_queue_create(void)

Create a Queue instance.

Definition: queue.c:47

dnn_get_width_idx_by_layout

static int dnn_get_width_idx_by_layout(DNNLayout layout)

Definition: dnn_interface.h:197

TaskItem::model

void * model

Definition: dnn_backend_common.h:44

fail

#define fail()

Definition: checkasm.h:188

DnnContext

Definition: dnn_interface.h:143

filter_ctx

static FilteringContext * filter_ctx

Definition: transcode.c:52

Queue

Linear double-ended data structure.

Definition: queue.c:33

ff_queue_push_back

int ff_queue_push_back(Queue *q, void *v)

Add data to the tail of the queue.

Definition: queue.c:130

THModel::jit_model

torch::jit::Module * jit_model

Definition: dnn_backend_torch.cpp:41

AV_LOG_ERROR

#define AV_LOG_ERROR

Something went wrong and cannot losslessly be recovered.

Definition: log.h:180

LastLevelTaskItem::task

TaskItem * task

Definition: dnn_backend_common.h:58

destroy_request_item

static void destroy_request_item(THRequestItem **arg)

Definition: dnn_backend_torch.cpp:101

th_create_inference_request

static THInferRequest * th_create_inference_request(void)

Definition: dnn_backend_torch.cpp:408

ff_queue_destroy

void ff_queue_destroy(Queue *q)

Destroy the Queue instance.

Definition: queue.c:72

DNNData

Definition: dnn_interface.h:69

DNNModule::clazz

const AVClass clazz

Definition: dnn_interface.h:176

ff_dnn_fill_gettingoutput_task

int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)

Allocate input and output frames and fill the Task with execution parameters.

Definition: dnn_backend_common.c:156

DNNModel::get_output

int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)

Definition: dnn_interface.h:106

ctx

AVFormatContext * ctx

Definition: movenc.c:49

TaskItem::inference_todo

uint32_t inference_todo

Definition: dnn_backend_common.h:52

DL_NCHW

@ DL_NCHW

Definition: dnn_interface.h:65

dnn_load_model_th

static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)

Definition: dnn_backend_torch.cpp:419

arg

const char * arg

Definition: jacosubdec.c:67

if(ret)

Definition: filter_design.txt:179

ff_safe_queue_size

size_t ff_safe_queue_size(SafeQueue *sq)

Return the length of the SafeQueue.

Definition: safe_queue.c:80

ff_proc_from_frame_to_dnn

int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)

Definition: dnn_io_proc.c:182

THRequestItem::exec_module

DNNAsyncExecModule exec_module

Definition: dnn_backend_torch.cpp:55

NULL

#define NULL

Definition: coverity.c:32

sizes

static const int sizes[][2]

Definition: img2dec.c:60

get_input_th

static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)

Definition: dnn_backend_torch.cpp:146

ff_safe_queue_create

SafeQueue * ff_safe_queue_create(void)

Create and initialize a SafeQueue instance.

Definition: safe_queue.c:52

DNNModel::frame_post_proc

FramePrePostProc frame_post_proc

Definition: dnn_interface.h:113

get_output_th

static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)

Definition: dnn_backend_torch.cpp:365

ff_dnn_async_module_cleanup

int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)

Join the Async Execution thread and set module pointers to NULL.

Definition: dnn_backend_common.c:86

infer_completion_callback

static void infer_completion_callback(void *args)

Definition: dnn_backend_torch.cpp:263

TaskItem::in_frame

AVFrame * in_frame

Definition: dnn_backend_common.h:45

extract_lltask_from_task

static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)

Definition: dnn_backend_torch.cpp:66

inputs

these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs

Definition: filter_design.txt:243

THInferRequest::output

torch::Tensor * output

Definition: dnn_backend_torch.cpp:48

TaskItem::async

uint8_t async

Definition: dnn_backend_common.h:49

TaskItem::inference_done

uint32_t inference_done

Definition: dnn_backend_common.h:53

queue.h

DNNModel::func_type

DNNFunctionType func_type

Definition: dnn_interface.h:101

avpriv_report_missing_feature

void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2

Log a generic warning message about a missing feature.

ff_safe_queue_destroy

void ff_safe_queue_destroy(SafeQueue *sq)

Destroy the SafeQueue instance.

Definition: safe_queue.c:69

DNN_FLOAT

@ DNN_FLOAT

Definition: dnn_interface.h:41

dnn_get_result_th

static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)

Definition: dnn_backend_torch.cpp:551

ff_dnn_fill_task

int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)

Fill the Task for Backend Execution.

Definition: dnn_backend_common.c:50

input

and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input

Definition: filter_design.txt:172

DNN_DEFINE_CLASS

#define DNN_DEFINE_CLASS(fname)

Definition: dnn_backend_common.h:39

THRequestItem

Definition: dnn_backend_torch.cpp:52

ff_safe_queue_push_back

int ff_safe_queue_push_back(SafeQueue *sq, void *v)

Add data to the tail of queue in the SafeQueue after locking mutex.

Definition: safe_queue.c:95

th_start_inference

static int th_start_inference(void *args)