FFmpeg: libavfilter/dnn/dnn_backend_tf.c Source File

FFmpeg

[フレーム]

dnn_backend_tf.c

Go to the documentation of this file.

1 /*

3 *

4 * This file is part of FFmpeg.

5 *

6 * FFmpeg is free software; you can redistribute it and/or

7 * modify it under the terms of the GNU Lesser General Public

8 * License as published by the Free Software Foundation; either

9 * version 2.1 of the License, or (at your option) any later version.

10 *

11 * FFmpeg is distributed in the hope that it will be useful,

12 * but WITHOUT ANY WARRANTY; without even the implied warranty of

13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

14 * Lesser General Public License for more details.

15 *

16 * You should have received a copy of the GNU Lesser General Public

17 * License along with FFmpeg; if not, write to the Free Software

18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

19 */

21 /**

22 * @file

23 * DNN tensorflow backend implementation.

24 */

26 #include "dnn_backend_tf.h"

27 #include "dnn_backend_native.h"

28 #include "dnn_backend_native_layer_conv2d.h"

29 #include "dnn_backend_native_layer_depth2space.h"

30 #include "libavformat/avio.h"

31 #include "libavutil/avassert.h"

32 #include "libavutil/avstring.h"

33 #include "libavutil/cpu.h"

34 #include "libavcodec/defs.h"

35 #include "../internal.h"

36 #include "dnn_backend_native_layer_pad.h"

37 #include "dnn_backend_native_layer_maximum.h"

38 #include "dnn_io_proc.h"

39 #include "dnn_backend_common.h"

40 #include "safe_queue.h"

41 #include <tensorflow/c/c_api.h>

43 typedef struct TFOptions{

44 char *sess_config;

45 uint8_t async;

46 uint32_t nireq;

47 } TFOptions;

49 typedef struct TFContext {

50 const AVClass *class;

51 TFOptions options;

52 } TFContext;

54 typedef struct TFModel{

55 TFContext ctx;

56 DNNModel *model;

57 TF_Graph *graph;

58 TF_Session *session;

59 TF_Status *status;

60 SafeQueue *request_queue;

61 Queue *lltask_queue;

62 Queue *task_queue;

63 } TFModel;

65 /**

66 * Stores execution parameters for single

67 * call to the TensorFlow C API

68 */

69 typedef struct TFInferRequest {

70 TF_Output *tf_outputs;

71 TF_Tensor **output_tensors;

72 TF_Output *tf_input;

73 TF_Tensor *input_tensor;

74 } TFInferRequest;

76 typedef struct TFRequestItem {

77 TFInferRequest *infer_request;

78 LastLevelTaskItem *lltask;

79 TF_Status *status;

80 DNNAsyncExecModule exec_module;

81 } TFRequestItem;

83 #define OFFSET(x) offsetof(TFContext, x)

84 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM

85 static const AVOption dnn_tensorflow_options[] = {

86 { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },

87 DNN_BACKEND_COMMON_OPTIONS

88 { NULL }

89 };

91 AVFILTER_DEFINE_CLASS(dnn_tensorflow);

93 static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue);

94 static void infer_completion_callback(void *args);

95 static inline void destroy_request_item(TFRequestItem **arg);

97 static void free_buffer(void *data, size_t length)

98 {

99 av_freep(&data);

100 }

101

102 /**

103 * Free the contents of TensorFlow inference request.

104 * It does not free the TFInferRequest instance.

105 *

106 * @param request pointer to TFInferRequest instance.

107 * NULL pointer is allowed.

108 */

109 static void tf_free_request(TFInferRequest *request)

110 {

111 if (!request)

112 return;

113 if (request->input_tensor) {

114 TF_DeleteTensor(request->input_tensor);

115 request->input_tensor = NULL;

116 }

117 av_freep(&request->tf_input);

118 av_freep(&request->tf_outputs);

119 if (request->output_tensors) {

120 int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]);

121 for (uint32_t i = 0; i < nb_output; ++i) {

122 if (request->output_tensors[i]) {

123 TF_DeleteTensor(request->output_tensors[i]);

124 request->output_tensors[i] = NULL;

125 }

126 }

127 av_freep(&request->output_tensors);

128 }

129 }

130

131 /**

132 * Create a TensorFlow inference request. All properties

133 * are initially unallocated and set as NULL.

134 *

135 * @return pointer to the allocated TFInferRequest instance.

136 */

137 static TFInferRequest *tf_create_inference_request(void)

138 {

139 TFInferRequest *infer_request = av_malloc(sizeof(TFInferRequest));

140 if (!infer_request) {

141 return NULL;

142 }

143 infer_request->tf_outputs = NULL;

144 infer_request->tf_input = NULL;

145 infer_request->input_tensor = NULL;

146 infer_request->output_tensors = NULL;

147 return infer_request;

148 }

149

150 /**

151 * Start synchronous inference for the TensorFlow model.

152 *

153 * @param request pointer to the TFRequestItem for inference

154 * @retval 0 if execution is successful

155 * @retval AVERROR(EINVAL) if request is NULL

156 * @retval DNN_GENERIC_ERROR if execution fails

157 */

158 static int tf_start_inference(void *args)

159 {

160 TFRequestItem *request = args;

161 TFInferRequest *infer_request = request->infer_request;

162 LastLevelTaskItem *lltask = request->lltask;

163 TaskItem *task = lltask->task;

164 TFModel *tf_model = task->model;

165

166 if (!request) {

167 av_log(&tf_model->ctx, AV_LOG_ERROR, "TFRequestItem is NULL\n");

168 return AVERROR(EINVAL);

169 }

170

171 TF_SessionRun(tf_model->session, NULL,

172 infer_request->tf_input, &infer_request->input_tensor, 1,

173 infer_request->tf_outputs, infer_request->output_tensors,

174 task->nb_output, NULL, 0, NULL,

175 request->status);

176 if (TF_GetCode(request->status) != TF_OK) {

177 av_log(&tf_model->ctx, AV_LOG_ERROR, "%s", TF_Message(request->status));

178 tf_free_request(infer_request);

179 if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {

180 destroy_request_item(&request);

181 }

182 return DNN_GENERIC_ERROR;

183 }

184 return 0;

185 }

186

187 /**

188 * Free the TFRequestItem completely.

189 *

190 * @param arg Address of the TFInferRequest instance.

191 */

192 static inline void destroy_request_item(TFRequestItem **arg) {

193 TFRequestItem *request;

194 if (!arg) {

195 return;

196 }

197 request = *arg;

198 tf_free_request(request->infer_request);

199 av_freep(&request->infer_request);

200 av_freep(&request->lltask);

201 TF_DeleteStatus(request->status);

202 ff_dnn_async_module_cleanup(&request->exec_module);

203 av_freep(arg);

204 }

205

206 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)

207 {

208 TFModel *tf_model = task->model;

209 TFContext *ctx = &tf_model->ctx;

210 LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));

211 if (!lltask) {

212 av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");

213 return AVERROR(ENOMEM);

214 }

215 task->inference_todo = 1;

216 task->inference_done = 0;

217 lltask->task = task;

218 if (ff_queue_push_back(lltask_queue, lltask) < 0) {

219 av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");

220 av_freep(&lltask);

221 return AVERROR(ENOMEM);

222 }

223 return 0;

224 }

225

226 static TF_Buffer *read_graph(const char *model_filename)

227 {

228 TF_Buffer *graph_buf;

229 unsigned char *graph_data = NULL;

230 AVIOContext *model_file_context;

231 long size, bytes_read;

232

233 if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){

234 return NULL;

235 }

236

237 size = avio_size(model_file_context);

238

239 graph_data = av_malloc(size);

240 if (!graph_data){

241 avio_closep(&model_file_context);

242 return NULL;

243 }

244 bytes_read = avio_read(model_file_context, graph_data, size);

245 avio_closep(&model_file_context);

246 if (bytes_read != size){

247 av_freep(&graph_data);

248 return NULL;

249 }

250

251 graph_buf = TF_NewBuffer();

252 graph_buf->data = graph_data;

253 graph_buf->length = size;

254 graph_buf->data_deallocator = free_buffer;

255

256 return graph_buf;

257 }

258

259 static TF_Tensor *allocate_input_tensor(const DNNData *input)

260 {

261 TF_DataType dt;

262 size_t size;

263 int64_t input_dims[] = {1, input->height, input->width, input->channels};

264 switch (input->dt) {

265 case DNN_FLOAT:

266 dt = TF_FLOAT;

267 size = sizeof(float);

268 break;

269 case DNN_UINT8:

270 dt = TF_UINT8;

271 size = 1;

272 break;

273 default:

274 av_assert0(!"should not reach here");

275 }

276

277 return TF_AllocateTensor(dt, input_dims, 4,

278 input_dims[1] * input_dims[2] * input_dims[3] * size);

279 }

280

281 static int get_input_tf(void *model, DNNData *input, const char *input_name)

282 {

283 TFModel *tf_model = model;

284 TFContext *ctx = &tf_model->ctx;

285 TF_Status *status;

286 int64_t dims[4];

287

288 TF_Output tf_output;

289 tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);

290 if (!tf_output.oper) {

291 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);

292 return AVERROR(EINVAL);

293 }

294

295 tf_output.index = 0;

296 input->dt = TF_OperationOutputType(tf_output);

297 input->order = DCO_RGB;

298

299 status = TF_NewStatus();

300 TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);

301 if (TF_GetCode(status) != TF_OK){

302 TF_DeleteStatus(status);

303 av_log(ctx, AV_LOG_ERROR, "Failed to get input tensor shape: number of dimension incorrect\n");

304 return DNN_GENERIC_ERROR;

305 }

306 TF_DeleteStatus(status);

307

308 // currently only NHWC is supported

309 av_assert0(dims[0] == 1 || dims[0] == -1);

310 input->height = dims[1];

311 input->width = dims[2];

312 input->channels = dims[3];

313

314 return 0;

315 }

316

317 static int get_output_tf(void *model, const char *input_name, int input_width, int input_height,

318 const char *output_name, int *output_width, int *output_height)

319 {

320 int ret;

321 TFModel *tf_model = model;

322 TFContext *ctx = &tf_model->ctx;

323 TaskItem task;

324 TFRequestItem *request;

325 DNNExecBaseParams exec_params = {

326 .input_name = input_name,

327 .output_names = &output_name,

328 .nb_output = 1,

329 .in_frame = NULL,

330 .out_frame = NULL,

331 };

332

333 ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, tf_model, input_height, input_width, ctx);

334 if (ret != 0) {

335 goto err;

336 }

337

338 ret = extract_lltask_from_task(&task, tf_model->lltask_queue);

339 if (ret != 0) {

340 av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");

341 goto err;

342 }

343

344 request = ff_safe_queue_pop_front(tf_model->request_queue);

345 if (!request) {

346 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");

347 ret = AVERROR(EINVAL);

348 goto err;

349 }

350

351 ret = execute_model_tf(request, tf_model->lltask_queue);

352 *output_width = task.out_frame->width;

353 *output_height = task.out_frame->height;

354

355 err:

356 av_frame_free(&task.out_frame);

357 av_frame_free(&task.in_frame);

358 return ret;

359 }

360

361 #define SPACE_CHARS " \t\r\n"

362 static int hex_to_data(uint8_t *data, const char *p)

363 {

364 int c, len, v;

365

366 len = 0;

367 v = 1;

368 for (;;) {

369 p += strspn(p, SPACE_CHARS);

370 if (*p == '0円')

371 break;

372 c = av_toupper((unsigned char) *p++);

373 if (c >= '0' && c <= '9')

374 c = c - '0';

375 else if (c >= 'A' && c <= 'F')

376 c = c - 'A' + 10;

377 else

378 break;

379 v = (v << 4) | c;

380 if (v & 0x100) {

381 if (data) {

382 data[len] = v;

383 }

384 len++;

385 v = 1;

386 }

387 }

388 return len;

389 }

390

391 static int load_tf_model(TFModel *tf_model, const char *model_filename)

392 {

393 TFContext *ctx = &tf_model->ctx;

394 TF_Buffer *graph_def;

395 TF_ImportGraphDefOptions *graph_opts;

396 TF_SessionOptions *sess_opts;

397 const TF_Operation *init_op;

398 uint8_t *sess_config = NULL;

399 int sess_config_length = 0;

400

401 // prepare the sess config data

402 if (tf_model->ctx.options.sess_config != NULL) {

403 const char *config;

404 /*

405 tf_model->ctx.options.sess_config is hex to present the serialized proto

406 required by TF_SetConfig below, so we need to first generate the serialized

407 proto in a python script, tools/python/tf_sess_config.py is a script example

408 to generate the configs of sess_config.

409 */

410 if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {

411 av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");

412 return AVERROR(EINVAL);

413 }

414 config = tf_model->ctx.options.sess_config + 2;

415 sess_config_length = hex_to_data(NULL, config);

416

417 sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);

418 if (!sess_config) {

419 av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");

420 return AVERROR(ENOMEM);

421 }

422 if (hex_to_data(sess_config, config) < 0) {

423 av_log(ctx, AV_LOG_ERROR, "failed to convert hex to data\n");

424 return AVERROR(EINVAL);

425 }

426 }

427

428 graph_def = read_graph(model_filename);

429 if (!graph_def){

430 av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);

431 av_freep(&sess_config);

432 return AVERROR(EINVAL);

433 }

434 tf_model->graph = TF_NewGraph();

435 tf_model->status = TF_NewStatus();

436 graph_opts = TF_NewImportGraphDefOptions();

437 TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);

438 TF_DeleteImportGraphDefOptions(graph_opts);

439 TF_DeleteBuffer(graph_def);

440 if (TF_GetCode(tf_model->status) != TF_OK){

441 TF_DeleteGraph(tf_model->graph);

442 TF_DeleteStatus(tf_model->status);

443 av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");

444 av_freep(&sess_config);

445 return DNN_GENERIC_ERROR;

446 }

447

448 init_op = TF_GraphOperationByName(tf_model->graph, "init");

449 sess_opts = TF_NewSessionOptions();

450

451 if (sess_config) {

452 TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);

453 av_freep(&sess_config);

454 if (TF_GetCode(tf_model->status) != TF_OK) {

455 TF_DeleteGraph(tf_model->graph);

456 TF_DeleteStatus(tf_model->status);

457 TF_DeleteSessionOptions(sess_opts);

458 av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",

459 tf_model->ctx.options.sess_config);

460 return DNN_GENERIC_ERROR;

461 }

462 }

463

464 tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);

465 TF_DeleteSessionOptions(sess_opts);

466 if (TF_GetCode(tf_model->status) != TF_OK)

467 {

468 TF_DeleteGraph(tf_model->graph);

469 TF_DeleteStatus(tf_model->status);

470 av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");

471 return DNN_GENERIC_ERROR;

472 }

473

474 // Run initialization operation with name "init" if it is present in graph

475 if (init_op){

476 TF_SessionRun(tf_model->session, NULL,

477 NULL, NULL, 0,

478 NULL, NULL, 0,

479 &init_op, 1, NULL, tf_model->status);

480 if (TF_GetCode(tf_model->status) != TF_OK)

481 {

482 TF_DeleteSession(tf_model->session, tf_model->status);

483 TF_DeleteGraph(tf_model->graph);

484 TF_DeleteStatus(tf_model->status);

485 av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");

486 return DNN_GENERIC_ERROR;

487 }

488 }

489

490 return 0;

491 }

492

493 #define NAME_BUFFER_SIZE 256

494

495 static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,

496 ConvolutionalParams* params, const int layer)

497 {

498 TFContext *ctx = &tf_model->ctx;

499 TF_Operation *op;

500 TF_OperationDescription *op_desc;

501 TF_Output input;

502 int64_t strides[] = {1, 1, 1, 1};

503 TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;

504 int64_t dims[4];

505 int dims_len;

506 char name_buffer[NAME_BUFFER_SIZE];

507 int32_t size;

508

509 size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;

510 input.index = 0;

511

512 snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);

513 op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);

514 TF_SetAttrType(op_desc, "dtype", TF_FLOAT);

515 dims[0] = params->output_num;

516 dims[1] = params->kernel_size;

517 dims[2] = params->kernel_size;

518 dims[3] = params->input_num;

519 dims_len = 4;

520 kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));

521 memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));

522 TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);

523 if (TF_GetCode(tf_model->status) != TF_OK){

524 goto err;

525 }

526 op = TF_FinishOperation(op_desc, tf_model->status);

527 if (TF_GetCode(tf_model->status) != TF_OK){

528 goto err;

529 }

530

531 snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);

532 op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);

533 input.oper = op;

534 TF_AddInput(op_desc, input);

535 input.oper = transpose_op;

536 TF_AddInput(op_desc, input);

537 TF_SetAttrType(op_desc, "T", TF_FLOAT);

538 TF_SetAttrType(op_desc, "Tperm", TF_INT32);

539 op = TF_FinishOperation(op_desc, tf_model->status);

540 if (TF_GetCode(tf_model->status) != TF_OK){

541 goto err;

542 }

543

544 snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);

545 op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);

546 input.oper = *cur_op;

547 TF_AddInput(op_desc, input);

548 input.oper = op;

549 TF_AddInput(op_desc, input);

550 TF_SetAttrType(op_desc, "T", TF_FLOAT);

551 TF_SetAttrIntList(op_desc, "strides", strides, 4);

552 TF_SetAttrString(op_desc, "padding", "VALID", 5);

553 *cur_op = TF_FinishOperation(op_desc, tf_model->status);

554 if (TF_GetCode(tf_model->status) != TF_OK){

555 goto err;

556 }

557

558 snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);

559 op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);

560 TF_SetAttrType(op_desc, "dtype", TF_FLOAT);

561 dims[0] = params->output_num;

562 dims_len = 1;

563 biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));

564 memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));

565 TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);

566 if (TF_GetCode(tf_model->status) != TF_OK){

567 goto err;

568 }

569 op = TF_FinishOperation(op_desc, tf_model->status);

570 if (TF_GetCode(tf_model->status) != TF_OK){

571 goto err;

572 }

573

574 snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);

575 op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);

576 input.oper = *cur_op;

577 TF_AddInput(op_desc, input);

578 input.oper = op;

579 TF_AddInput(op_desc, input);

580 TF_SetAttrType(op_desc, "T", TF_FLOAT);

581 *cur_op = TF_FinishOperation(op_desc, tf_model->status);

582 if (TF_GetCode(tf_model->status) != TF_OK){

583 goto err;

584 }

585

586 snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);

587 switch (params->activation){

588 case RELU:

589 op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);

590 break;

591 case TANH:

592 op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);

593 break;

594 case SIGMOID:

595 op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);

596 break;

597 default:

598 avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);

599 return AVERROR(ENOSYS);

600 }

601 input.oper = *cur_op;

602 TF_AddInput(op_desc, input);

603 TF_SetAttrType(op_desc, "T", TF_FLOAT);

604 *cur_op = TF_FinishOperation(op_desc, tf_model->status);

605 if (TF_GetCode(tf_model->status) != TF_OK){

606 goto err;

607 }

608

609 return 0;

610 err:

611 TF_DeleteTensor(kernel_tensor);

612 TF_DeleteTensor(biases_tensor);

613 av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);

614 return DNN_GENERIC_ERROR;

615 }

616

617 static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,

618 DepthToSpaceParams *params, const int layer)

619 {

620 TFContext *ctx = &tf_model->ctx;

621 TF_OperationDescription *op_desc;

622 TF_Output input;

623 char name_buffer[NAME_BUFFER_SIZE];

624

625 snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);

626 op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);

627 input.oper = *cur_op;

628 input.index = 0;

629 TF_AddInput(op_desc, input);

630 TF_SetAttrType(op_desc, "T", TF_FLOAT);

631 TF_SetAttrInt(op_desc, "block_size", params->block_size);

632 *cur_op = TF_FinishOperation(op_desc, tf_model->status);

633 if (TF_GetCode(tf_model->status) != TF_OK){

634 av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);

635 return DNN_GENERIC_ERROR;

636 }

637

638 return 0;

639 }

640

641 static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,

642 LayerPadParams *params, const int layer)

643 {

644 TFContext *ctx = &tf_model->ctx;

645 TF_Operation *op;

646 TF_Tensor *tensor;

647 TF_OperationDescription *op_desc;

648 TF_Output input;

649 int32_t *pads;

650 int64_t pads_shape[] = {4, 2};

651

652 char name_buffer[NAME_BUFFER_SIZE];

653 snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);

654

655 op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);

656 TF_SetAttrType(op_desc, "dtype", TF_INT32);

657 tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));

658 pads = (int32_t *)TF_TensorData(tensor);

659 pads[0] = params->paddings[0][0];

660 pads[1] = params->paddings[0][1];

661 pads[2] = params->paddings[1][0];

662 pads[3] = params->paddings[1][1];

663 pads[4] = params->paddings[2][0];

664 pads[5] = params->paddings[2][1];

665 pads[6] = params->paddings[3][0];

666 pads[7] = params->paddings[3][1];

667 TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);

668 if (TF_GetCode(tf_model->status) != TF_OK){

669 TF_DeleteTensor(tensor);

670 av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);

671 return DNN_GENERIC_ERROR;

672 }

673 op = TF_FinishOperation(op_desc, tf_model->status);

674 if (TF_GetCode(tf_model->status) != TF_OK){

675 TF_DeleteTensor(tensor);

676 av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);

677 return DNN_GENERIC_ERROR;

678 }

679

680 op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");

681 input.oper = *cur_op;

682 input.index = 0;

683 TF_AddInput(op_desc, input);

684 input.oper = op;

685 TF_AddInput(op_desc, input);

686 TF_SetAttrType(op_desc, "T", TF_FLOAT);

687 TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);

688 TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);

689 *cur_op = TF_FinishOperation(op_desc, tf_model->status);

690 if (TF_GetCode(tf_model->status) != TF_OK){

691 TF_DeleteTensor(tensor);

692 av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);

693 return DNN_GENERIC_ERROR;

694 }

695

696 return 0;

697 }

698

699 static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,

700 DnnLayerMaximumParams *params, const int layer)

701 {

702 TFContext *ctx = &tf_model->ctx;

703 TF_Operation *op;

704 TF_Tensor *tensor;

705 TF_OperationDescription *op_desc;

706 TF_Output input;

707 float *y;

708

709 char name_buffer[NAME_BUFFER_SIZE];

710 snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);

711

712 op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);

713 TF_SetAttrType(op_desc, "dtype", TF_FLOAT);

714 tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));

715 y = (float *)TF_TensorData(tensor);

716 *y = params->val.y;

717 TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);

718 if (TF_GetCode(tf_model->status) != TF_OK){

719 TF_DeleteTensor(tensor);

720 av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);

721 return DNN_GENERIC_ERROR;

722 }

723 op = TF_FinishOperation(op_desc, tf_model->status);

724 if (TF_GetCode(tf_model->status) != TF_OK){

725 TF_DeleteTensor(tensor);

726 av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);

727 return DNN_GENERIC_ERROR;

728 }

729

730 snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);

731 op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);

732 input.oper = *cur_op;

733 input.index = 0;

734 TF_AddInput(op_desc, input);

735 input.oper = op;

736 TF_AddInput(op_desc, input);

737 TF_SetAttrType(op_desc, "T", TF_FLOAT);

738 *cur_op = TF_FinishOperation(op_desc, tf_model->status);

739 if (TF_GetCode(tf_model->status) != TF_OK){

740 TF_DeleteTensor(tensor);

741 av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);

742 return DNN_GENERIC_ERROR;

743 }

744

745 return 0;

746 }

747

748 static int load_native_model(TFModel *tf_model, const char *model_filename)

749 {

750 TFContext *ctx = &tf_model->ctx;

751 int32_t layer;

752 TF_OperationDescription *op_desc;

753 TF_Operation *op;

754 TF_Operation *transpose_op;

755 TF_Tensor *tensor = NULL;

756 TF_Output input;

757 int32_t *transpose_perm;

758 int64_t transpose_perm_shape[] = {4};

759 int64_t input_shape[] = {1, -1, -1, -1};

760 int layer_add_res;

761 DNNModel *model = NULL;

762 NativeModel *native_model;

763

764 model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);

765 if (!model){

766 av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");

767 return AVERROR(EINVAL);

768 }

769

770 native_model = model->model;

771 tf_model->graph = TF_NewGraph();

772 tf_model->status = TF_NewStatus();

773

774 #define CLEANUP_ON_ERROR(tf_model) \

775 { \

776 TF_DeleteTensor(tensor); \

777 TF_DeleteGraph(tf_model->graph); \

778 TF_DeleteStatus(tf_model->status); \

779 av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \

780 return DNN_GENERIC_ERROR; \

781 }

782

783 op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");

784 TF_SetAttrType(op_desc, "dtype", TF_FLOAT);

785 TF_SetAttrShape(op_desc, "shape", input_shape, 4);

786 op = TF_FinishOperation(op_desc, tf_model->status);

787 if (TF_GetCode(tf_model->status) != TF_OK){

788 CLEANUP_ON_ERROR(tf_model);

789 }

790

791 op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");

792 TF_SetAttrType(op_desc, "dtype", TF_INT32);

793 tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));

794 transpose_perm = (int32_t *)TF_TensorData(tensor);

795 transpose_perm[0] = 1;

796 transpose_perm[1] = 2;

797 transpose_perm[2] = 3;

798 transpose_perm[3] = 0;

799 TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);

800 if (TF_GetCode(tf_model->status) != TF_OK){

801 CLEANUP_ON_ERROR(tf_model);

802 }

803 transpose_op = TF_FinishOperation(op_desc, tf_model->status);

804 if (TF_GetCode(tf_model->status) != TF_OK){

805 CLEANUP_ON_ERROR(tf_model);

806 }

807

808 for (layer = 0; layer < native_model->layers_num; ++layer){

809 switch (native_model->layers[layer].type){

810 case DLT_INPUT:

811 layer_add_res = 0;

812 break;

813 case DLT_CONV2D:

814 layer_add_res = add_conv_layer(tf_model, transpose_op, &op,

815 (ConvolutionalParams *)native_model->layers[layer].params, layer);

816 break;

817 case DLT_DEPTH_TO_SPACE:

818 layer_add_res = add_depth_to_space_layer(tf_model, &op,

819 (DepthToSpaceParams *)native_model->layers[layer].params, layer);

820 break;

821 case DLT_MIRROR_PAD:

822 layer_add_res = add_pad_layer(tf_model, &op,

823 (LayerPadParams *)native_model->layers[layer].params, layer);

824 break;

825 case DLT_MAXIMUM:

826 layer_add_res = add_maximum_layer(tf_model, &op,

827 (DnnLayerMaximumParams *)native_model->layers[layer].params, layer);

828 break;

829 default:

830 CLEANUP_ON_ERROR(tf_model);

831 }

832

833 if (layer_add_res != 0){

834 CLEANUP_ON_ERROR(tf_model);

835 }

836 }

837

838 op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");

839 input.oper = op;

840 input.index = 0;

841 TF_AddInput(op_desc, input);

842 TF_FinishOperation(op_desc, tf_model->status);

843 if (TF_GetCode(tf_model->status) != TF_OK){

844 CLEANUP_ON_ERROR(tf_model);

845 }

846

847 ff_dnn_free_model_native(&model);

848

849 return 0;

850 }

851

852 DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)

853 {

854 DNNModel *model = NULL;

855 TFModel *tf_model = NULL;

856 TFContext *ctx = NULL;

857

858 model = av_mallocz(sizeof(DNNModel));

859 if (!model){

860 return NULL;

861 }

862

863 tf_model = av_mallocz(sizeof(TFModel));

864 if (!tf_model){

865 av_freep(&model);

866 return NULL;

867 }

868 tf_model->model = model;

869 ctx = &tf_model->ctx;

870 ctx->class = &dnn_tensorflow_class;

871

872 //parse options

873 av_opt_set_defaults(ctx);

874 if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {

875 av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);

876 goto err;

877 }

878

879 if (load_tf_model(tf_model, model_filename) != 0){

880 if (load_native_model(tf_model, model_filename) != 0){

881 goto err;

882 }

883 }

884

885 if (ctx->options.nireq <= 0) {

886 ctx->options.nireq = av_cpu_count() / 2 + 1;

887 }

888

889 #if !HAVE_PTHREAD_CANCEL

890 if (ctx->options.async) {

891 ctx->options.async = 0;

892 av_log(filter_ctx, AV_LOG_WARNING, "pthread is not supported, roll back to sync.\n");

893 }

894 #endif

895

896 tf_model->request_queue = ff_safe_queue_create();

897 if (!tf_model->request_queue) {

898 goto err;

899 }

900

901 for (int i = 0; i < ctx->options.nireq; i++) {

902 TFRequestItem *item = av_mallocz(sizeof(*item));

903 if (!item) {

904 goto err;

905 }

906 item->lltask = NULL;

907 item->infer_request = tf_create_inference_request();

908 if (!item->infer_request) {

909 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for TensorFlow inference request\n");

910 av_freep(&item);

911 goto err;

912 }

913 item->status = TF_NewStatus();

914 item->exec_module.start_inference = &tf_start_inference;

915 item->exec_module.callback = &infer_completion_callback;

916 item->exec_module.args = item;

917

918 if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {

919 destroy_request_item(&item);

920 goto err;

921 }

922 }

923

924 tf_model->lltask_queue = ff_queue_create();

925 if (!tf_model->lltask_queue) {

926 goto err;

927 }

928

929 tf_model->task_queue = ff_queue_create();

930 if (!tf_model->task_queue) {

931 goto err;

932 }

933

934 model->model = tf_model;

935 model->get_input = &get_input_tf;

936 model->get_output = &get_output_tf;

937 model->options = options;

938 model->filter_ctx = filter_ctx;

939 model->func_type = func_type;

940

941 return model;

942 err:

943 ff_dnn_free_model_tf(&model);

944 return NULL;

945 }

946

947 static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request) {

948 DNNData input;

949 LastLevelTaskItem *lltask;

950 TaskItem *task;

951 TFInferRequest *infer_request;

952 TFContext *ctx = &tf_model->ctx;

953 int ret = 0;

954

955 lltask = ff_queue_pop_front(tf_model->lltask_queue);

956 av_assert0(lltask);

957 task = lltask->task;

958 request->lltask = lltask;

959

960 ret = get_input_tf(tf_model, &input, task->input_name);

961 if (ret != 0) {

962 goto err;

963 }

964

965 infer_request = request->infer_request;

966 input.height = task->in_frame->height;

967 input.width = task->in_frame->width;

968

969 infer_request->tf_input = av_malloc(sizeof(TF_Output));

970 if (!infer_request->tf_input) {

971 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");

972 ret = AVERROR(ENOMEM);

973 goto err;

974 }

975

976 infer_request->tf_input->oper = TF_GraphOperationByName(tf_model->graph, task->input_name);

977 if (!infer_request->tf_input->oper){

978 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);

979 ret = DNN_GENERIC_ERROR;

980 goto err;

981 }

982 infer_request->tf_input->index = 0;

983

984 infer_request->input_tensor = allocate_input_tensor(&input);

985 if (!infer_request->input_tensor){

986 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");

987 ret = AVERROR(ENOMEM);

988 goto err;

989 }

990 input.data = (float *)TF_TensorData(infer_request->input_tensor);

991

992 switch (tf_model->model->func_type) {

993 case DFT_PROCESS_FRAME:

994 if (task->do_ioproc) {

995 if (tf_model->model->frame_pre_proc != NULL) {

996 tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx);

997 } else {

998 ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);

999 }

1000 }

1001 break;

1002 case DFT_ANALYTICS_DETECT:

1003 ff_frame_to_dnn_detect(task->in_frame, &input, ctx);

1004 break;

1005 default:

1006 avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type);

1007 break;

1008 }

1009

1010 infer_request->tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output));

1011 if (infer_request->tf_outputs == NULL) {

1012 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n");

1013 ret = AVERROR(ENOMEM);

1014 goto err;

1015 }

1016

1017 infer_request->output_tensors = av_calloc(task->nb_output, sizeof(*infer_request->output_tensors));

1018 if (!infer_request->output_tensors) {

1019 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n");

1020 ret = AVERROR(ENOMEM);

1021 goto err;

1022 }

1023

1024 for (int i = 0; i < task->nb_output; ++i) {

1025 infer_request->output_tensors[i] = NULL;

1026 infer_request->tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]);

1027 if (!infer_request->tf_outputs[i].oper) {

1028 av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]);

1029 ret = DNN_GENERIC_ERROR;

1030 goto err;

1031 }

1032 infer_request->tf_outputs[i].index = 0;

1033 }

1034

1035 return 0;

1036 err:

1037 tf_free_request(infer_request);

1038 return ret;

1039 }

1040

1041 static void infer_completion_callback(void *args) {

1042 TFRequestItem *request = args;

1043 LastLevelTaskItem *lltask = request->lltask;

1044 TaskItem *task = lltask->task;

1045 DNNData *outputs;

1046 TFInferRequest *infer_request = request->infer_request;

1047 TFModel *tf_model = task->model;

1048 TFContext *ctx = &tf_model->ctx;

1049

1050 outputs = av_malloc_array(task->nb_output, sizeof(*outputs));

1051 if (!outputs) {

1052 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *outputs\n");

1053 goto err;

1054 }

1055

1056 for (uint32_t i = 0; i < task->nb_output; ++i) {

1057 outputs[i].height = TF_Dim(infer_request->output_tensors[i], 1);

1058 outputs[i].width = TF_Dim(infer_request->output_tensors[i], 2);

1059 outputs[i].channels = TF_Dim(infer_request->output_tensors[i], 3);

1060 outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);

1061 outputs[i].dt = TF_TensorType(infer_request->output_tensors[i]);

1062 }

1063 switch (tf_model->model->func_type) {

1064 case DFT_PROCESS_FRAME:

1065 //it only support 1 output if it's frame in & frame out

1066 if (task->do_ioproc) {

1067 if (tf_model->model->frame_post_proc != NULL) {

1068 tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx);

1069 } else {

1070 ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);

1071 }

1072 } else {

1073 task->out_frame->width = outputs[0].width;

1074 task->out_frame->height = outputs[0].height;

1075 }

1076 break;

1077 case DFT_ANALYTICS_DETECT:

1078 if (!tf_model->model->detect_post_proc) {

1079 av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");

1080 return;

1081 }

1082 tf_model->model->detect_post_proc(task->in_frame, outputs, task->nb_output, tf_model->model->filter_ctx);

1083 break;

1084 default:

1085 av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");

1086 goto err;

1087 }

1088 task->inference_done++;

1089 err:

1090 tf_free_request(infer_request);

1091 av_freep(&outputs);

1092

1093 if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {

1094 destroy_request_item(&request);

1095 av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");

1096 }

1097 }

1098

1099 static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)

1100 {

1101 TFModel *tf_model;

1102 TFContext *ctx;

1103 LastLevelTaskItem *lltask;

1104 TaskItem *task;

1105 int ret = 0;

1106

1107 if (ff_queue_size(lltask_queue) == 0) {

1108 destroy_request_item(&request);

1109 return 0;

1110 }

1111

1112 lltask = ff_queue_peek_front(lltask_queue);

1113 task = lltask->task;

1114 tf_model = task->model;

1115 ctx = &tf_model->ctx;

1116

1117 ret = fill_model_input_tf(tf_model, request);

1118 if (ret != 0) {

1119 goto err;

1120 }

1121

1122 if (task->async) {

1123 if (ff_dnn_start_inference_async(ctx, &request->exec_module) != 0) {

1124 goto err;

1125 }

1126 return 0;

1127 }

1128 else {

1129 ret = tf_start_inference(request);

1130 if (ret != 0) {

1131 goto err;

1132 }

1133 infer_completion_callback(request);

1134 return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;

1135 }

1136 err:

1137 tf_free_request(request->infer_request);

1138 if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {

1139 destroy_request_item(&request);

1140 }

1141 return ret;

1142 }

1143

1144 int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)

1145 {

1146 TFModel *tf_model = model->model;

1147 TFContext *ctx = &tf_model->ctx;

1148 TaskItem *task;

1149 TFRequestItem *request;

1150 int ret = 0;

1151

1152 ret = ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params);

1153 if (ret != 0) {

1154 return ret;

1155 }

1156

1157 task = av_malloc(sizeof(*task));

1158 if (!task) {

1159 av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");

1160 return AVERROR(ENOMEM);

1161 }

1162

1163 ret = ff_dnn_fill_task(task, exec_params, tf_model, ctx->options.async, 1);

1164 if (ret != 0) {

1165 av_freep(&task);

1166 return ret;

1167 }

1168

1169 if (ff_queue_push_back(tf_model->task_queue, task) < 0) {

1170 av_freep(&task);

1171 av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");

1172 return AVERROR(ENOMEM);

1173 }

1174

1175 ret = extract_lltask_from_task(task, tf_model->lltask_queue);

1176 if (ret != 0) {

1177 av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");

1178 return ret;

1179 }

1180

1181 request = ff_safe_queue_pop_front(tf_model->request_queue);

1182 if (!request) {

1183 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");

1184 return AVERROR(EINVAL);

1185 }

1186 return execute_model_tf(request, tf_model->lltask_queue);

1187 }

1188

1189 DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)

1190 {

1191 TFModel *tf_model = model->model;

1192 return ff_dnn_get_result_common(tf_model->task_queue, in, out);

1193 }

1194

1195 int ff_dnn_flush_tf(const DNNModel *model)

1196 {

1197 TFModel *tf_model = model->model;

1198 TFContext *ctx = &tf_model->ctx;

1199 TFRequestItem *request;

1200 int ret;

1201

1202 if (ff_queue_size(tf_model->lltask_queue) == 0) {

1203 // no pending task need to flush

1204 return 0;

1205 }

1206

1207 request = ff_safe_queue_pop_front(tf_model->request_queue);

1208 if (!request) {

1209 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");

1210 return AVERROR(EINVAL);

1211 }

1212

1213 ret = fill_model_input_tf(tf_model, request);

1214 if (ret != 0) {

1215 av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");

1216 if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {

1217 destroy_request_item(&request);

1218 }

1219 return ret;

1220 }

1221

1222 return ff_dnn_start_inference_async(ctx, &request->exec_module);

1223 }

1224

1225 void ff_dnn_free_model_tf(DNNModel **model)

1226 {

1227 TFModel *tf_model;

1228

1229 if (*model){

1230 tf_model = (*model)->model;

1231 while (ff_safe_queue_size(tf_model->request_queue) != 0) {

1232 TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);

1233 destroy_request_item(&item);

1234 }

1235 ff_safe_queue_destroy(tf_model->request_queue);

1236

1237 while (ff_queue_size(tf_model->lltask_queue) != 0) {

1238 LastLevelTaskItem *item = ff_queue_pop_front(tf_model->lltask_queue);

1239 av_freep(&item);

1240 }

1241 ff_queue_destroy(tf_model->lltask_queue);

1242

1243 while (ff_queue_size(tf_model->task_queue) != 0) {

1244 TaskItem *item = ff_queue_pop_front(tf_model->task_queue);

1245 av_frame_free(&item->in_frame);

1246 av_frame_free(&item->out_frame);

1247 av_freep(&item);

1248 }

1249 ff_queue_destroy(tf_model->task_queue);

1250

1251 if (tf_model->graph){

1252 TF_DeleteGraph(tf_model->graph);

1253 }

1254 if (tf_model->session){

1255 TF_CloseSession(tf_model->session, tf_model->status);

1256 TF_DeleteSession(tf_model->session, tf_model->status);

1257 }

1258 if (tf_model->status){

1259 TF_DeleteStatus(tf_model->status);

1260 }

1261 av_freep(&tf_model);

1262 av_freep(model);

1263 }

1264 }

AVFILTER_DEFINE_CLASS

AVFILTER_DEFINE_CLASS(dnn_tensorflow)

ff_dnn_get_result_tf

DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)

Definition: dnn_backend_tf.c:1189

TFOptions::sess_config

char * sess_config

Definition: dnn_backend_tf.c:44

AV_LOG_WARNING

#define AV_LOG_WARNING

Something somehow does not look correct.

Definition: log.h:186

TFInferRequest

Stores execution parameters for single call to the TensorFlow C API.

Definition: dnn_backend_tf.c:69

TFInferRequest::tf_outputs

TF_Output * tf_outputs

Definition: dnn_backend_tf.c:70

execute_model_tf

static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)

Definition: dnn_backend_tf.c:1099

FLAGS

#define FLAGS

Definition: dnn_backend_tf.c:84

status

they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link

AVERROR

Filter the word "frame" indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions

TFModel::graph

TF_Graph * graph

Definition: dnn_backend_tf.c:57

av_opt_set_defaults

void av_opt_set_defaults(void *s)

Set the values of all AVOption fields to their default values.

Definition: opt.c:1459

ff_safe_queue_pop_front

void * ff_safe_queue_pop_front(SafeQueue *sq)

Remove and free first element from the queue in SafeQueue.

Definition: safe_queue.c:105

out

FILE * out

Definition: movenc.c:54

TFModel::ctx

TFContext ctx

Definition: dnn_backend_tf.c:55

DNNAsyncExecModule

Common Async Execution Mechanism for the DNN Backends.

Definition: dnn_backend_common.h:58

DNNFunctionType

Definition: dnn_interface.h:52

extract_lltask_from_task

static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)

Definition: dnn_backend_tf.c:206

add_conv_layer

static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)

Definition: dnn_backend_tf.c:495

ff_queue_pop_front

void * ff_queue_pop_front(Queue *q)

Remove and free first element from the Queue.

Definition: queue.c:151

ff_check_exec_params

int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)

Definition: dnn_backend_common.c:29

ff_queue_size

size_t ff_queue_size(Queue *q)

Return the length of the Queue.

Definition: queue.c:88

DnnLayerMaximumParams::y

float y

Definition: dnn_backend_native_layer_maximum.h:36

DNN_GENERIC_ERROR

#define DNN_GENERIC_ERROR

Definition: dnn_interface.h:33

av_frame_free

void av_frame_free(AVFrame **frame)

Free the frame and any dynamically allocated objects in it, e.g.

Definition: frame.c:99

LastLevelTaskItem

Definition: dnn_backend_common.h:50

ConvolutionalParams::kernel

float * kernel

Definition: dnn_backend_native_layer_conv2d.h:33

test::height

int height

Definition: vc1dsp.c:39

AVFrame

This structure describes decoded (raw) audio or video data.

Definition: frame.h:330

AVFrame::width

int width

Definition: frame.h:402

ff_dnn_load_model_native

DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)

Definition: dnn_backend_native.c:139

SafeQueue

Double-ended queue with mutex locks ensuring data consistency while multithreading.

Definition: safe_queue.c:46

DnnLayerMaximumParams::val

union DnnLayerMaximumParams::@220 val

AVOption

AVOption.

Definition: opt.h:251

DNNModel::frame_pre_proc

FramePrePostProc frame_pre_proc

Definition: dnn_interface.h:101

TFInferRequest::input_tensor

TF_Tensor * input_tensor

Definition: dnn_backend_tf.c:73

data

const char data[16]

Definition: mxf.c:146

DNNExecBaseParams::input_name

const char * input_name

Definition: dnn_interface.h:68

load_tf_model

static int load_tf_model(TFModel *tf_model, const char *model_filename)

Definition: dnn_backend_tf.c:391

dnn_io_proc.h

TFModel::request_queue

SafeQueue * request_queue

Definition: dnn_backend_tf.c:60

TaskItem

Definition: dnn_backend_common.h:36

DNNAsyncExecModule::callback

void(* callback)(void *args)

Completion Callback for the backend.

Definition: dnn_backend_common.h:70

DnnLayerMaximumParams

Definition: dnn_backend_native_layer_maximum.h:33

avio_size

int64_t avio_size(AVIOContext *s)

Get the filesize.

Definition: aviobuf.c:344

tf_sess_config.config

config

Definition: tf_sess_config.py:33

OFFSET

#define OFFSET(x)

Definition: dnn_backend_tf.c:83

load_native_model

static int load_native_model(TFModel *tf_model, const char *model_filename)

Definition: dnn_backend_tf.c:748

av_malloc

#define av_malloc(s)

Definition: tableprint_vlc.h:30

dnn_backend_native_layer_pad.h

destroy_request_item

static void destroy_request_item(TFRequestItem **arg)

Free the TFRequestItem completely.

Definition: dnn_backend_tf.c:192

DNNModel::filter_ctx

AVFilterContext * filter_ctx

Definition: dnn_interface.h:90

ff_queue_create

Queue * ff_queue_create(void)

Create a Queue instance.

Definition: queue.c:47

DLT_INPUT

@ DLT_INPUT

Definition: dnn_backend_native.h:41

ff_dnn_execute_model_tf

int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)

Definition: dnn_backend_tf.c:1144

TaskItem::model

void * model

Definition: dnn_backend_common.h:37

filter_ctx

static FilteringContext * filter_ctx

Definition: transcode.c:51

get_input_tf

static int get_input_tf(void *model, DNNData *input, const char *input_name)

Definition: dnn_backend_tf.c:281

SIGMOID

@ SIGMOID

Definition: dnn_backend_native.h:55

SPACE_CHARS

#define SPACE_CHARS

Definition: dnn_backend_tf.c:361

Queue

Linear double-ended data structure.

Definition: queue.c:33

add_depth_to_space_layer

static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)

Definition: dnn_backend_tf.c:617

ff_queue_push_back

int ff_queue_push_back(Queue *q, void *v)

Add data to the tail of the queue.

Definition: queue.c:130

avassert.h

DNN_BACKEND_COMMON_OPTIONS

#define DNN_BACKEND_COMMON_OPTIONS

Definition: dnn_backend_common.h:31

DNN_TF

@ DNN_TF

Definition: dnn_interface.h:35

AV_LOG_ERROR

#define AV_LOG_ERROR

Something went wrong and cannot losslessly be recovered.

Definition: log.h:180

fill_model_input_tf

static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request)

Definition: dnn_backend_tf.c:947

TFRequestItem::exec_module

DNNAsyncExecModule exec_module

Definition: dnn_backend_tf.c:80

float

Definition: af_crystalizer.c:122

NativeModel::layers_num

int32_t layers_num

Definition: dnn_backend_native.h:128

ConvolutionalParams::input_num

int32_t input_num

Definition: dnn_backend_native_layer_conv2d.h:28

LastLevelTaskItem::task

TaskItem * task

Definition: dnn_backend_common.h:51

DLT_MAXIMUM

@ DLT_MAXIMUM

Definition: dnn_backend_native.h:45

TANH

@ TANH

Definition: dnn_backend_native.h:55

read_graph

static TF_Buffer * read_graph(const char *model_filename)

Definition: dnn_backend_tf.c:226

Layer::type

DNNLayerType type

Definition: dnn_backend_native.h:58

DLT_CONV2D

@ DLT_CONV2D

Definition: dnn_backend_native.h:42

static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)

Perform decode operation.

Definition: anm.c:76

ff_queue_destroy

void ff_queue_destroy(Queue *q)

Destroy the Queue instance.

Definition: queue.c:72

av_assert0

#define av_assert0(cond)

assert() equivalent, that is always enabled.

Definition: avassert.h:37

DNNData

Definition: dnn_interface.h:59

outputs

static const AVFilterPad outputs[]

Definition: af_acontrast.c:172

ConvolutionalParams::activation

DNNActivationFunc activation

Definition: dnn_backend_native_layer_conv2d.h:29

dnn_tensorflow_options

static const AVOption dnn_tensorflow_options[]

Definition: dnn_backend_tf.c:85

ff_dnn_fill_gettingoutput_task

int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)

Allocate input and output frames and fill the Task with execution parameters.

Definition: dnn_backend_common.c:162

LayerPadParams

Definition: dnn_backend_native_layer_pad.h:33

ctx

AVFormatContext * ctx

Definition: movenc.c:48

TaskItem::inference_todo

uint32_t inference_todo

Definition: dnn_backend_common.h:45

arg

const char * arg

Definition: jacosubdec.c:67

dnn_backend_native_layer_depth2space.h

ff_safe_queue_size

size_t ff_safe_queue_size(SafeQueue *sq)

Return the length of the SafeQueue.

Definition: safe_queue.c:80

ff_proc_from_frame_to_dnn

int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)

Definition: dnn_io_proc.c:100

ff_dnn_free_model_native

void ff_dnn_free_model_native(DNNModel **model)

Definition: dnn_backend_native.c:515

ff_dnn_flush_tf

int ff_dnn_flush_tf(const DNNModel *model)

Definition: dnn_backend_tf.c:1195

AVClass

Describe the class of an AVClass context structure.

Definition: log.h:66

Layer::params

void * params

Definition: dnn_backend_native.h:66

ff_frame_to_dnn_detect

int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)

Definition: dnn_io_proc.c:254

NULL

#define NULL

Definition: coverity.c:32

ff_safe_queue_create

SafeQueue * ff_safe_queue_create(void)

Create and initialize a SafeQueue instance.

Definition: safe_queue.c:52

DNNModel::frame_post_proc

FramePrePostProc frame_post_proc

Definition: dnn_interface.h:104

NativeModel

Definition: dnn_backend_native.h:124

av_opt_set_from_string

int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)

Parse the key-value pairs list in opts.

Definition: opt.c:1667

tf_create_inference_request

static TFInferRequest * tf_create_inference_request(void)

Create a TensorFlow inference request.

Definition: dnn_backend_tf.c:137

ff_dnn_async_module_cleanup

int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)

Join the Async Execution thread and set module pointers to NULL.

Definition: dnn_backend_common.c:92

TFModel::task_queue

Queue * task_queue

Definition: dnn_backend_tf.c:62

infer_completion_callback

static void infer_completion_callback(void *args)

Definition: dnn_backend_tf.c:1041

TaskItem::in_frame

AVFrame * in_frame

Definition: dnn_backend_common.h:38

TFModel::status

TF_Status * status

Definition: dnn_backend_tf.c:59

tf_free_request

static void tf_free_request(TFInferRequest *request)

Free the contents of TensorFlow inference request.

Definition: dnn_backend_tf.c:109

transpose_perm

static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)

Interpret the input data as in the following table:

Definition: twinvq.c:623

Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c

Definition: undefined.txt:32

ConvolutionalParams::kernel_size

int32_t kernel_size

Definition: dnn_backend_native_layer_conv2d.h:28

av_cpu_count

int av_cpu_count(void)

Definition: cpu.c:206

options

const OptionDef options[]

test::width

int width

Definition: vc1dsp.c:38

AVIOContext

Bytestream IO Context.

Definition: avio.h:166

DLT_MIRROR_PAD

@ DLT_MIRROR_PAD

Definition: dnn_backend_native.h:44

TaskItem::async

uint8_t async

Definition: dnn_backend_common.h:42

TaskItem::inference_done

uint32_t inference_done

Definition: dnn_backend_common.h:46

cpu.h

DNNModel::detect_post_proc

DetectPostProc detect_post_proc

Definition: dnn_interface.h:106

size

int size

Definition: twinvq_data.h:10344

NativeModel::layers

Layer * layers

Definition: dnn_backend_native.h:127

avio.h

DNNModel::func_type

DNNFunctionType func_type

Definition: dnn_interface.h:92

avpriv_report_missing_feature

void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2

Log a generic warning message about a missing feature.

TFOptions::nireq

uint32_t nireq

Definition: dnn_backend_tf.c:46

dnn_backend_native_layer_conv2d.h

ff_safe_queue_destroy

void ff_safe_queue_destroy(SafeQueue *sq)

Destroy the SafeQueue instance.

Definition: safe_queue.c:69

hex_to_data

static int hex_to_data(uint8_t *data, const char *p)

Definition: dnn_backend_tf.c:362

DNN_FLOAT

@ DNN_FLOAT

Definition: dnn_interface.h:37

NAME_BUFFER_SIZE

#define NAME_BUFFER_SIZE

Definition: dnn_backend_tf.c:493

tf_start_inference

static int tf_start_inference(void *args)

Start synchronous inference for the TensorFlow model.

Definition: dnn_backend_tf.c:158

ff_dnn_fill_task

int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)

Fill the Task for Backend Execution.

Definition: dnn_backend_common.c:56

DepthToSpaceParams

Definition: dnn_backend_native_layer_depth2space.h:33

dnn_backend_native.h

input

and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input

Definition: filter_design.txt:172

ff_dnn_load_model_tf

DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)

Definition: dnn_backend_tf.c:852

RELU

@ RELU

Definition: dnn_backend_native.h:55

ff_safe_queue_push_back

int ff_safe_queue_push_back(SafeQueue *sq, void *v)

Add data to the tail of queue in the SafeQueue after locking mutex.

Definition: safe_queue.c:95

avio_closep

int avio_closep(AVIOContext **s)

Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...

Definition: aviobuf.c:1280

ConvolutionalParams::output_num

int32_t output_num

Definition: dnn_backend_native_layer_conv2d.h:28

add_maximum_layer

static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)

Definition: dnn_backend_tf.c:699

TFContext::options

TFOptions options

Definition: dnn_backend_tf.c:51

#define i(width, name, range_min, range_max)

Definition: cbs_h2645.c:269

LayerPadParams::paddings

int32_t paddings[4][2]

Definition: dnn_backend_native_layer_pad.h:34

DFT_ANALYTICS_DETECT

@ DFT_ANALYTICS_DETECT

Definition: dnn_interface.h:55

TFRequestItem::status

TF_Status * status

Definition: dnn_backend_tf.c:79

TFInferRequest::output_tensors

TF_Tensor ** output_tensors

Definition: dnn_backend_tf.c:71

TFModel::session

TF_Session * session

Definition: dnn_backend_tf.c:58

TFRequestItem::infer_request

TFInferRequest * infer_request

Definition: dnn_backend_tf.c:77

av_malloc_array

#define av_malloc_array(a, b)

Definition: tableprint_vlc.h:31

DNNAsyncExecModule::start_inference

int(* start_inference)(void *request)

Synchronous inference function for the backend with corresponding request item as the argument.

Definition: dnn_backend_common.h:63

DNNAsyncExecModule::args

void * args

Argument for the execution functions.

Definition: dnn_backend_common.h:76

av_toupper

static av_const int av_toupper(int c)

Locale-independent conversion of ASCII characters to uppercase.

Definition: avstring.h:228

av_mallocz

void * av_mallocz(size_t size)

Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...

Definition: mem.c:254

TFOptions::async

uint8_t async

Definition: dnn_backend_tf.c:45

safe_queue.h

TaskItem::output_names

const char ** output_names

Definition: dnn_backend_common.h:41

len

int len

Definition: vorbis_enc_data.h:426

av_calloc

void * av_calloc(size_t nmemb, size_t size)

Definition: mem.c:262

TFInferRequest::tf_input

TF_Output * tf_input

Definition: dnn_backend_tf.c:72

TFContext

Definition: dnn_backend_tf.c:49

DepthToSpaceParams::block_size

int block_size

Definition: dnn_backend_native_layer_depth2space.h:34

ret

Definition: filter_design.txt:187

DNNModel::get_input

int(* get_input)(void *model, DNNData *input, const char *input_name)

Definition: dnn_interface.h:95

DNN_UINT8

@ DNN_UINT8

Definition: dnn_interface.h:37

TFModel::model

DNNModel * model

Definition: dnn_backend_tf.c:56

TFModel

Definition: dnn_backend_tf.c:54

AV_INPUT_BUFFER_PADDING_SIZE

#define AV_INPUT_BUFFER_PADDING_SIZE

Definition: defs.h:40

TaskItem::out_frame

AVFrame * out_frame

Definition: dnn_backend_common.h:39

AVFrame::height

int height

Definition: frame.h:402

allocate_input_tensor

static TF_Tensor * allocate_input_tensor(const DNNData *input)

Definition: dnn_backend_tf.c:259

dnn_backend_common.h

TFRequestItem::lltask

LastLevelTaskItem * lltask

Definition: dnn_backend_tf.c:78

defs.h

avio_read

int avio_read(AVIOContext *s, unsigned char *buf, int size)

Read size bytes from AVIOContext into buf.

Definition: aviobuf.c:633

ff_dnn_get_result_common

DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)

Extract input and output frame from the Task Queue after asynchronous inference.

Definition: dnn_backend_common.c:142

ff_queue_peek_front

void * ff_queue_peek_front(Queue *q)

Return a pointer to the data at the head of the queue.

Definition: queue.c:93

DCO_RGB

@ DCO_RGB

Definition: dnn_interface.h:42

avio_open

int avio_open(AVIOContext **s, const char *url, int flags)

Create and initialize a AVIOContext for accessing the resource indicated by url.

Definition: aviobuf.c:1215

AVFilterContext

An instance of a filter.

Definition: avfilter.h:392

ff_dnn_free_model_tf

void ff_dnn_free_model_tf(DNNModel **model)

Definition: dnn_backend_tf.c:1225

ff_dnn_start_inference_async

int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)

Start asynchronous inference routine for the TensorFlow model on a detached thread.

Definition: dnn_backend_common.c:111

DNNModel

Definition: dnn_interface.h:84

AVIO_FLAG_READ

#define AVIO_FLAG_READ

read-only

Definition: avio.h:623

TaskItem::input_name

const char * input_name

Definition: dnn_backend_common.h:40

DNNModel::options

const char * options

Definition: dnn_interface.h:88

av_freep

#define av_freep(p)

Definition: tableprint_vlc.h:34

TFOptions

Definition: dnn_backend_tf.c:43

free_buffer

static void free_buffer(void *data, size_t length)

Definition: dnn_backend_tf.c:97

int32_t

Definition: audioconvert.c:56

get_output_tf

static int get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)

Definition: dnn_backend_tf.c:317

DNNExecBaseParams

Definition: dnn_interface.h:67

av_log

#define av_log(a,...)

Definition: tableprint_vlc.h:27

TaskItem::do_ioproc

uint8_t do_ioproc

Definition: dnn_backend_common.h:43

DNNModel::get_output

int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)

Definition: dnn_interface.h:97

DLT_DEPTH_TO_SPACE

@ DLT_DEPTH_TO_SPACE

Definition: dnn_backend_native.h:43

avstring.h

dnn_backend_native_layer_maximum.h

AV_OPT_TYPE_STRING

@ AV_OPT_TYPE_STRING

Definition: opt.h:229

TFRequestItem

Definition: dnn_backend_tf.c:76

DNNAsyncStatusType