FFmpeg: libavfilter/dnn/dnn_backend_tf.c Source File

FFmpeg
dnn_backend_tf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN tensorflow backend implementation.
24  */
25 
26 #include "dnn_backend_tf.h"
27 #include "dnn_backend_native.h"
28 #include "dnn_backend_native_layer_conv2d.h"
29 #include "dnn_backend_native_layer_depth2space.h"
30 #include "libavformat/avio.h"
31 #include "libavutil/avassert.h"
32 #include "libavutil/avstring.h"
33 #include "libavutil/cpu.h"
34 #include "libavcodec/defs.h"
35 #include "../internal.h"
36 #include "dnn_backend_native_layer_pad.h"
37 #include "dnn_backend_native_layer_maximum.h"
38 #include "dnn_io_proc.h"
39 #include "dnn_backend_common.h"
40 #include "safe_queue.h"
41 #include <tensorflow/c/c_api.h>
42 
43  typedef struct TFOptions{
44   char *sess_config;
45   uint8_t async;
46   uint32_t nireq;
47 } TFOptions;
48 
49  typedef struct TFContext {
50   const AVClass *class;
51   TFOptions options;
52 } TFContext;
53 
54  typedef struct TFModel{
55   TFContext ctx;
56   DNNModel *model;
57   TF_Graph *graph;
58   TF_Session *session;
59   TF_Status *status;
60   SafeQueue *request_queue;
61   Queue *lltask_queue;
62   Queue *task_queue;
63 } TFModel;
64 
65 /**
66  * Stores execution parameters for single
67  * call to the TensorFlow C API
68  */
69  typedef struct TFInferRequest {
70   TF_Output *tf_outputs;
71   TF_Tensor **output_tensors;
72   TF_Output *tf_input;
73   TF_Tensor *input_tensor;
74 } TFInferRequest;
75 
76  typedef struct TFRequestItem {
77   TFInferRequest *infer_request;
78   LastLevelTaskItem *lltask;
79   TF_Status *status;
80   DNNAsyncExecModule exec_module;
81 } TFRequestItem;
82 
83  #define OFFSET(x) offsetof(TFContext, x)
84  #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
85  static const AVOption dnn_tensorflow_options[] = {
86  { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
87  DNN_BACKEND_COMMON_OPTIONS
88  { NULL }
89 };
90 
91 AVFILTER_DEFINE_CLASS(dnn_tensorflow);
92 
93 static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *lltask_queue);
94 static void infer_completion_callback(void *args);
95 static inline void destroy_request_item(TFRequestItem **arg);
96 
97  static void free_buffer(void *data, size_t length)
98 {
99  av_freep(&data);
100 }
101 
102 /**
103  * Free the contents of TensorFlow inference request.
104  * It does not free the TFInferRequest instance.
105  *
106  * @param request pointer to TFInferRequest instance.
107  * NULL pointer is allowed.
108  */
109  static void tf_free_request(TFInferRequest *request)
110 {
111  if (!request)
112  return;
113  if (request->input_tensor) {
114  TF_DeleteTensor(request->input_tensor);
115  request->input_tensor = NULL;
116  }
117  av_freep(&request->tf_input);
118  av_freep(&request->tf_outputs);
119  if (request->output_tensors) {
120  int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]);
121  for (uint32_t i = 0; i < nb_output; ++i) {
122  if (request->output_tensors[i]) {
123  TF_DeleteTensor(request->output_tensors[i]);
124  request->output_tensors[i] = NULL;
125  }
126  }
127  av_freep(&request->output_tensors);
128  }
129 }
130 
131 /**
132  * Create a TensorFlow inference request. All properties
133  * are initially unallocated and set as NULL.
134  *
135  * @return pointer to the allocated TFInferRequest instance.
136  */
137  static TFInferRequest *tf_create_inference_request(void)
138 {
139  TFInferRequest *infer_request = av_malloc(sizeof(TFInferRequest));
140  if (!infer_request) {
141  return NULL;
142  }
143  infer_request->tf_outputs = NULL;
144  infer_request->tf_input = NULL;
145  infer_request->input_tensor = NULL;
146  infer_request->output_tensors = NULL;
147  return infer_request;
148 }
149 
150 /**
151  * Start synchronous inference for the TensorFlow model.
152  *
153  * @param request pointer to the TFRequestItem for inference
154  * @retval DNN_SUCCESS if execution is successful
155  * @retval DNN_ERROR if execution fails
156  */
157  static DNNReturnType tf_start_inference(void *args)
158 {
159  TFRequestItem *request = args;
160  TFInferRequest *infer_request = request->infer_request;
161  LastLevelTaskItem *lltask = request->lltask;
162  TaskItem *task = lltask->task;
163  TFModel *tf_model = task->model;
164 
165  if (!request) {
166  av_log(&tf_model->ctx, AV_LOG_ERROR, "TFRequestItem is NULL\n");
167  return DNN_ERROR;
168  }
169 
170  TF_SessionRun(tf_model->session, NULL,
171  infer_request->tf_input, &infer_request->input_tensor, 1,
172  infer_request->tf_outputs, infer_request->output_tensors,
173  task->nb_output, NULL, 0, NULL,
174  request->status);
175  if (TF_GetCode(request->status) != TF_OK) {
176  av_log(&tf_model->ctx, AV_LOG_ERROR, "%s", TF_Message(request->status));
177  tf_free_request(infer_request);
178  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
179  destroy_request_item(&request);
180  }
181  return DNN_ERROR;
182  }
183  return DNN_SUCCESS;
184 }
185 
186 /**
187  * Free the TFRequestItem completely.
188  *
189  * @param arg Address of the TFInferRequest instance.
190  */
191  static inline void destroy_request_item(TFRequestItem **arg) {
192  TFRequestItem *request;
193  if (!arg) {
194  return;
195  }
196  request = *arg;
197  tf_free_request(request->infer_request);
198  av_freep(&request->infer_request);
199  av_freep(&request->lltask);
200  TF_DeleteStatus(request->status);
201  ff_dnn_async_module_cleanup(&request->exec_module);
202  av_freep(arg);
203 }
204 
205  static DNNReturnType extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
206 {
207  TFModel *tf_model = task->model;
208  TFContext *ctx = &tf_model->ctx;
209  LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));
210  if (!lltask) {
211  av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");
212  return DNN_ERROR;
213  }
214  task->inference_todo = 1;
215  task->inference_done = 0;
216  lltask->task = task;
217  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
218  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
219  av_freep(&lltask);
220  return DNN_ERROR;
221  }
222  return DNN_SUCCESS;
223 }
224 
225  static TF_Buffer *read_graph(const char *model_filename)
226 {
227  TF_Buffer *graph_buf;
228  unsigned char *graph_data = NULL;
229  AVIOContext *model_file_context;
230  long size, bytes_read;
231 
232  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
233  return NULL;
234  }
235 
236  size = avio_size(model_file_context);
237 
238  graph_data = av_malloc(size);
239  if (!graph_data){
240  avio_closep(&model_file_context);
241  return NULL;
242  }
243  bytes_read = avio_read(model_file_context, graph_data, size);
244  avio_closep(&model_file_context);
245  if (bytes_read != size){
246  av_freep(&graph_data);
247  return NULL;
248  }
249 
250  graph_buf = TF_NewBuffer();
251  graph_buf->data = graph_data;
252  graph_buf->length = size;
253  graph_buf->data_deallocator = free_buffer;
254 
255  return graph_buf;
256 }
257 
258  static TF_Tensor *allocate_input_tensor(const DNNData *input)
259 {
260  TF_DataType dt;
261  size_t size;
262  int64_t input_dims[] = {1, input->height, input->width, input->channels};
263  switch (input->dt) {
264  case DNN_FLOAT:
265  dt = TF_FLOAT;
266  size = sizeof(float);
267  break;
268  case DNN_UINT8:
269  dt = TF_UINT8;
270  size = 1;
271  break;
272  default:
273  av_assert0(!"should not reach here");
274  }
275 
276  return TF_AllocateTensor(dt, input_dims, 4,
277  input_dims[1] * input_dims[2] * input_dims[3] * size);
278 }
279 
280  static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
281 {
282  TFModel *tf_model = model;
283  TFContext *ctx = &tf_model->ctx;
284  TF_Status *status;
285  int64_t dims[4];
286 
287  TF_Output tf_output;
288  tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);
289  if (!tf_output.oper) {
290  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
291  return DNN_ERROR;
292  }
293 
294  tf_output.index = 0;
295  input->dt = TF_OperationOutputType(tf_output);
296  input->order = DCO_RGB;
297 
298  status = TF_NewStatus();
299  TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);
300  if (TF_GetCode(status) != TF_OK){
301  TF_DeleteStatus(status);
302  av_log(ctx, AV_LOG_ERROR, "Failed to get input tensor shape: number of dimension incorrect\n");
303  return DNN_ERROR;
304  }
305  TF_DeleteStatus(status);
306 
307  // currently only NHWC is supported
308  av_assert0(dims[0] == 1 || dims[0] == -1);
309  input->height = dims[1];
310  input->width = dims[2];
311  input->channels = dims[3];
312 
313  return DNN_SUCCESS;
314 }
315 
316  static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height,
317  const char *output_name, int *output_width, int *output_height)
318 {
319  DNNReturnType ret;
320  TFModel *tf_model = model;
321  TFContext *ctx = &tf_model->ctx;
322  TaskItem task;
323  TFRequestItem *request;
324  DNNExecBaseParams exec_params = {
325  .input_name = input_name,
326  .output_names = &output_name,
327  .nb_output = 1,
328  .in_frame = NULL,
329  .out_frame = NULL,
330  };
331 
332  if (ff_dnn_fill_gettingoutput_task(&task, &exec_params, tf_model, input_height, input_width, ctx) != DNN_SUCCESS) {
333  goto err;
334  }
335 
336  if (extract_lltask_from_task(&task, tf_model->lltask_queue) != DNN_SUCCESS) {
337  av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
338  ret = DNN_ERROR;
339  goto err;
340  }
341 
342  request = ff_safe_queue_pop_front(tf_model->request_queue);
343  if (!request) {
344  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
345  ret = DNN_ERROR;
346  goto err;
347  }
348 
349  ret = execute_model_tf(request, tf_model->lltask_queue);
350  *output_width = task.out_frame->width;
351  *output_height = task.out_frame->height;
352 
353 err:
354  av_frame_free(&task.out_frame);
355  av_frame_free(&task.in_frame);
356  return ret;
357 }
358 
359  #define SPACE_CHARS " \t\r\n"
360  static int hex_to_data(uint8_t *data, const char *p)
361 {
362  int c, len, v;
363 
364  len = 0;
365  v = 1;
366  for (;;) {
367  p += strspn(p, SPACE_CHARS);
368  if (*p == '0円')
369  break;
370  c = av_toupper((unsigned char) *p++);
371  if (c >= '0' && c <= '9')
372  c = c - '0';
373  else if (c >= 'A' && c <= 'F')
374  c = c - 'A' + 10;
375  else
376  break;
377  v = (v << 4) | c;
378  if (v & 0x100) {
379  if (data) {
380  data[len] = v;
381  }
382  len++;
383  v = 1;
384  }
385  }
386  return len;
387 }
388 
389  static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
390 {
391  TFContext *ctx = &tf_model->ctx;
392  TF_Buffer *graph_def;
393  TF_ImportGraphDefOptions *graph_opts;
394  TF_SessionOptions *sess_opts;
395  const TF_Operation *init_op;
396  uint8_t *sess_config = NULL;
397  int sess_config_length = 0;
398 
399  // prepare the sess config data
400  if (tf_model->ctx.options.sess_config != NULL) {
401  const char *config;
402  /*
403  tf_model->ctx.options.sess_config is hex to present the serialized proto
404  required by TF_SetConfig below, so we need to first generate the serialized
405  proto in a python script, tools/python/tf_sess_config.py is a script example
406  to generate the configs of sess_config.
407  */
408  if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {
409  av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");
410  return DNN_ERROR;
411  }
412  config = tf_model->ctx.options.sess_config + 2;
413  sess_config_length = hex_to_data(NULL, config);
414 
415  sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);
416  if (!sess_config) {
417  av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");
418  return DNN_ERROR;
419  }
420  if (hex_to_data(sess_config, config) < 0) {
421  av_log(ctx, AV_LOG_ERROR, "failed to convert hex to data\n");
422  return DNN_ERROR;
423  }
424  }
425 
426  graph_def = read_graph(model_filename);
427  if (!graph_def){
428  av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);
429  av_freep(&sess_config);
430  return DNN_ERROR;
431  }
432  tf_model->graph = TF_NewGraph();
433  tf_model->status = TF_NewStatus();
434  graph_opts = TF_NewImportGraphDefOptions();
435  TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
436  TF_DeleteImportGraphDefOptions(graph_opts);
437  TF_DeleteBuffer(graph_def);
438  if (TF_GetCode(tf_model->status) != TF_OK){
439  TF_DeleteGraph(tf_model->graph);
440  TF_DeleteStatus(tf_model->status);
441  av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");
442  av_freep(&sess_config);
443  return DNN_ERROR;
444  }
445 
446  init_op = TF_GraphOperationByName(tf_model->graph, "init");
447  sess_opts = TF_NewSessionOptions();
448 
449  if (sess_config) {
450  TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);
451  av_freep(&sess_config);
452  if (TF_GetCode(tf_model->status) != TF_OK) {
453  TF_DeleteGraph(tf_model->graph);
454  TF_DeleteStatus(tf_model->status);
455  TF_DeleteSessionOptions(sess_opts);
456  av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",
457  tf_model->ctx.options.sess_config);
458  return DNN_ERROR;
459  }
460  }
461 
462  tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
463  TF_DeleteSessionOptions(sess_opts);
464  if (TF_GetCode(tf_model->status) != TF_OK)
465  {
466  TF_DeleteGraph(tf_model->graph);
467  TF_DeleteStatus(tf_model->status);
468  av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");
469  return DNN_ERROR;
470  }
471 
472  // Run initialization operation with name "init" if it is present in graph
473  if (init_op){
474  TF_SessionRun(tf_model->session, NULL,
475  NULL, NULL, 0,
476  NULL, NULL, 0,
477  &init_op, 1, NULL, tf_model->status);
478  if (TF_GetCode(tf_model->status) != TF_OK)
479  {
480  TF_DeleteSession(tf_model->session, tf_model->status);
481  TF_DeleteGraph(tf_model->graph);
482  TF_DeleteStatus(tf_model->status);
483  av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");
484  return DNN_ERROR;
485  }
486  }
487 
488  return DNN_SUCCESS;
489 }
490 
491  #define NAME_BUFFER_SIZE 256
492 
493  static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
494  ConvolutionalParams* params, const int layer)
495 {
496  TFContext *ctx = &tf_model->ctx;
497  TF_Operation *op;
498  TF_OperationDescription *op_desc;
499  TF_Output input;
500  int64_t strides[] = {1, 1, 1, 1};
501  TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
502  int64_t dims[4];
503  int dims_len;
504  char name_buffer[NAME_BUFFER_SIZE];
505  int32_t size;
506 
507  size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
508  input.index = 0;
509 
510  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
511  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
512  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
513  dims[0] = params->output_num;
514  dims[1] = params->kernel_size;
515  dims[2] = params->kernel_size;
516  dims[3] = params->input_num;
517  dims_len = 4;
518  kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
519  memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
520  TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
521  if (TF_GetCode(tf_model->status) != TF_OK){
522  goto err;
523  }
524  op = TF_FinishOperation(op_desc, tf_model->status);
525  if (TF_GetCode(tf_model->status) != TF_OK){
526  goto err;
527  }
528 
529  snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
530  op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
531  input.oper = op;
532  TF_AddInput(op_desc, input);
533  input.oper = transpose_op;
534  TF_AddInput(op_desc, input);
535  TF_SetAttrType(op_desc, "T", TF_FLOAT);
536  TF_SetAttrType(op_desc, "Tperm", TF_INT32);
537  op = TF_FinishOperation(op_desc, tf_model->status);
538  if (TF_GetCode(tf_model->status) != TF_OK){
539  goto err;
540  }
541 
542  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
543  op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
544  input.oper = *cur_op;
545  TF_AddInput(op_desc, input);
546  input.oper = op;
547  TF_AddInput(op_desc, input);
548  TF_SetAttrType(op_desc, "T", TF_FLOAT);
549  TF_SetAttrIntList(op_desc, "strides", strides, 4);
550  TF_SetAttrString(op_desc, "padding", "VALID", 5);
551  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
552  if (TF_GetCode(tf_model->status) != TF_OK){
553  goto err;
554  }
555 
556  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
557  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
558  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
559  dims[0] = params->output_num;
560  dims_len = 1;
561  biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
562  memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
563  TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
564  if (TF_GetCode(tf_model->status) != TF_OK){
565  goto err;
566  }
567  op = TF_FinishOperation(op_desc, tf_model->status);
568  if (TF_GetCode(tf_model->status) != TF_OK){
569  goto err;
570  }
571 
572  snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
573  op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
574  input.oper = *cur_op;
575  TF_AddInput(op_desc, input);
576  input.oper = op;
577  TF_AddInput(op_desc, input);
578  TF_SetAttrType(op_desc, "T", TF_FLOAT);
579  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
580  if (TF_GetCode(tf_model->status) != TF_OK){
581  goto err;
582  }
583 
584  snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
585  switch (params->activation){
586  case RELU:
587  op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
588  break;
589  case TANH:
590  op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
591  break;
592  case SIGMOID:
593  op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
594  break;
595  default:
596  avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
597  return DNN_ERROR;
598  }
599  input.oper = *cur_op;
600  TF_AddInput(op_desc, input);
601  TF_SetAttrType(op_desc, "T", TF_FLOAT);
602  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
603  if (TF_GetCode(tf_model->status) != TF_OK){
604  goto err;
605  }
606 
607  return DNN_SUCCESS;
608 err:
609  TF_DeleteTensor(kernel_tensor);
610  TF_DeleteTensor(biases_tensor);
611  av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
612  return DNN_ERROR;
613 }
614 
615  static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
616  DepthToSpaceParams *params, const int layer)
617 {
618  TFContext *ctx = &tf_model->ctx;
619  TF_OperationDescription *op_desc;
620  TF_Output input;
621  char name_buffer[NAME_BUFFER_SIZE];
622 
623  snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
624  op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
625  input.oper = *cur_op;
626  input.index = 0;
627  TF_AddInput(op_desc, input);
628  TF_SetAttrType(op_desc, "T", TF_FLOAT);
629  TF_SetAttrInt(op_desc, "block_size", params->block_size);
630  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
631  if (TF_GetCode(tf_model->status) != TF_OK){
632  av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);
633  return DNN_ERROR;
634  }
635 
636  return DNN_SUCCESS;
637 }
638 
639  static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
640  LayerPadParams *params, const int layer)
641 {
642  TFContext *ctx = &tf_model->ctx;
643  TF_Operation *op;
644  TF_Tensor *tensor;
645  TF_OperationDescription *op_desc;
646  TF_Output input;
647  int32_t *pads;
648  int64_t pads_shape[] = {4, 2};
649 
650  char name_buffer[NAME_BUFFER_SIZE];
651  snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
652 
653  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
654  TF_SetAttrType(op_desc, "dtype", TF_INT32);
655  tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
656  pads = (int32_t *)TF_TensorData(tensor);
657  pads[0] = params->paddings[0][0];
658  pads[1] = params->paddings[0][1];
659  pads[2] = params->paddings[1][0];
660  pads[3] = params->paddings[1][1];
661  pads[4] = params->paddings[2][0];
662  pads[5] = params->paddings[2][1];
663  pads[6] = params->paddings[3][0];
664  pads[7] = params->paddings[3][1];
665  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
666  if (TF_GetCode(tf_model->status) != TF_OK){
667  TF_DeleteTensor(tensor);
668  av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
669  return DNN_ERROR;
670  }
671  op = TF_FinishOperation(op_desc, tf_model->status);
672  if (TF_GetCode(tf_model->status) != TF_OK){
673  TF_DeleteTensor(tensor);
674  av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
675  return DNN_ERROR;
676  }
677 
678  op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
679  input.oper = *cur_op;
680  input.index = 0;
681  TF_AddInput(op_desc, input);
682  input.oper = op;
683  TF_AddInput(op_desc, input);
684  TF_SetAttrType(op_desc, "T", TF_FLOAT);
685  TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
686  TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
687  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
688  if (TF_GetCode(tf_model->status) != TF_OK){
689  TF_DeleteTensor(tensor);
690  av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
691  return DNN_ERROR;
692  }
693 
694  return DNN_SUCCESS;
695 }
696 
697  static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
698  DnnLayerMaximumParams *params, const int layer)
699 {
700  TFContext *ctx = &tf_model->ctx;
701  TF_Operation *op;
702  TF_Tensor *tensor;
703  TF_OperationDescription *op_desc;
704  TF_Output input;
705  float *y;
706 
707  char name_buffer[NAME_BUFFER_SIZE];
708  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
709 
710  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
711  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
712  tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
713  y = (float *)TF_TensorData(tensor);
714  *y = params->val.y;
715  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
716  if (TF_GetCode(tf_model->status) != TF_OK){
717  TF_DeleteTensor(tensor);
718  av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
719  return DNN_ERROR;
720  }
721  op = TF_FinishOperation(op_desc, tf_model->status);
722  if (TF_GetCode(tf_model->status) != TF_OK){
723  TF_DeleteTensor(tensor);
724  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
725  return DNN_ERROR;
726  }
727 
728  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
729  op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
730  input.oper = *cur_op;
731  input.index = 0;
732  TF_AddInput(op_desc, input);
733  input.oper = op;
734  TF_AddInput(op_desc, input);
735  TF_SetAttrType(op_desc, "T", TF_FLOAT);
736  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
737  if (TF_GetCode(tf_model->status) != TF_OK){
738  TF_DeleteTensor(tensor);
739  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
740  return DNN_ERROR;
741  }
742 
743  return DNN_SUCCESS;
744 }
745 
746  static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
747 {
748  TFContext *ctx = &tf_model->ctx;
749  int32_t layer;
750  TF_OperationDescription *op_desc;
751  TF_Operation *op;
752  TF_Operation *transpose_op;
753  TF_Tensor *tensor = NULL;
754  TF_Output input;
755  int32_t *transpose_perm;
756  int64_t transpose_perm_shape[] = {4};
757  int64_t input_shape[] = {1, -1, -1, -1};
758  DNNReturnType layer_add_res;
759  DNNModel *model = NULL;
760  NativeModel *native_model;
761 
762  model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
763  if (!model){
764  av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
765  return DNN_ERROR;
766  }
767 
768  native_model = model->model;
769  tf_model->graph = TF_NewGraph();
770  tf_model->status = TF_NewStatus();
771 
772 #define CLEANUP_ON_ERROR(tf_model) \
773  { \
774  TF_DeleteTensor(tensor); \
775  TF_DeleteGraph(tf_model->graph); \
776  TF_DeleteStatus(tf_model->status); \
777  av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
778  return DNN_ERROR; \
779  }
780 
781  op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
782  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
783  TF_SetAttrShape(op_desc, "shape", input_shape, 4);
784  op = TF_FinishOperation(op_desc, tf_model->status);
785  if (TF_GetCode(tf_model->status) != TF_OK){
786  CLEANUP_ON_ERROR(tf_model);
787  }
788 
789  op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
790  TF_SetAttrType(op_desc, "dtype", TF_INT32);
791  tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
792  transpose_perm = (int32_t *)TF_TensorData(tensor);
793  transpose_perm[0] = 1;
794  transpose_perm[1] = 2;
795  transpose_perm[2] = 3;
796  transpose_perm[3] = 0;
797  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
798  if (TF_GetCode(tf_model->status) != TF_OK){
799  CLEANUP_ON_ERROR(tf_model);
800  }
801  transpose_op = TF_FinishOperation(op_desc, tf_model->status);
802  if (TF_GetCode(tf_model->status) != TF_OK){
803  CLEANUP_ON_ERROR(tf_model);
804  }
805 
806  for (layer = 0; layer < native_model->layers_num; ++layer){
807  switch (native_model->layers[layer].type){
808  case DLT_INPUT:
809  layer_add_res = DNN_SUCCESS;
810  break;
811  case DLT_CONV2D:
812  layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
813  (ConvolutionalParams *)native_model->layers[layer].params, layer);
814  break;
815  case DLT_DEPTH_TO_SPACE:
816  layer_add_res = add_depth_to_space_layer(tf_model, &op,
817  (DepthToSpaceParams *)native_model->layers[layer].params, layer);
818  break;
819  case DLT_MIRROR_PAD:
820  layer_add_res = add_pad_layer(tf_model, &op,
821  (LayerPadParams *)native_model->layers[layer].params, layer);
822  break;
823  case DLT_MAXIMUM:
824  layer_add_res = add_maximum_layer(tf_model, &op,
825  (DnnLayerMaximumParams *)native_model->layers[layer].params, layer);
826  break;
827  default:
828  CLEANUP_ON_ERROR(tf_model);
829  }
830 
831  if (layer_add_res != DNN_SUCCESS){
832  CLEANUP_ON_ERROR(tf_model);
833  }
834  }
835 
836  op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
837  input.oper = op;
838  input.index = 0;
839  TF_AddInput(op_desc, input);
840  TF_FinishOperation(op_desc, tf_model->status);
841  if (TF_GetCode(tf_model->status) != TF_OK){
842  CLEANUP_ON_ERROR(tf_model);
843  }
844 
845  ff_dnn_free_model_native(&model);
846 
847  return DNN_SUCCESS;
848 }
849 
850  DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
851 {
852  DNNModel *model = NULL;
853  TFModel *tf_model = NULL;
854  TFContext *ctx = NULL;
855 
856  model = av_mallocz(sizeof(DNNModel));
857  if (!model){
858  return NULL;
859  }
860 
861  tf_model = av_mallocz(sizeof(TFModel));
862  if (!tf_model){
863  av_freep(&model);
864  return NULL;
865  }
866  tf_model->model = model;
867  ctx = &tf_model->ctx;
868  ctx->class = &dnn_tensorflow_class;
869 
870  //parse options
871  av_opt_set_defaults(ctx);
872  if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {
873  av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
874  goto err;
875  }
876 
877  if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
878  if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
879  goto err;
880  }
881  }
882 
883  if (ctx->options.nireq <= 0) {
884  ctx->options.nireq = av_cpu_count() / 2 + 1;
885  }
886 
887 #if !HAVE_PTHREAD_CANCEL
888  if (ctx->options.async) {
889  ctx->options.async = 0;
890  av_log(filter_ctx, AV_LOG_WARNING, "pthread is not supported, roll back to sync.\n");
891  }
892 #endif
893 
894  tf_model->request_queue = ff_safe_queue_create();
895  if (!tf_model->request_queue) {
896  goto err;
897  }
898 
899  for (int i = 0; i < ctx->options.nireq; i++) {
900  TFRequestItem *item = av_mallocz(sizeof(*item));
901  if (!item) {
902  goto err;
903  }
904  item->lltask = NULL;
905  item->infer_request = tf_create_inference_request();
906  if (!item->infer_request) {
907  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for TensorFlow inference request\n");
908  av_freep(&item);
909  goto err;
910  }
911  item->status = TF_NewStatus();
912  item->exec_module.start_inference = &tf_start_inference;
913  item->exec_module.callback = &infer_completion_callback;
914  item->exec_module.args = item;
915 
916  if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {
917  destroy_request_item(&item);
918  goto err;
919  }
920  }
921 
922  tf_model->lltask_queue = ff_queue_create();
923  if (!tf_model->lltask_queue) {
924  goto err;
925  }
926 
927  tf_model->task_queue = ff_queue_create();
928  if (!tf_model->task_queue) {
929  goto err;
930  }
931 
932  model->model = tf_model;
933  model->get_input = &get_input_tf;
934  model->get_output = &get_output_tf;
935  model->options = options;
936  model->filter_ctx = filter_ctx;
937  model->func_type = func_type;
938 
939  return model;
940 err:
941  ff_dnn_free_model_tf(&model);
942  return NULL;
943 }
944 
945  static DNNReturnType fill_model_input_tf(TFModel *tf_model, TFRequestItem *request) {
946  DNNData input;
947  LastLevelTaskItem *lltask;
948  TaskItem *task;
949  TFInferRequest *infer_request;
950  TFContext *ctx = &tf_model->ctx;
951 
952  lltask = ff_queue_pop_front(tf_model->lltask_queue);
953  av_assert0(lltask);
954  task = lltask->task;
955  request->lltask = lltask;
956 
957  if (get_input_tf(tf_model, &input, task->input_name) != DNN_SUCCESS) {
958  goto err;
959  }
960 
961  infer_request = request->infer_request;
962  input.height = task->in_frame->height;
963  input.width = task->in_frame->width;
964 
965  infer_request->tf_input = av_malloc(sizeof(TF_Output));
966  if (!infer_request->tf_input) {
967  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
968  goto err;
969  }
970 
971  infer_request->tf_input->oper = TF_GraphOperationByName(tf_model->graph, task->input_name);
972  if (!infer_request->tf_input->oper){
973  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
974  goto err;
975  }
976  infer_request->tf_input->index = 0;
977 
978  infer_request->input_tensor = allocate_input_tensor(&input);
979  if (!infer_request->input_tensor){
980  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
981  goto err;
982  }
983  input.data = (float *)TF_TensorData(infer_request->input_tensor);
984 
985  switch (tf_model->model->func_type) {
986  case DFT_PROCESS_FRAME:
987  if (task->do_ioproc) {
988  if (tf_model->model->frame_pre_proc != NULL) {
989  tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx);
990  } else {
991  ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);
992  }
993  }
994  break;
995  case DFT_ANALYTICS_DETECT:
996  ff_frame_to_dnn_detect(task->in_frame, &input, ctx);
997  break;
998  default:
999  avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type);
1000  break;
1001  }
1002 
1003  infer_request->tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output));
1004  if (infer_request->tf_outputs == NULL) {
1005  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n");
1006  goto err;
1007  }
1008 
1009  infer_request->output_tensors = av_calloc(task->nb_output, sizeof(*infer_request->output_tensors));
1010  if (!infer_request->output_tensors) {
1011  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n");
1012  goto err;
1013  }
1014 
1015  for (int i = 0; i < task->nb_output; ++i) {
1016  infer_request->output_tensors[i] = NULL;
1017  infer_request->tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]);
1018  if (!infer_request->tf_outputs[i].oper) {
1019  av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]);
1020  goto err;
1021  }
1022  infer_request->tf_outputs[i].index = 0;
1023  }
1024 
1025  return DNN_SUCCESS;
1026 err:
1027  tf_free_request(infer_request);
1028  return DNN_ERROR;
1029 }
1030 
1031  static void infer_completion_callback(void *args) {
1032  TFRequestItem *request = args;
1033  LastLevelTaskItem *lltask = request->lltask;
1034  TaskItem *task = lltask->task;
1035  DNNData *outputs;
1036  TFInferRequest *infer_request = request->infer_request;
1037  TFModel *tf_model = task->model;
1038  TFContext *ctx = &tf_model->ctx;
1039 
1040  outputs = av_malloc_array(task->nb_output, sizeof(*outputs));
1041  if (!outputs) {
1042  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *outputs\n");
1043  goto err;
1044  }
1045 
1046  for (uint32_t i = 0; i < task->nb_output; ++i) {
1047  outputs[i].height = TF_Dim(infer_request->output_tensors[i], 1);
1048  outputs[i].width = TF_Dim(infer_request->output_tensors[i], 2);
1049  outputs[i].channels = TF_Dim(infer_request->output_tensors[i], 3);
1050  outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
1051  outputs[i].dt = TF_TensorType(infer_request->output_tensors[i]);
1052  }
1053  switch (tf_model->model->func_type) {
1054  case DFT_PROCESS_FRAME:
1055  //it only support 1 output if it's frame in & frame out
1056  if (task->do_ioproc) {
1057  if (tf_model->model->frame_post_proc != NULL) {
1058  tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx);
1059  } else {
1060  ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
1061  }
1062  } else {
1063  task->out_frame->width = outputs[0].width;
1064  task->out_frame->height = outputs[0].height;
1065  }
1066  break;
1067  case DFT_ANALYTICS_DETECT:
1068  if (!tf_model->model->detect_post_proc) {
1069  av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
1070  return;
1071  }
1072  tf_model->model->detect_post_proc(task->in_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
1073  break;
1074  default:
1075  av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
1076  goto err;
1077  }
1078  task->inference_done++;
1079 err:
1080  tf_free_request(infer_request);
1081  av_freep(&outputs);
1082 
1083  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1084  destroy_request_item(&request);
1085  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
1086  }
1087 }
1088 
1089  static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
1090 {
1091  TFModel *tf_model;
1092  TFContext *ctx;
1093  LastLevelTaskItem *lltask;
1094  TaskItem *task;
1095 
1096  if (ff_queue_size(lltask_queue) == 0) {
1097  destroy_request_item(&request);
1098  return DNN_SUCCESS;
1099  }
1100 
1101  lltask = ff_queue_peek_front(lltask_queue);
1102  task = lltask->task;
1103  tf_model = task->model;
1104  ctx = &tf_model->ctx;
1105 
1106  if (fill_model_input_tf(tf_model, request) != DNN_SUCCESS) {
1107  goto err;
1108  }
1109 
1110  if (task->async) {
1111  if (ff_dnn_start_inference_async(ctx, &request->exec_module) != DNN_SUCCESS) {
1112  goto err;
1113  }
1114  return DNN_SUCCESS;
1115  } else {
1116  if (tf_start_inference(request) != DNN_SUCCESS) {
1117  goto err;
1118  }
1119  infer_completion_callback(request);
1120  return (task->inference_done == task->inference_todo) ? DNN_SUCCESS : DNN_ERROR;
1121  }
1122 err:
1123  tf_free_request(request->infer_request);
1124  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1125  destroy_request_item(&request);
1126  }
1127  return DNN_ERROR;
1128 }
1129 
1130  DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
1131 {
1132  TFModel *tf_model = model->model;
1133  TFContext *ctx = &tf_model->ctx;
1134  TaskItem *task;
1135  TFRequestItem *request;
1136 
1137  if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) {
1138  return DNN_ERROR;
1139  }
1140 
1141  task = av_malloc(sizeof(*task));
1142  if (!task) {
1143  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
1144  return DNN_ERROR;
1145  }
1146 
1147  if (ff_dnn_fill_task(task, exec_params, tf_model, ctx->options.async, 1) != DNN_SUCCESS) {
1148  av_freep(&task);
1149  return DNN_ERROR;
1150  }
1151 
1152  if (ff_queue_push_back(tf_model->task_queue, task) < 0) {
1153  av_freep(&task);
1154  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
1155  return DNN_ERROR;
1156  }
1157 
1158  if (extract_lltask_from_task(task, tf_model->lltask_queue) != DNN_SUCCESS) {
1159  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
1160  return DNN_ERROR;
1161  }
1162 
1163  request = ff_safe_queue_pop_front(tf_model->request_queue);
1164  if (!request) {
1165  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1166  return DNN_ERROR;
1167  }
1168  return execute_model_tf(request, tf_model->lltask_queue);
1169 }
1170 
1171  DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
1172 {
1173  TFModel *tf_model = model->model;
1174  return ff_dnn_get_result_common(tf_model->task_queue, in, out);
1175 }
1176 
1177  DNNReturnType ff_dnn_flush_tf(const DNNModel *model)
1178 {
1179  TFModel *tf_model = model->model;
1180  TFContext *ctx = &tf_model->ctx;
1181  TFRequestItem *request;
1182  DNNReturnType ret;
1183 
1184  if (ff_queue_size(tf_model->lltask_queue) == 0) {
1185  // no pending task need to flush
1186  return DNN_SUCCESS;
1187  }
1188 
1189  request = ff_safe_queue_pop_front(tf_model->request_queue);
1190  if (!request) {
1191  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1192  return DNN_ERROR;
1193  }
1194 
1195  ret = fill_model_input_tf(tf_model, request);
1196  if (ret != DNN_SUCCESS) {
1197  av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
1198  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1199  destroy_request_item(&request);
1200  }
1201  return ret;
1202  }
1203 
1204  return ff_dnn_start_inference_async(ctx, &request->exec_module);
1205 }
1206 
1207  void ff_dnn_free_model_tf(DNNModel **model)
1208 {
1209  TFModel *tf_model;
1210 
1211  if (*model){
1212  tf_model = (*model)->model;
1213  while (ff_safe_queue_size(tf_model->request_queue) != 0) {
1214  TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);
1215  destroy_request_item(&item);
1216  }
1217  ff_safe_queue_destroy(tf_model->request_queue);
1218 
1219  while (ff_queue_size(tf_model->lltask_queue) != 0) {
1220  LastLevelTaskItem *item = ff_queue_pop_front(tf_model->lltask_queue);
1221  av_freep(&item);
1222  }
1223  ff_queue_destroy(tf_model->lltask_queue);
1224 
1225  while (ff_queue_size(tf_model->task_queue) != 0) {
1226  TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
1227  av_frame_free(&item->in_frame);
1228  av_frame_free(&item->out_frame);
1229  av_freep(&item);
1230  }
1231  ff_queue_destroy(tf_model->task_queue);
1232 
1233  if (tf_model->graph){
1234  TF_DeleteGraph(tf_model->graph);
1235  }
1236  if (tf_model->session){
1237  TF_CloseSession(tf_model->session, tf_model->status);
1238  TF_DeleteSession(tf_model->session, tf_model->status);
1239  }
1240  if (tf_model->status){
1241  TF_DeleteStatus(tf_model->status);
1242  }
1243  av_freep(&tf_model);
1244  av_freep(model);
1245  }
1246 }
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(dnn_tensorflow)
ff_dnn_get_result_tf
DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_tf.c:1171
TFOptions::sess_config
char * sess_config
Definition: dnn_backend_tf.c:44
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
TFInferRequest
Stores execution parameters for single call to the TensorFlow C API.
Definition: dnn_backend_tf.c:69
TFInferRequest::tf_outputs
TF_Output * tf_outputs
Definition: dnn_backend_tf.c:70
FLAGS
#define FLAGS
Definition: dnn_backend_tf.c:84
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
ff_dnn_fill_task
DNNReturnType ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:56
filter_ctx
static FilteringContext * filter_ctx
Definition: transcoding.c:49
TFModel::graph
TF_Graph * graph
Definition: dnn_backend_tf.c:57
av_opt_set_defaults
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
Definition: opt.c:1364
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
out
FILE * out
Definition: movenc.c:54
TFModel::ctx
TFContext ctx
Definition: dnn_backend_tf.c:55
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:58
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
tf_start_inference
static DNNReturnType tf_start_inference(void *args)
Start synchronous inference for the TensorFlow model.
Definition: dnn_backend_tf.c:157
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:29
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DnnLayerMaximumParams::y
float y
Definition: dnn_backend_native_layer_maximum.h:36
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:109
LastLevelTaskItem
Definition: dnn_backend_common.h:50
ConvolutionalParams::kernel
float * kernel
Definition: dnn_backend_native_layer_conv2d.h:33
ff_dnn_start_inference_async
DNNReturnType ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
Definition: dnn_backend_common.c:111
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:317
AVFrame::width
int width
Definition: frame.h:389
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:139
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
AVOption
AVOption.
Definition: opt.h:247
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
TFInferRequest::input_tensor
TF_Tensor * input_tensor
Definition: dnn_backend_tf.c:73
data
const char data[16]
Definition: mxf.c:143
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
fill_model_input_tf
static DNNReturnType fill_model_input_tf(TFModel *tf_model, TFRequestItem *request)
Definition: dnn_backend_tf.c:945
DnnLayerMaximumParams::val
union DnnLayerMaximumParams::@200 val
dnn_io_proc.h
TFModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_tf.c:60
TaskItem
Definition: dnn_backend_common.h:36
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:70
DnnLayerMaximumParams
Definition: dnn_backend_native_layer_maximum.h:33
add_pad_layer
static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
Definition: dnn_backend_tf.c:639
get_output_tf
static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_tf.c:316
avio_size
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:352
tf_sess_config.config
config
Definition: tf_sess_config.py:33
OFFSET
#define OFFSET(x)
Definition: dnn_backend_tf.c:83
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
dnn_backend_native_layer_pad.h
destroy_request_item
static void destroy_request_item(TFRequestItem **arg)
Free the TFRequestItem completely.
Definition: dnn_backend_tf.c:191
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
DLT_INPUT
@ DLT_INPUT
Definition: dnn_backend_native.h:41
ff_proc_from_dnn_to_frame
DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:27
ff_proc_from_frame_to_dnn
DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:100
TaskItem::model
void * model
Definition: dnn_backend_common.h:37
DNN_SUCCESS
@ DNN_SUCCESS
Definition: dnn_interface.h:33
DNNModel::get_output
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
get_input_tf
static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_tf.c:280
SIGMOID
@ SIGMOID
Definition: dnn_backend_native.h:55
SPACE_CHARS
#define SPACE_CHARS
Definition: dnn_backend_tf.c:359
Queue
Linear double-ended data structure.
Definition: queue.c:33
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
avassert.h
DNN_BACKEND_COMMON_OPTIONS
#define DNN_BACKEND_COMMON_OPTIONS
Definition: dnn_backend_common.h:31
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
TFRequestItem::exec_module
DNNAsyncExecModule exec_module
Definition: dnn_backend_tf.c:80
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:128
ConvolutionalParams::input_num
int32_t input_num
Definition: dnn_backend_native_layer_conv2d.h:28
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:51
DLT_MAXIMUM
@ DLT_MAXIMUM
Definition: dnn_backend_native.h:45
TANH
@ TANH
Definition: dnn_backend_native.h:55
read_graph
static TF_Buffer * read_graph(const char *model_filename)
Definition: dnn_backend_tf.c:225
add_maximum_layer
static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
Definition: dnn_backend_tf.c:697
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:58
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:42
op
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
Definition: anm.c:75
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
load_native_model
static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:746
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNReturnType
DNNReturnType
Definition: dnn_interface.h:33
DNNData
Definition: dnn_interface.h:59
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:172
ConvolutionalParams::activation
DNNActivationFunc activation
Definition: dnn_backend_native_layer_conv2d.h:29
dnn_tensorflow_options
static const AVOption dnn_tensorflow_options[]
Definition: dnn_backend_tf.c:85
LayerPadParams
Definition: dnn_backend_native_layer_pad.h:33
ctx
AVFormatContext * ctx
Definition: movenc.c:48
DNNModel::get_input
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:45
extract_lltask_from_task
static DNNReturnType extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_tf.c:205
arg
const char * arg
Definition: jacosubdec.c:67
dnn_backend_native_layer_depth2space.h
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:511
load_tf_model
static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:389
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
Layer::params
void * params
Definition: dnn_backend_native.h:66
NULL
#define NULL
Definition: coverity.c:32
add_conv_layer
static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
Definition: dnn_backend_tf.c:493
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
NativeModel
Definition: dnn_backend_native.h:124
av_opt_set_from_string
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1565
tf_create_inference_request
static TFInferRequest * tf_create_inference_request(void)
Create a TensorFlow inference request.
Definition: dnn_backend_tf.c:137
TFModel::task_queue
Queue * task_queue
Definition: dnn_backend_tf.c:62
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_tf.c:1031
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:38
TFModel::status
TF_Status * status
Definition: dnn_backend_tf.c:59
tf_free_request
static void tf_free_request(TFInferRequest *request)
Free the contents of TensorFlow inference request.
Definition: dnn_backend_tf.c:109
transpose_perm
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Definition: twinvq.c:620
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
ConvolutionalParams::kernel_size
int32_t kernel_size
Definition: dnn_backend_native_layer_conv2d.h:28
av_cpu_count
int av_cpu_count(void)
Definition: cpu.c:191
options
const OptionDef options[]
AVIOContext
Bytestream IO Context.
Definition: avio.h:161
DLT_MIRROR_PAD
@ DLT_MIRROR_PAD
Definition: dnn_backend_native.h:44
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:42
ff_dnn_fill_gettingoutput_task
DNNReturnType ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:161
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:46
cpu.h
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:106
ff_dnn_execute_model_tf
DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_tf.c:1130
execute_model_tf
static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
Definition: dnn_backend_tf.c:1089
size
int size
Definition: twinvq_data.h:10344
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:127
avio.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
TFOptions::nireq
uint32_t nireq
Definition: dnn_backend_tf.c:46
dnn_backend_native_layer_conv2d.h
DNNAsyncExecModule::start_inference
DNNReturnType(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:63
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
ff_dnn_flush_tf
DNNReturnType ff_dnn_flush_tf(const DNNModel *model)
Definition: dnn_backend_tf.c:1177
hex_to_data
static int hex_to_data(uint8_t *data, const char *p)
Definition: dnn_backend_tf.c:360
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
NAME_BUFFER_SIZE
#define NAME_BUFFER_SIZE
Definition: dnn_backend_tf.c:491
ff_frame_to_dnn_detect
DNNReturnType ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:252
DepthToSpaceParams
Definition: dnn_backend_native_layer_depth2space.h:33
dnn_backend_native.h
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_dnn_load_model_tf
DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_tf.c:850
RELU
@ RELU
Definition: dnn_backend_native.h:55
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
avio_closep
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1285
ConvolutionalParams::output_num
int32_t output_num
Definition: dnn_backend_native_layer_conv2d.h:28
TFContext::options
TFOptions options
Definition: dnn_backend_tf.c:51
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:271
LayerPadParams::paddings
int32_t paddings[4][2]
Definition: dnn_backend_native_layer_pad.h:34
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
TFRequestItem::status
TF_Status * status
Definition: dnn_backend_tf.c:79
TFInferRequest::output_tensors
TF_Tensor ** output_tensors
Definition: dnn_backend_tf.c:71
TFModel::session
TF_Session * session
Definition: dnn_backend_tf.c:58
TFRequestItem::infer_request
TFInferRequest * infer_request
Definition: dnn_backend_tf.c:77
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:76
av_toupper
static av_const int av_toupper(int c)
Locale-independent conversion of ASCII characters to uppercase.
Definition: avstring.h:236
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:263
TFOptions::async
uint8_t async
Definition: dnn_backend_tf.c:45
safe_queue.h
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:41
len
int len
Definition: vorbis_enc_data.h:426
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:271
TFInferRequest::tf_input
TF_Output * tf_input
Definition: dnn_backend_tf.c:72
TFContext
Definition: dnn_backend_tf.c:49
DepthToSpaceParams::block_size
int block_size
Definition: dnn_backend_native_layer_depth2space.h:34
ret
ret
Definition: filter_design.txt:187
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
TFModel::model
DNNModel * model
Definition: dnn_backend_tf.c:56
TFModel
Definition: dnn_backend_tf.c:54
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:39
AVFrame::height
int height
Definition: frame.h:389
allocate_input_tensor
static TF_Tensor * allocate_input_tensor(const DNNData *input)
Definition: dnn_backend_tf.c:258
add_depth_to_space_layer
static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
Definition: dnn_backend_tf.c:615
dnn_backend_common.h
TFRequestItem::lltask
LastLevelTaskItem * lltask
Definition: dnn_backend_tf.c:78
defs.h
avio_read
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:641
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:141
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
avio_open
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1220
AVFilterContext
An instance of a filter.
Definition: avfilter.h:402
ff_dnn_free_model_tf
void ff_dnn_free_model_tf(DNNModel **model)
Definition: dnn_backend_tf.c:1207
DNNModel
Definition: dnn_interface.h:84
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:621
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:40
DNNModel::options
const char * options
Definition: dnn_interface.h:88
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
TFOptions
Definition: dnn_backend_tf.c:43
free_buffer
static void free_buffer(void *data, size_t length)
Definition: dnn_backend_tf.c:97
int32_t
int32_t
Definition: audioconvert.c:56
DNNExecBaseParams
Definition: dnn_interface.h:67
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
ff_dnn_async_module_cleanup
DNNReturnType ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:92
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:43
DLT_DEPTH_TO_SPACE
@ DLT_DEPTH_TO_SPACE
Definition: dnn_backend_native.h:43
avstring.h
dnn_backend_native_layer_maximum.h
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:228
TFRequestItem
Definition: dnn_backend_tf.c:76
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
dnn_backend_tf.h
snprintf
#define snprintf
Definition: snprintf.h:34
CLEANUP_ON_ERROR
#define CLEANUP_ON_ERROR(tf_model)
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
TFModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_tf.c:61
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:44
ConvolutionalParams
Definition: dnn_backend_native_layer_conv2d.h:27
DNNModel::model
void * model
Definition: dnn_interface.h:86
ConvolutionalParams::biases
float * biases
Definition: dnn_backend_native_layer_conv2d.h:34
Generated on Wed Aug 24 2022 21:38:08 for FFmpeg by   doxygen 1.8.17

AltStyle によって変換されたページ (->オリジナル) /