343 lines
10 KiB
C
Executable File
343 lines
10 KiB
C
Executable File
/*
|
|
* Copyright (c) 2018 Sergey Lavrushkin
|
|
*
|
|
* This file is part of FFmpeg.
|
|
*
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
/**
|
|
* @file
|
|
* DNN native backend implementation.
|
|
*/
|
|
|
|
#include "dnn_backend_native.h"
|
|
#include "libavutil/avassert.h"
|
|
#include "dnn_backend_native_layer_conv2d.h"
|
|
#include "dnn_backend_native_layers.h"
|
|
|
|
static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
|
|
{
|
|
ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
|
|
|
|
for (int i = 0; i < network->operands_num; ++i) {
|
|
DnnOperand *oprd = &network->operands[i];
|
|
if (strcmp(oprd->name, input_name) == 0) {
|
|
if (oprd->type != DOT_INPUT)
|
|
return DNN_ERROR;
|
|
input->dt = oprd->data_type;
|
|
av_assert0(oprd->dims[0] == 1);
|
|
input->height = oprd->dims[1];
|
|
input->width = oprd->dims[2];
|
|
input->channels = oprd->dims[3];
|
|
return DNN_SUCCESS;
|
|
}
|
|
}
|
|
|
|
// do not find the input operand
|
|
return DNN_ERROR;
|
|
}
|
|
|
|
static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
|
|
{
|
|
ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
|
|
DnnOperand *oprd = NULL;
|
|
|
|
if (network->layers_num <= 0 || network->operands_num <= 0)
|
|
return DNN_ERROR;
|
|
|
|
/* inputs */
|
|
for (int i = 0; i < network->operands_num; ++i) {
|
|
oprd = &network->operands[i];
|
|
if (strcmp(oprd->name, input_name) == 0) {
|
|
if (oprd->type != DOT_INPUT)
|
|
return DNN_ERROR;
|
|
break;
|
|
}
|
|
oprd = NULL;
|
|
}
|
|
|
|
if (!oprd)
|
|
return DNN_ERROR;
|
|
|
|
oprd->dims[0] = 1;
|
|
oprd->dims[1] = input->height;
|
|
oprd->dims[2] = input->width;
|
|
oprd->dims[3] = input->channels;
|
|
|
|
av_freep(&oprd->data);
|
|
oprd->length = calculate_operand_data_length(oprd);
|
|
if (oprd->length <= 0)
|
|
return DNN_ERROR;
|
|
oprd->data = av_malloc(oprd->length);
|
|
if (!oprd->data)
|
|
return DNN_ERROR;
|
|
|
|
input->data = oprd->data;
|
|
|
|
/* outputs */
|
|
network->nb_output = 0;
|
|
av_freep(&network->output_indexes);
|
|
network->output_indexes = av_mallocz_array(nb_output, sizeof(*network->output_indexes));
|
|
if (!network->output_indexes)
|
|
return DNN_ERROR;
|
|
|
|
for (uint32_t i = 0; i < nb_output; ++i) {
|
|
const char *output_name = output_names[i];
|
|
for (int j = 0; j < network->operands_num; ++j) {
|
|
oprd = &network->operands[j];
|
|
if (strcmp(oprd->name, output_name) == 0) {
|
|
network->output_indexes[network->nb_output++] = j;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (network->nb_output != nb_output)
|
|
return DNN_ERROR;
|
|
|
|
return DNN_SUCCESS;
|
|
}
|
|
|
|
// Loads model and its parameters that are stored in a binary file with following structure:
|
|
// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
|
|
// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
|
|
// For DEPTH_TO_SPACE layer: block_size
|
|
DNNModel *ff_dnn_load_model_native(const char *model_filename)
|
|
{
|
|
DNNModel *model = NULL;
|
|
char header_expected[] = "FFMPEGDNNNATIVE";
|
|
char *buf;
|
|
size_t size;
|
|
int version, header_size, major_version_expected = 1;
|
|
ConvolutionalNetwork *network = NULL;
|
|
AVIOContext *model_file_context;
|
|
int file_size, dnn_size, parsed_size;
|
|
int32_t layer;
|
|
DNNLayerType layer_type;
|
|
|
|
if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
|
|
return NULL;
|
|
}
|
|
file_size = avio_size(model_file_context);
|
|
|
|
model = av_mallocz(sizeof(DNNModel));
|
|
if (!model){
|
|
goto fail;
|
|
}
|
|
|
|
/**
|
|
* check file header with string and version
|
|
*/
|
|
size = sizeof(header_expected);
|
|
buf = av_malloc(size);
|
|
if (!buf) {
|
|
goto fail;
|
|
}
|
|
|
|
// size - 1 to skip the ending '\0' which is not saved in file
|
|
avio_get_str(model_file_context, size - 1, buf, size);
|
|
dnn_size = size - 1;
|
|
if (strncmp(buf, header_expected, size) != 0) {
|
|
av_freep(&buf);
|
|
goto fail;
|
|
}
|
|
av_freep(&buf);
|
|
|
|
version = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
if (version != major_version_expected) {
|
|
goto fail;
|
|
}
|
|
|
|
// currently no need to check minor version
|
|
version = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
header_size = dnn_size;
|
|
|
|
network = av_mallocz(sizeof(ConvolutionalNetwork));
|
|
if (!network){
|
|
goto fail;
|
|
}
|
|
model->model = (void *)network;
|
|
|
|
avio_seek(model_file_context, file_size - 8, SEEK_SET);
|
|
network->layers_num = (int32_t)avio_rl32(model_file_context);
|
|
network->operands_num = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 8;
|
|
avio_seek(model_file_context, header_size, SEEK_SET);
|
|
|
|
network->layers = av_mallocz(network->layers_num * sizeof(Layer));
|
|
if (!network->layers){
|
|
goto fail;
|
|
}
|
|
|
|
network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
|
|
if (!network->operands){
|
|
goto fail;
|
|
}
|
|
|
|
for (layer = 0; layer < network->layers_num; ++layer){
|
|
layer_type = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
|
|
if (layer_type >= DLT_COUNT) {
|
|
goto fail;
|
|
}
|
|
|
|
network->layers[layer].type = layer_type;
|
|
parsed_size = layer_funcs[layer_type].pf_load(&network->layers[layer], model_file_context, file_size, network->operands_num);
|
|
if (!parsed_size) {
|
|
goto fail;
|
|
}
|
|
dnn_size += parsed_size;
|
|
}
|
|
|
|
for (int32_t i = 0; i < network->operands_num; ++i){
|
|
DnnOperand *oprd;
|
|
int32_t name_len;
|
|
int32_t operand_index = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
|
|
if (operand_index >= network->operands_num) {
|
|
goto fail;
|
|
}
|
|
|
|
oprd = &network->operands[operand_index];
|
|
name_len = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
|
|
avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
|
|
dnn_size += name_len;
|
|
|
|
oprd->type = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
|
|
oprd->data_type = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
|
|
for (int32_t dim = 0; dim < 4; ++dim) {
|
|
oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
|
|
dnn_size += 4;
|
|
}
|
|
|
|
oprd->isNHWC = 1;
|
|
}
|
|
|
|
avio_closep(&model_file_context);
|
|
|
|
if (dnn_size != file_size){
|
|
ff_dnn_free_model_native(&model);
|
|
return NULL;
|
|
}
|
|
|
|
model->set_input_output = &set_input_output_native;
|
|
model->get_input = &get_input_native;
|
|
|
|
return model;
|
|
|
|
fail:
|
|
ff_dnn_free_model_native(&model);
|
|
avio_closep(&model_file_context);
|
|
return NULL;
|
|
}
|
|
|
|
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
|
|
{
|
|
ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
|
|
int32_t layer;
|
|
uint32_t nb = FFMIN(nb_output, network->nb_output);
|
|
|
|
if (network->layers_num <= 0 || network->operands_num <= 0)
|
|
return DNN_ERROR;
|
|
if (!network->operands[0].data)
|
|
return DNN_ERROR;
|
|
|
|
for (layer = 0; layer < network->layers_num; ++layer){
|
|
DNNLayerType layer_type = network->layers[layer].type;
|
|
layer_funcs[layer_type].pf_exec(network->operands,
|
|
network->layers[layer].input_operand_indexes,
|
|
network->layers[layer].output_operand_index,
|
|
network->layers[layer].params);
|
|
}
|
|
|
|
for (uint32_t i = 0; i < nb; ++i) {
|
|
DnnOperand *oprd = &network->operands[network->output_indexes[i]];
|
|
outputs[i].data = oprd->data;
|
|
outputs[i].height = oprd->dims[1];
|
|
outputs[i].width = oprd->dims[2];
|
|
outputs[i].channels = oprd->dims[3];
|
|
outputs[i].dt = oprd->data_type;
|
|
}
|
|
|
|
return DNN_SUCCESS;
|
|
}
|
|
|
|
int32_t calculate_operand_dims_count(const DnnOperand *oprd)
|
|
{
|
|
int32_t result = 1;
|
|
for (int i = 0; i < 4; ++i)
|
|
result *= oprd->dims[i];
|
|
|
|
return result;
|
|
}
|
|
|
|
int32_t calculate_operand_data_length(const DnnOperand* oprd)
|
|
{
|
|
// currently, we just support DNN_FLOAT
|
|
uint64_t len = sizeof(float);
|
|
for (int i = 0; i < 4; i++) {
|
|
len *= oprd->dims[i];
|
|
if (len > INT32_MAX)
|
|
return 0;
|
|
}
|
|
return len;
|
|
}
|
|
|
|
void ff_dnn_free_model_native(DNNModel **model)
|
|
{
|
|
ConvolutionalNetwork *network;
|
|
ConvolutionalParams *conv_params;
|
|
int32_t layer;
|
|
|
|
if (*model)
|
|
{
|
|
if ((*model)->model) {
|
|
network = (ConvolutionalNetwork *)(*model)->model;
|
|
if (network->layers) {
|
|
for (layer = 0; layer < network->layers_num; ++layer){
|
|
if (network->layers[layer].type == DLT_CONV2D){
|
|
conv_params = (ConvolutionalParams *)network->layers[layer].params;
|
|
av_freep(&conv_params->kernel);
|
|
av_freep(&conv_params->biases);
|
|
}
|
|
av_freep(&network->layers[layer].params);
|
|
}
|
|
av_freep(&network->layers);
|
|
}
|
|
|
|
if (network->operands) {
|
|
for (uint32_t operand = 0; operand < network->operands_num; ++operand)
|
|
av_freep(&network->operands[operand].data);
|
|
av_freep(&network->operands);
|
|
}
|
|
|
|
av_freep(&network->output_indexes);
|
|
av_freep(&network);
|
|
}
|
|
av_freep(model);
|
|
}
|
|
}
|