dlib C++ Library - dnn_introduction3_ex.cpp

// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
/*
 This is an example illustrating the use of the deep learning tools from the
 dlib C++ Library. I'm assuming you have already read the dnn_introduction_ex.cpp and
 the dnn_introduction2_ex.cpp examples. So in this example program I'm going to go
 over a transfer learning example, which includes:
 - Defining a layer visitor to modify the some network parameters for fine-tuning
 - Using pretrained layers of a network for another task
*/
#include <dlib/dnn.h>
#include <iostream>
// This header file includes a generic definition of the most common ResNet architectures
#include "resnet.h"
using namespace std;
using namespace dlib;
// In this simple example we will show how to load a pretrained network and use it for a
// different task. In particular, we will load a ResNet50 trained on ImageNet, adjust
// some of its parameters and use it as a pretrained backbone for some metric learning
// task.
// Let's start by defining a network that will use the ResNet50 backbone from resnet.h
namespace model
{
 template<template<typename> class BN>
 using net_type = loss_metric<
 fc_no_bias<128,
 avg_pool_everything<
 typename resnet::def<BN>::template backbone_50<
 input_rgb_image
 >>>>;
 using train = net_type<bn_con>;
 using infer = net_type<affine>;
}
// Next, we define a layer visitor that will modify the weight decay of a network. The
// main interest of this class is to show how one can define custom visitors that modify
// some network parameters.
class visitor_weight_decay_multiplier
{
public:
 visitor_weight_decay_multiplier(double new_weight_decay_multiplier_) :
 new_weight_decay_multiplier(new_weight_decay_multiplier_) {}
 template <typename layer>
 void operator()(layer& l) const
 {
 set_weight_decay_multiplier(l, new_weight_decay_multiplier);
 }
private:
 double new_weight_decay_multiplier;
};
int main() try
{
 // Let's instantiate our network in train mode.
 model::train net;
 // We create a new scope so that resources from the loaded network are freed
 // automatically when leaving the scope.
 {
 // Now, let's define the classic ResNet50 network and load the pretrained model on
 // ImageNet.
 resnet::train_50 resnet50;
 std::vector<string> labels;
 deserialize("resnet50_1000_imagenet_classifier.dnn") >> resnet50 >> labels;
 // For transfer learning, we are only interested in the ResNet50's backbone, which
 // lays below the loss and the fc layers, so we can extract it as:
 auto backbone = std::move(resnet50.subnet().subnet());
 // We can now assign ResNet50's backbone to our network skipping the different
 // layers, in our case, the loss layer and the fc layer:
 net.subnet().subnet() = backbone;
 // An alternative way to use the pretrained network on a different
 // network is to extract the relevant part of the network (we remove
 // loss and fc layers), stack the new layers on top of it and assign
 // the network.
 using net_type = loss_metric<fc_no_bias<128, decltype(backbone)>>;
 net_type net2;
 net2.subnet().subnet() = backbone;
 }
 // We can use the visit_layers function to modify the weight decay of the entire
 // network:
 visit_computational_layers(net, visitor_weight_decay_multiplier(0.001));
 // We can also use predefined visitors to affect the learning rate of the whole
 // network.
 set_all_learning_rate_multipliers(net, 0.5);
 // Modifying the learning rates of a network is a common practice for fine tuning, for
 // this reason it is already provided. However, it is implemented internally using a
 // visitor that is very similar to the one defined in this example.
 // Usually, we want to freeze the network, except for the top layers:
 visit_computational_layers(net.subnet().subnet(), visitor_weight_decay_multiplier(0));
 set_all_learning_rate_multipliers(net.subnet().subnet(), 0);
 // Alternatively, we can use the visit_layers_range to modify only a specific set of
 // layers:
 visit_computational_layers_range<0, 2>(net, visitor_weight_decay_multiplier(1));
 // Sometimes we might want to set the learning rate differently throughout the network.
 // Here we show how to use adjust the learning rate at the different ResNet50's
 // convolutional blocks:
 set_learning_rate_multipliers_range< 0, 2>(net, 1);
 set_learning_rate_multipliers_range< 2, 38>(net, 0.1);
 set_learning_rate_multipliers_range< 38, 107>(net, 0.01);
 set_learning_rate_multipliers_range<107, 154>(net, 0.001);
 set_learning_rate_multipliers_range<154, 193>(net, 0.0001);
 // Finally, we can check the results by printing the network. But before, if we
 // forward an image through the network, we will see tensors shape at every layer.
 matrix<rgb_pixel> image(224, 224);
 assign_all_pixels(image, rgb_pixel(0, 0, 0));
 std::vector<matrix<rgb_pixel>> minibatch(1, image);
 resizable_tensor input;
 net.to_tensor(minibatch.begin(), minibatch.end(), input);
 net.forward(input);
 cout << net << endl;
 cout << "input size=(" <<
 "num:" << input.num_samples() << ", " <<
 "k:" << input.k() << ", " <<
 "nr:" << input.nr() << ", "
 "nc:" << input.nc() << ")" << endl;
 // We can also print the number of parameters of the network:
 cout << "number of network parameters: " << count_parameters(net) << endl;
 // From this point on, we can fine-tune the new network using this pretrained backbone
 // on another task, such as the one showed in dnn_metric_learning_on_images_ex.cpp.
 return EXIT_SUCCESS;
}
catch (const serialization_error& e)
{
 cout << e.what() << endl;
 cout << "You need to download a copy of the file resnet50_1000_imagenet_classifier.dnn" << endl;
 cout << "available at http://dlib.net/files/resnet50_1000_imagenet_classifier.dnn.bz2" << endl;
 cout << endl;
 return EXIT_FAILURE;
}
catch (const exception& e)
{
 cout << e.what() << endl;
 return EXIT_FAILURE;
}