dlib C++ Library - dnn_semantic_segmentation_train_ex.cpp

// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
/*
 This example shows how to train a semantic segmentation net using the PASCAL VOC2012
 dataset. For an introduction to what segmentation is, see the accompanying header file
 dnn_semantic_segmentation_ex.h.
 Instructions how to run the example:
 1. Download the PASCAL VOC2012 data, and untar it somewhere.
 http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar
 2. Build the dnn_semantic_segmentation_train_ex example program.
 3. Run:
 ./dnn_semantic_segmentation_train_ex /path/to/VOC2012
 4. Wait while the network is being trained.
 5. Build the dnn_semantic_segmentation_ex example program.
 6. Run:
 ./dnn_semantic_segmentation_ex /path/to/VOC2012-or-other-images
 It would be a good idea to become familiar with dlib's DNN tooling before reading this
 example. So you should read dnn_introduction_ex.cpp and dnn_introduction2_ex.cpp
 before reading this example program.
*/
#include "dnn_semantic_segmentation_ex.h"
#include <iostream>
#include <dlib/data_io.h>
#include <dlib/image_transforms.h>
#include <dlib/dir_nav.h>
#include <iterator>
#include <thread>
using namespace std;
using namespace dlib;
// A single training sample. A mini-batch comprises many of these.
struct training_sample
{
 matrix<rgb_pixel> input_image;
 matrix<uint16_t> label_image; // The ground-truth label of each pixel.
};
// ----------------------------------------------------------------------------------------
rectangle make_random_cropping_rect(
 const matrix<rgb_pixel>& img,
 dlib::rand& rnd
)
{
 // figure out what rectangle we want to crop from the image
 double mins = 0.466666666, maxs = 0.875;
 auto scale = mins + rnd.get_random_double()*(maxs-mins);
 auto size = scale*std::min(img.nr(), img.nc());
 rectangle rect(size, size);
 // randomly shift the box around
 point offset(rnd.get_random_32bit_number()%(img.nc()-rect.width()),
 rnd.get_random_32bit_number()%(img.nr()-rect.height()));
 return move_rect(rect, offset);
}
// ----------------------------------------------------------------------------------------
void randomly_crop_image (
 const matrix<rgb_pixel>& input_image,
 const matrix<uint16_t>& label_image,
 training_sample& crop,
 dlib::rand& rnd
)
{
 const auto rect = make_random_cropping_rect(input_image, rnd);
 const chip_details chip_details(rect, chip_dims(227, 227));
 // Crop the input image.
 extract_image_chip(input_image, chip_details, crop.input_image, interpolate_bilinear());
 // Crop the labels correspondingly. However, note that here bilinear
 // interpolation would make absolutely no sense - you wouldn't say that
 // a bicycle is half-way between an aeroplane and a bird, would you?
 extract_image_chip(label_image, chip_details, crop.label_image, interpolate_nearest_neighbor());
 // Also randomly flip the input image and the labels.
 if (rnd.get_random_double() > 0.5)
 {
 crop.input_image = fliplr(crop.input_image);
 crop.label_image = fliplr(crop.label_image);
 }
 // And then randomly adjust the colors.
 apply_random_color_offset(crop.input_image, rnd);
}
// ----------------------------------------------------------------------------------------
// Calculate the per-pixel accuracy on a dataset whose file names are supplied as a parameter.
double calculate_accuracy(anet_type& anet, const std::vector<image_info>& dataset)
{
 int num_right = 0;
 int num_wrong = 0;
 matrix<rgb_pixel> input_image;
 matrix<rgb_pixel> rgb_label_image;
 matrix<uint16_t> index_label_image;
 matrix<uint16_t> net_output;
 for (const auto& image_info : dataset)
 {
 // Load the input image.
 load_image(input_image, image_info.image_filename);
 // Load the ground-truth (RGB) labels.
 load_image(rgb_label_image, image_info.class_label_filename);
 // Create predictions for each pixel. At this point, the type of each prediction
 // is an index (a value between 0 and 20). Note that the net may return an image
 // that is not exactly the same size as the input.
 const matrix<uint16_t> temp = anet(input_image);
 // Convert the RGB values to indexes.
 rgb_label_image_to_index_label_image(rgb_label_image, index_label_image);
 // Crop the net output to be exactly the same size as the input.
 const chip_details chip_details(
 centered_rect(temp.nc() / 2, temp.nr() / 2, input_image.nc(), input_image.nr()),
 chip_dims(input_image.nr(), input_image.nc())
 );
 extract_image_chip(temp, chip_details, net_output, interpolate_nearest_neighbor());
 const long nr = index_label_image.nr();
 const long nc = index_label_image.nc();
 // Compare the predicted values to the ground-truth values.
 for (long r = 0; r < nr; ++r)
 {
 for (long c = 0; c < nc; ++c)
 {
 const uint16_t truth = index_label_image(r, c);
 if (truth != dlib::loss_multiclass_log_per_pixel_::label_to_ignore)
 {
 const uint16_t prediction = net_output(r, c);
 if (prediction == truth)
 {
 ++num_right;
 }
 else
 {
 ++num_wrong;
 }
 }
 }
 }
 }
 // Return the accuracy estimate.
 return num_right / static_cast<double>(num_right + num_wrong);
}
// ----------------------------------------------------------------------------------------
int main(int argc, char** argv) try
{
 if (argc < 2 || argc > 3)
 {
 cout << "To run this program you need a copy of the PASCAL VOC2012 dataset." << endl;
 cout << endl;
 cout << "You call this program like this: " << endl;
 cout << "./dnn_semantic_segmentation_train_ex /path/to/VOC2012 [minibatch-size]" << endl;
 return 1;
 }
 cout << "\nSCANNING PASCAL VOC2012 DATASET\n" << endl;
 const auto listing = get_pascal_voc2012_train_listing(argv[1]);
 cout << "images in dataset: " << listing.size() << endl;
 if (listing.size() == 0)
 {
 cout << "Didn't find the VOC2012 dataset. " << endl;
 return 1;
 }
 // a mini-batch smaller than the default can be used with GPUs having less memory
 const unsigned int minibatch_size = argc == 3 ? std::stoi(argv[2]) : 23;
 cout << "mini-batch size: " << minibatch_size << endl;
 const double initial_learning_rate = 0.1;
 const double weight_decay = 0.0001;
 const double momentum = 0.9;
 bnet_type bnet;
 dnn_trainer<bnet_type> trainer(bnet,sgd(weight_decay, momentum));
 trainer.be_verbose();
 trainer.set_learning_rate(initial_learning_rate);
 trainer.set_synchronization_file("pascal_voc2012_trainer_state_file.dat", std::chrono::minutes(10));
 // This threshold is probably excessively large.
 trainer.set_iterations_without_progress_threshold(5000);
 // Since the progress threshold is so large might as well set the batch normalization
 // stats window to something big too.
 set_all_bn_running_stats_window_sizes(bnet, 1000);
 // Output training parameters.
 cout << endl << trainer << endl;
 std::vector<matrix<rgb_pixel>> samples;
 std::vector<matrix<uint16_t>> labels;
 // Start a bunch of threads that read images from disk and pull out random crops. It's
 // important to be sure to feed the GPU fast enough to keep it busy. Using multiple
 // thread for this kind of data preparation helps us do that. Each thread puts the
 // crops into the data queue.
 dlib::pipe<training_sample> data(200);
 auto f = [&data, &listing](time_t seed)
 {
 dlib::rand rnd(time(0)+seed);
 matrix<rgb_pixel> input_image;
 matrix<rgb_pixel> rgb_label_image;
 matrix<uint16_t> index_label_image;
 training_sample temp;
 while(data.is_enabled())
 {
 // Pick a random input image.
 const image_info& image_info = listing[rnd.get_random_32bit_number()%listing.size()];
 // Load the input image.
 load_image(input_image, image_info.image_filename);
 // Load the ground-truth (RGB) labels.
 load_image(rgb_label_image, image_info.class_label_filename);
 // Convert the RGB values to indexes.
 rgb_label_image_to_index_label_image(rgb_label_image, index_label_image);
 // Randomly pick a part of the image.
 randomly_crop_image(input_image, index_label_image, temp, rnd);
 // Push the result to be used by the trainer.
 data.enqueue(temp);
 }
 };
 std::thread data_loader1([f](){ f(1); });
 std::thread data_loader2([f](){ f(2); });
 std::thread data_loader3([f](){ f(3); });
 std::thread data_loader4([f](){ f(4); });
 // The main training loop. Keep making mini-batches and giving them to the trainer.
 // We will run until the learning rate has dropped by a factor of 1e-4.
 while(trainer.get_learning_rate() >= 1e-4)
 {
 samples.clear();
 labels.clear();
 // make a mini-batch
 training_sample temp;
 while(samples.size() < minibatch_size)
 {
 data.dequeue(temp);
 samples.push_back(std::move(temp.input_image));
 labels.push_back(std::move(temp.label_image));
 }
 trainer.train_one_step(samples, labels);
 }
 // Training done, tell threads to stop and make sure to wait for them to finish before
 // moving on.
 data.disable();
 data_loader1.join();
 data_loader2.join();
 data_loader3.join();
 data_loader4.join();
 // also wait for threaded processing to stop in the trainer.
 trainer.get_net();
 bnet.clean();
 cout << "saving network" << endl;
 serialize(semantic_segmentation_net_filename) << bnet;
 // Make a copy of the network to use it for inference.
 anet_type anet = bnet;
 cout << "Testing the network..." << endl;
 // Find the accuracy of the newly trained network on both the training and the validation sets.
 cout << "train accuracy : " << calculate_accuracy(anet, get_pascal_voc2012_train_listing(argv[1])) << endl;
 cout << "val accuracy : " << calculate_accuracy(anet, get_pascal_voc2012_val_listing(argv[1])) << endl;
}
catch(std::exception& e)
{
 cout << e.what() << endl;
}