// Copyright (C) 2015 Davis E. King (davis@dlib.net) // License: Boost Software License See LICENSE.txt for the full license. #undef DLIB_DNn_INPUT_ABSTRACT_H_ #ifdef DLIB_DNn_INPUT_ABSTRACT_H_ #include "../matrix.h" #include "../pixel.h" namespace dlib{// ---------------------------------------------------------------------------------------- classEXAMPLE_INPUT_LAYER{/*! WHAT THIS OBJECT REPRESENTS Each deep neural network model in dlib begins with an input layer. The job of the input layer is to convert an input_type into a tensor. Nothing more and nothing less. Note that there is no dlib::EXAMPLE_INPUT_LAYER type. It is shown here purely to document the interface that an input layer object must implement. If you are using some kind of image or matrix object as your input_type then you can use the provided dlib::input layer defined below. Otherwise, you need to define your own custom input layer. THREAD SAFETY to_tensor() must be thread safe. That is, multiple threads must be able to make calls to to_tensor() on a single instance of this object at the same time. !*/ public:EXAMPLE_INPUT_LAYER( ); /*! ensures - Default constructs this object. This function is not required to do anything in particular but it must exist, that is, it is required that layer objects be default constructable. !*/EXAMPLE_INPUT_LAYER( const EXAMPLE_INPUT_LAYER& item ); /*! ensures - EXAMPLE_INPUT_LAYER objects are copy constructable !*/EXAMPLE_INPUT_LAYER( const some_other_input_layer_type& item ); /*! ensures - Constructs this object from item. This form of constructor is optional but it allows you to provide a conversion from one input layer type to another. For example, the following code is valid only if my_input_layer2 can be constructed from my_input_layer1: relu<fc<relu<fc<my_input_layer1>>>> my_dnn1; relu<fc<relu<fc<my_input_layer2>>>> my_dnn2(my_dnn1); This kind of pattern is useful if you want to use one type of input layer during training but a different type of layer during testing since it allows you to easily convert between related deep neural network types. !*/ typedef whatever_type_to_tensor_expects input_type; template <typename forward_iterator>voidto_tensor( forward_iterator ibegin, forward_iterator iend, resizable_tensor& data ) const; /*! requires - [ibegin, iend) is an iterator range over input_type objects. - std::distance(ibegin,iend) > 0 ensures - Converts the iterator range into a tensor and stores it into #data. - #data.num_samples()%distance(ibegin,iend) == 0. Normally you would have #data.num_samples() == distance(ibegin,iend) but you can also expand the output by some integer factor so long as the loss you use can deal with it correctly. - The data in the ith sample of #data corresponds to the input_type object *(ibegin+i/sample_expansion_factor). where sample_expansion_factor==#data.num_samples()/distance(ibegin,iend). !*/}; std::ostream&operator<<(std::ostream& out, const EXAMPLE_INPUT_LAYER& item); /*! print a string describing this layer. !*/voidto_xml(const EXAMPLE_INPUT_LAYER& item, std::ostream& out); /*! This function is optional, but required if you want to print your networks with net_to_xml(). Therefore, to_xml() prints a layer as XML. !*/voidserialize(const EXAMPLE_INPUT_LAYER& item, std::ostream& out);voiddeserialize(EXAMPLE_INPUT_LAYER& item, std::istream& in); /*! provides serialization support !*/ // ---------------------------------------------------------------------------------------- template < typename T > classinput{/*! REQUIREMENTS ON T One of the following must be true: - T is a matrix or array2d object and it must contain some kind of pixel type. I.e. pixel_traits<T::type> must be defined. - T is a std::array<matrix<U>> where U is any built in scalar type like float, double, or unsigned char. WHAT THIS OBJECT REPRESENTS This is a basic input layer that simply copies images into a tensor. !*/ public: typedef T input_type; template <typename forward_iterator>voidto_tensor( forward_iterator ibegin, forward_iterator iend, resizable_tensor& data ) const; /*! requires - [ibegin, iend) is an iterator range over input_type objects. - std::distance(ibegin,iend) > 0 - The input range should contain image objects that all have the same dimensions. ensures - Converts the iterator range into a tensor and stores it into #data. In particular, if the input images have R rows, C columns, and K channels (where K is given by pixel_traits::num or std::array::size() if std::array inputs are used) then we will have: - #data.num_samples() == std::distance(ibegin,iend) - #data.nr() == R - #data.nc() == C - #data.k() == K For example, a matrix<float,3,3> would turn into a tensor with 3 rows, 3 columns, and k()==1. Or a matrix<rgb_pixel,4,5> would turn into a tensor with 4 rows, 5 columns, and k()==3 (since rgb_pixels have 3 channels). Or a std::array<matrix<float,3,3>,5> would turn into a tensor with 3 rows and columns, and k()==5 channels. - If the input data contains pixels of type unsigned char, rgb_pixel, or other pixel types with a basic_pixel_type of unsigned char then each value written to the output tensor is first divided by 256.0 so that the resulting outputs are all in the range [0,1]. !*/ // Provided for compatibility with input_rgb_image_pyramid's interfaceboolimage_contained_point( const tensor& data, const point& p) const{return get_rect(data).contains(p);}drectangletensor_space_to_image_space( const tensor& /*data*/, drectangle r) const{return r;}drectangleimage_space_to_tensor_space( const tensor& /*data*/,double/*scale*/, drectangle r ) const{return r;}}; // ---------------------------------------------------------------------------------------- classinput_rgb_image{/*! WHAT THIS OBJECT REPRESENTS This input layer works with RGB images of type matrix<rgb_pixel>. It is very similar to the dlib::input layer except that it allows you to subtract the average color value from each color channel when converting an image to a tensor. !*/ public: typedef matrix<rgb_pixel> input_type;input_rgb_image( ); /*! ensures - #get_avg_red() == 122.782 - #get_avg_green() == 117.001 - #get_avg_blue() == 104.298 !*/input_rgb_image(floatavg_red,floatavg_green,floatavg_blue ); /*! ensures - #get_avg_red() == avg_red - #get_avg_green() == avg_green - #get_avg_blue() == avg_blue !*/floatget_avg_red( ) const; /*! ensures - returns the value subtracted from the red color channel. !*/floatget_avg_green( ) const; /*! ensures - returns the value subtracted from the green color channel. !*/floatget_avg_blue( ) const; /*! ensures - returns the value subtracted from the blue color channel. !*/ template <typename forward_iterator>voidto_tensor( forward_iterator ibegin, forward_iterator iend, resizable_tensor& data ) const; /*! requires - [ibegin, iend) is an iterator range over input_type objects. - std::distance(ibegin,iend) > 0 - The input range should contain images that all have the same dimensions. ensures - Converts the iterator range into a tensor and stores it into #data. In particular, if the input images have R rows, C columns then we will have: - #data.num_samples() == std::distance(ibegin,iend) - #data.nr() == R - #data.nc() == C - #data.k() == 3 Moreover, each color channel is normalized by having its average value subtracted (according to get_avg_red(), get_avg_green(), or get_avg_blue()) and then is divided by 256.0. !*/ // Provided for compatibility with input_rgb_image_pyramid's interfaceboolimage_contained_point( const tensor& data, const point& p) const{return get_rect(data).contains(p);}drectangletensor_space_to_image_space( const tensor& /*data*/, drectangle r) const{return r;}drectangleimage_space_to_tensor_space( const tensor& /*data*/,double/*scale*/, drectangle r ) const{return r;}}; // ---------------------------------------------------------------------------------------- template <size_tNR,size_tNC=NR> classinput_rgb_image_sized{/*! WHAT THIS OBJECT REPRESENTS This layer has an interface and behavior identical to input_rgb_image except that it requires input images to have NR rows and NC columns. This is checked by a DLIB_CASSERT inside to_tensor(). You can also convert between input_rgb_image and input_rgb_image_sized by copy construction or assignment. !*/}; // ---------------------------------------------------------------------------------------- classinput_rgb_image_pair{/*! WHAT THIS OBJECT REPRESENTS This input layer works with std::pair of RGB images of type matrix<rgb_pixel>. It is useful when you want to input image pairs that are related to each other, for instance, they are different distorted views of the same original image. It is mainly supposed to be used with unsupervised loss functions such as loss_barlow_twins_. You can also convert between input_rgb_image and input_rgb_image_pair by copy construction or assignment. !*/ public: typedef std::pair<matrix<rgb_pixel>, matrix<rgb_pixel>> input_type;input_rgb_image_pair( ); /*! ensures - #get_avg_red() == 122.782 - #get_avg_green() == 117.001 - #get_avg_blue() == 104.298 !*/input_rgb_image_pair(floatavg_red,floatavg_green,floatavg_blue ); /*! ensures - #get_avg_red() == avg_red - #get_avg_green() == avg_green - #get_avg_blue() == avg_blue !*/ inlineinput_rgb_image_pair( const input_rgb_image& item ); /*! ensures - #get_avg_red() == item.get_avg_red() - #get_avg_green() == item.get_avg_green() - #get_avg_blue() == item.get_avg_blue() !*/ template <size_tNR,size_tNC> inlineinput_rgb_image_pair( const input_rgb_image_sized<NR, NC>& item ); /*! ensures - #get_avg_red() == item.get_avg_red() - #get_avg_green() == item.get_avg_green() - #get_avg_blue() == item.get_avg_blue() !*/floatget_avg_red( ) const; /*! ensures - returns the value subtracted from the red color channel. !*/floatget_avg_green( ) const; /*! ensures - returns the value subtracted from the green color channel. !*/floatget_avg_blue( ) const; /*! ensures - returns the value subtracted from the blue color channel. !*/voidto_tensor( forward_iterator ibegin, forward_iterator iend, resizable_tensor& data ) const; /*! requires - [ibegin, iend) is an iterator range over input_type objects. - std::distance(ibegin,iend) > 0 - The input range should contain images that all have the same dimensions. ensures - Converts the iterator range into a tensor and stores it into #data. In particular, if the input images have R rows, C columns then we will have: - #data.num_samples() == 2 * std::distance(ibegin,iend) - #data.nr() == R - #data.nc() == C - #data.k() == 3 Moreover, each color channel is normalized by having its average value subtracted (according to get_avg_red(), get_avg_green(), or get_avg_blue()) and then is divided by 256.0. Additionally, the first elements in each pair are placed in the first half of the batch, and the second elements in the second half. !*/ // Provided for compatibility with input_rgb_image_pyramid's interfaceboolimage_contained_point( const tensor& data, const point& p) const{return get_rect(data).contains(p);}drectangletensor_space_to_image_space( const tensor& /*data*/, drectangle r) const{return r;}drectangleimage_space_to_tensor_space( const tensor& /*data*/,double/*scale*/, drectangle r ) const{return r;}// ---------------------------------------------------------------------------------------- template < typename PYRAMID_TYPE > classinput_grayscale_image_pyramid{/*! REQUIREMENTS ON PYRAMID_TYPE PYRAMID_TYPE must be an instance of the dlib::pyramid_down template. WHAT THIS OBJECT REPRESENTS This input layer works with gray scale images of type matrix<unsigned char>. It is identical to input layer except that it outputs a tensor containing a tiled image pyramid of each input image rather than a simple copy of each image. The tiled image pyramid is created using create_tiled_pyramid(). !*/ public: typedef matrix<unsignedchar> input_type; typedef PYRAMID_TYPE pyramid_type;input_grayscale_image_pyramid( ); /*! ensures - #get_pyramid_padding() == 10 - #get_pyramid_outer_padding() == 11 !*/unsignedlongget_pyramid_padding( ) const; /*! ensures - When this object creates a pyramid it will call create_tiled_pyramid() and set create_tiled_pyramid's pyramid_padding parameter to get_pyramid_padding(). !*/voidset_pyramid_padding(unsignedlongvalue ); /*! ensures - #get_pyramid_padding() == value !*/unsignedlongget_pyramid_outer_padding( ) const; /*! ensures - When this object creates a pyramid it will call create_tiled_pyramid() and set create_tiled_pyramid's pyramid_outer_padding parameter to get_pyramid_outer_padding(). !*/voidset_pyramid_outer_padding(unsignedlongvalue ); /*! ensures - #get_pyramid_outer_padding() == value !*/ template <typename forward_iterator>voidto_tensor( forward_iterator ibegin, forward_iterator iend, resizable_tensor& data ) const; /*! requires - [ibegin, iend) is an iterator range over input_type objects. - std::distance(ibegin,iend) > 0 - The input range should contain images that all have the same dimensions. ensures - Converts the iterator range into a tensor and stores it into #data. In particular, we will have: - #data.num_samples() == std::distance(ibegin,iend) - #data.k() == 1 - Each sample in #data contains a tiled image pyramid of the corresponding input image. The tiled pyramid is created by create_tiled_pyramid(). Moreover, each pixel is normalized, dividing them by 256.0. !*/boolimage_contained_point( const tensor& data, const point& p ) const; /*! requires - data is a tensor that was produced by this->to_tensor() ensures - Since data is a tensor that is built from a bunch of identically sized images, we can ask if those images were big enough to contain the point p. This function returns the answer to that question. !*/ drectangleimage_space_to_tensor_space( const tensor& data,doublescale, drectangle r ) const; /*! requires - data is a tensor that was produced by this->to_tensor() - 0 < scale <= 1 ensures - This function maps from to_tensor()'s input image space to its output tensor space. Therefore, given that data is a tensor produced by to_tensor(), image_space_to_tensor_space() allows you to ask for the rectangle in data that corresponds to a rectangle in the original image space. Note that since the output tensor contains an image pyramid, there are multiple points in the output tensor that correspond to any input location. So you must also specify a scale so we know what level of the pyramid is needed. So given a rectangle r in an input image, you can ask, what rectangle in data corresponds to r when things are scale times smaller? That rectangle is returned by this function. - A scale of 1 means we don't move anywhere in the pyramid scale space relative to the input image while smaller values of scale mean we move down the pyramid. !*/ drectangletensor_space_to_image_space( const tensor& data, drectangle r ) const; /*! requires - data is a tensor that was produced by this->to_tensor() ensures - This function maps from to_tensor()'s output tensor space to its input image space. Therefore, given that data is a tensor produced by to_tensor(), tensor_space_to_image_space() allows you to ask for the rectangle in the input image that corresponds to a rectangle in data. - It should be noted that this function isn't always an inverse of image_space_to_tensor_space(). This is because you can ask image_space_to_tensor_space() for the coordinates of points outside the input image and they will be mapped to somewhere that doesn't have an inverse. But for points actually inside the input image this function performs an approximate inverse mapping. I.e. when image_contained_point(data,center(r))==true there is an approximate inverse. !*/}; // ---------------------------------------------------------------------------------------- template < typename PYRAMID_TYPE > classinput_rgb_image_pyramid{/*! REQUIREMENTS ON PYRAMID_TYPE PYRAMID_TYPE must be an instance of the dlib::pyramid_down template. WHAT THIS OBJECT REPRESENTS This input layer works with RGB images of type matrix<rgb_pixel>. It is identical to input_rgb_image except that it outputs a tensor containing a tiled image pyramid of each input image rather than a simple copy of each image. The tiled image pyramid is created using create_tiled_pyramid(). !*/ public: typedef matrix<rgb_pixel> input_type; typedef PYRAMID_TYPE pyramid_type;input_rgb_image_pyramid( ); /*! ensures - #get_avg_red() == 122.782 - #get_avg_green() == 117.001 - #get_avg_blue() == 104.298 - #get_pyramid_padding() == 10 - #get_pyramid_outer_padding() == 11 !*/input_rgb_image_pyramid(floatavg_red,floatavg_green,floatavg_blue ); /*! ensures - #get_avg_red() == avg_red - #get_avg_green() == avg_green - #get_avg_blue() == avg_blue - #get_pyramid_padding() == 10 - #get_pyramid_outer_padding() == 11 !*/floatget_avg_red( ) const; /*! ensures - returns the value subtracted from the red color channel. !*/floatget_avg_green( ) const; /*! ensures - returns the value subtracted from the green color channel. !*/floatget_avg_blue( ) const; /*! ensures - returns the value subtracted from the blue color channel. !*/unsignedlongget_pyramid_padding( ) const; /*! ensures - When this object creates a pyramid it will call create_tiled_pyramid() and set create_tiled_pyramid's pyramid_padding parameter to get_pyramid_padding(). !*/voidset_pyramid_padding(unsignedlongvalue ); /*! ensures - #get_pyramid_padding() == value !*/unsignedlongget_pyramid_outer_padding( ) const; /*! ensures - When this object creates a pyramid it will call create_tiled_pyramid() and set create_tiled_pyramid's pyramid_outer_padding parameter to get_pyramid_outer_padding(). !*/voidset_pyramid_outer_padding(unsignedlongvalue ); /*! ensures - #get_pyramid_outer_padding() == value !*/ template <typename forward_iterator>voidto_tensor( forward_iterator ibegin, forward_iterator iend, resizable_tensor& data ) const; /*! requires - [ibegin, iend) is an iterator range over input_type objects. - std::distance(ibegin,iend) > 0 - The input range should contain images that all have the same dimensions. ensures - Converts the iterator range into a tensor and stores it into #data. In particular, we will have: - #data.num_samples() == std::distance(ibegin,iend) - #data.k() == 3 - Each sample in #data contains a tiled image pyramid of the corresponding input image. The tiled pyramid is created by create_tiled_pyramid(). Moreover, each color channel is normalized by having its average value subtracted (according to get_avg_red(), get_avg_green(), or get_avg_blue()) and then is divided by 256.0. !*/boolimage_contained_point( const tensor& data, const point& p ) const; /*! requires - data is a tensor that was produced by this->to_tensor() ensures - Since data is a tensor that is built from a bunch of identically sized images, we can ask if those images were big enough to contain the point p. This function returns the answer to that question. !*/ drectangleimage_space_to_tensor_space( const tensor& data,doublescale, drectangle r ) const; /*! requires - data is a tensor that was produced by this->to_tensor() - 0 < scale <= 1 ensures - This function maps from to_tensor()'s input image space to its output tensor space. Therefore, given that data is a tensor produced by to_tensor(), image_space_to_tensor_space() allows you to ask for the rectangle in data that corresponds to a rectangle in the original image space. Note that since the output tensor contains an image pyramid, there are multiple points in the output tensor that correspond to any input location. So you must also specify a scale so we know what level of the pyramid is needed. So given a rectangle r in an input image, you can ask, what rectangle in data corresponds to r when things are scale times smaller? That rectangle is returned by this function. - A scale of 1 means we don't move anywhere in the pyramid scale space relative to the input image while smaller values of scale mean we move down the pyramid. !*/ drectangletensor_space_to_image_space( const tensor& data, drectangle r ) const; /*! requires - data is a tensor that was produced by this->to_tensor() ensures - This function maps from to_tensor()'s output tensor space to its input image space. Therefore, given that data is a tensor produced by to_tensor(), tensor_space_to_image_space() allows you to ask for the rectangle in the input image that corresponds to a rectangle in data. - It should be noted that this function isn't always an inverse of image_space_to_tensor_space(). This is because you can ask image_space_to_tensor_space() for the coordinates of points outside the input image and they will be mapped to somewhere that doesn't have an inverse. But for points actually inside the input image this function performs an approximate inverse mapping. I.e. when image_contained_point(data,center(r))==true there is an approximate inverse. !*/}; // ----------------------------------------------------------------------------------------}#endif // DLIB_DNn_INPUT_ABSTRACT_H_