diff --git a/apps/traincascade/CMakeLists.txt b/apps/traincascade/CMakeLists.txt index cca56361e37f84fa3c621ecd055cc77f4cea8c46..ab32b4cfb53fb181c940087a5c97c31b5be487ab 100644 --- a/apps/traincascade/CMakeLists.txt +++ b/apps/traincascade/CMakeLists.txt @@ -1,4 +1,4 @@ -set(OPENCV_TRAINCASCADE_DEPS opencv_core opencv_ml opencv_imgproc opencv_photo opencv_objdetect opencv_imgcodecs opencv_videoio opencv_highgui opencv_calib3d opencv_video opencv_features2d) +set(OPENCV_TRAINCASCADE_DEPS opencv_core opencv_imgproc opencv_objdetect opencv_imgcodecs opencv_highgui opencv_calib3d opencv_features2d) ocv_check_dependencies(${OPENCV_TRAINCASCADE_DEPS}) if(NOT OCV_DEPENDENCIES_FOUND) @@ -10,13 +10,10 @@ project(traincascade) ocv_include_directories("${CMAKE_CURRENT_SOURCE_DIR}" "${OpenCV_SOURCE_DIR}/include/opencv") ocv_include_modules(${OPENCV_TRAINCASCADE_DEPS}) -set(traincascade_files traincascade.cpp - cascadeclassifier.cpp cascadeclassifier.h - boost.cpp boost.h features.cpp traincascade_features.h - haarfeatures.cpp haarfeatures.h - lbpfeatures.cpp lbpfeatures.h - HOGfeatures.cpp HOGfeatures.h - imagestorage.cpp imagestorage.h) +file(GLOB SRCS *.cpp) +file(GLOB HDRS *.h*) + +set(traincascade_files ${SRCS} ${HDRS}) set(the_target opencv_traincascade) add_executable(${the_target} ${traincascade_files}) diff --git a/apps/traincascade/boost.h b/apps/traincascade/boost.h index 0edf776a5bfca6a48235ec1fbf843abee641574e..48d4789b9cb4c88048abb8fea916b1a8c2ae8712 100644 --- a/apps/traincascade/boost.h +++ b/apps/traincascade/boost.h @@ -2,7 +2,7 @@ #define _OPENCV_BOOST_H_ #include "traincascade_features.h" -#include "ml.h" +#include "old_ml.hpp" struct CvCascadeBoostParams : CvBoostParams { diff --git a/apps/traincascade/cascadeclassifier.h b/apps/traincascade/cascadeclassifier.h index 93be478b4e12de52d3471f9788cb2b9da1b1cfb9..6d6cb5b3f9d72bf1aae38a35b05d13d67e3544c2 100644 --- a/apps/traincascade/cascadeclassifier.h +++ b/apps/traincascade/cascadeclassifier.h @@ -7,8 +7,6 @@ #include "lbpfeatures.h" #include "HOGfeatures.h" //new #include "boost.h" -#include "cv.h" -#include "cxcore.h" #define CC_CASCADE_FILENAME "cascade.xml" #define CC_PARAMS_FILENAME "params.xml" diff --git a/apps/traincascade/old_ml.hpp b/apps/traincascade/old_ml.hpp new file mode 100644 index 0000000000000000000000000000000000000000..6ec31a025d605a1fe61a3b2d259529f0998e1575 --- /dev/null +++ b/apps/traincascade/old_ml.hpp @@ -0,0 +1,2165 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_ML_HPP__ +#define __OPENCV_ML_HPP__ + +#ifdef __cplusplus +# include "opencv2/core.hpp" +#endif + +#include "opencv2/core/core_c.h" +#include + +#ifdef __cplusplus + +#include +#include + +// Apple defines a check() macro somewhere in the debug headers +// that interferes with a method definiton in this header +#undef check + +/****************************************************************************************\ +* Main struct definitions * +\****************************************************************************************/ + +/* log(2*PI) */ +#define CV_LOG2PI (1.8378770664093454835606594728112) + +/* columns of matrix are training samples */ +#define CV_COL_SAMPLE 0 + +/* rows of matrix are training samples */ +#define CV_ROW_SAMPLE 1 + +#define CV_IS_ROW_SAMPLE(flags) ((flags) & CV_ROW_SAMPLE) + +struct CvVectors +{ + int type; + int dims, count; + CvVectors* next; + union + { + uchar** ptr; + float** fl; + double** db; + } data; +}; + +#if 0 +/* A structure, representing the lattice range of statmodel parameters. + It is used for optimizing statmodel parameters by cross-validation method. + The lattice is logarithmic, so must be greater then 1. */ +typedef struct CvParamLattice +{ + double min_val; + double max_val; + double step; +} +CvParamLattice; + +CV_INLINE CvParamLattice cvParamLattice( double min_val, double max_val, + double log_step ) +{ + CvParamLattice pl; + pl.min_val = MIN( min_val, max_val ); + pl.max_val = MAX( min_val, max_val ); + pl.step = MAX( log_step, 1. ); + return pl; +} + +CV_INLINE CvParamLattice cvDefaultParamLattice( void ) +{ + CvParamLattice pl = {0,0,0}; + return pl; +} +#endif + +/* Variable type */ +#define CV_VAR_NUMERICAL 0 +#define CV_VAR_ORDERED 0 +#define CV_VAR_CATEGORICAL 1 + +#define CV_TYPE_NAME_ML_SVM "opencv-ml-svm" +#define CV_TYPE_NAME_ML_KNN "opencv-ml-knn" +#define CV_TYPE_NAME_ML_NBAYES "opencv-ml-bayesian" +#define CV_TYPE_NAME_ML_EM "opencv-ml-em" +#define CV_TYPE_NAME_ML_BOOSTING "opencv-ml-boost-tree" +#define CV_TYPE_NAME_ML_TREE "opencv-ml-tree" +#define CV_TYPE_NAME_ML_ANN_MLP "opencv-ml-ann-mlp" +#define CV_TYPE_NAME_ML_CNN "opencv-ml-cnn" +#define CV_TYPE_NAME_ML_RTREES "opencv-ml-random-trees" +#define CV_TYPE_NAME_ML_ERTREES "opencv-ml-extremely-randomized-trees" +#define CV_TYPE_NAME_ML_GBT "opencv-ml-gradient-boosting-trees" + +#define CV_TRAIN_ERROR 0 +#define CV_TEST_ERROR 1 + +class CvStatModel +{ +public: + CvStatModel(); + virtual ~CvStatModel(); + + virtual void clear(); + + CV_WRAP virtual void save( const char* filename, const char* name=0 ) const; + CV_WRAP virtual void load( const char* filename, const char* name=0 ); + + virtual void write( CvFileStorage* storage, const char* name ) const; + virtual void read( CvFileStorage* storage, CvFileNode* node ); + +protected: + const char* default_model_name; +}; + +/****************************************************************************************\ +* Normal Bayes Classifier * +\****************************************************************************************/ + +/* The structure, representing the grid range of statmodel parameters. + It is used for optimizing statmodel accuracy by varying model parameters, + the accuracy estimate being computed by cross-validation. + The grid is logarithmic, so must be greater then 1. */ + +class CvMLData; + +struct CvParamGrid +{ + // SVM params type + enum { SVM_C=0, SVM_GAMMA=1, SVM_P=2, SVM_NU=3, SVM_COEF=4, SVM_DEGREE=5 }; + + CvParamGrid() + { + min_val = max_val = step = 0; + } + + CvParamGrid( double min_val, double max_val, double log_step ); + //CvParamGrid( int param_id ); + bool check() const; + + CV_PROP_RW double min_val; + CV_PROP_RW double max_val; + CV_PROP_RW double step; +}; + +inline CvParamGrid::CvParamGrid( double _min_val, double _max_val, double _log_step ) +{ + min_val = _min_val; + max_val = _max_val; + step = _log_step; +} + +class CvNormalBayesClassifier : public CvStatModel +{ +public: + CV_WRAP CvNormalBayesClassifier(); + virtual ~CvNormalBayesClassifier(); + + CvNormalBayesClassifier( const CvMat* trainData, const CvMat* responses, + const CvMat* varIdx=0, const CvMat* sampleIdx=0 ); + + virtual bool train( const CvMat* trainData, const CvMat* responses, + const CvMat* varIdx = 0, const CvMat* sampleIdx=0, bool update=false ); + + virtual float predict( const CvMat* samples, CV_OUT CvMat* results=0, CV_OUT CvMat* results_prob=0 ) const; + CV_WRAP virtual void clear(); + + CV_WRAP CvNormalBayesClassifier( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& varIdx=cv::Mat(), const cv::Mat& sampleIdx=cv::Mat() ); + CV_WRAP virtual bool train( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& varIdx = cv::Mat(), const cv::Mat& sampleIdx=cv::Mat(), + bool update=false ); + CV_WRAP virtual float predict( const cv::Mat& samples, CV_OUT cv::Mat* results=0, CV_OUT cv::Mat* results_prob=0 ) const; + + virtual void write( CvFileStorage* storage, const char* name ) const; + virtual void read( CvFileStorage* storage, CvFileNode* node ); + +protected: + int var_count, var_all; + CvMat* var_idx; + CvMat* cls_labels; + CvMat** count; + CvMat** sum; + CvMat** productsum; + CvMat** avg; + CvMat** inv_eigen_values; + CvMat** cov_rotate_mats; + CvMat* c; +}; + + +/****************************************************************************************\ +* K-Nearest Neighbour Classifier * +\****************************************************************************************/ + +// k Nearest Neighbors +class CvKNearest : public CvStatModel +{ +public: + + CV_WRAP CvKNearest(); + virtual ~CvKNearest(); + + CvKNearest( const CvMat* trainData, const CvMat* responses, + const CvMat* sampleIdx=0, bool isRegression=false, int max_k=32 ); + + virtual bool train( const CvMat* trainData, const CvMat* responses, + const CvMat* sampleIdx=0, bool is_regression=false, + int maxK=32, bool updateBase=false ); + + virtual float find_nearest( const CvMat* samples, int k, CV_OUT CvMat* results=0, + const float** neighbors=0, CV_OUT CvMat* neighborResponses=0, CV_OUT CvMat* dist=0 ) const; + + CV_WRAP CvKNearest( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& sampleIdx=cv::Mat(), bool isRegression=false, int max_k=32 ); + + CV_WRAP virtual bool train( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& sampleIdx=cv::Mat(), bool isRegression=false, + int maxK=32, bool updateBase=false ); + + virtual float find_nearest( const cv::Mat& samples, int k, cv::Mat* results=0, + const float** neighbors=0, cv::Mat* neighborResponses=0, + cv::Mat* dist=0 ) const; + CV_WRAP virtual float find_nearest( const cv::Mat& samples, int k, CV_OUT cv::Mat& results, + CV_OUT cv::Mat& neighborResponses, CV_OUT cv::Mat& dists) const; + + virtual void clear(); + int get_max_k() const; + int get_var_count() const; + int get_sample_count() const; + bool is_regression() const; + + virtual float write_results( int k, int k1, int start, int end, + const float* neighbor_responses, const float* dist, CvMat* _results, + CvMat* _neighbor_responses, CvMat* _dist, Cv32suf* sort_buf ) const; + + virtual void find_neighbors_direct( const CvMat* _samples, int k, int start, int end, + float* neighbor_responses, const float** neighbors, float* dist ) const; + +protected: + + int max_k, var_count; + int total; + bool regression; + CvVectors* samples; +}; + +/****************************************************************************************\ +* Support Vector Machines * +\****************************************************************************************/ + +// SVM training parameters +struct CvSVMParams +{ + CvSVMParams(); + CvSVMParams( int svm_type, int kernel_type, + double degree, double gamma, double coef0, + double Cvalue, double nu, double p, + CvMat* class_weights, CvTermCriteria term_crit ); + + CV_PROP_RW int svm_type; + CV_PROP_RW int kernel_type; + CV_PROP_RW double degree; // for poly + CV_PROP_RW double gamma; // for poly/rbf/sigmoid/chi2 + CV_PROP_RW double coef0; // for poly/sigmoid + + CV_PROP_RW double C; // for CV_SVM_C_SVC, CV_SVM_EPS_SVR and CV_SVM_NU_SVR + CV_PROP_RW double nu; // for CV_SVM_NU_SVC, CV_SVM_ONE_CLASS, and CV_SVM_NU_SVR + CV_PROP_RW double p; // for CV_SVM_EPS_SVR + CvMat* class_weights; // for CV_SVM_C_SVC + CV_PROP_RW CvTermCriteria term_crit; // termination criteria +}; + + +struct CvSVMKernel +{ + typedef void (CvSVMKernel::*Calc)( int vec_count, int vec_size, const float** vecs, + const float* another, float* results ); + CvSVMKernel(); + CvSVMKernel( const CvSVMParams* params, Calc _calc_func ); + virtual bool create( const CvSVMParams* params, Calc _calc_func ); + virtual ~CvSVMKernel(); + + virtual void clear(); + virtual void calc( int vcount, int n, const float** vecs, const float* another, float* results ); + + const CvSVMParams* params; + Calc calc_func; + + virtual void calc_non_rbf_base( int vec_count, int vec_size, const float** vecs, + const float* another, float* results, + double alpha, double beta ); + virtual void calc_intersec( int vcount, int var_count, const float** vecs, + const float* another, float* results ); + virtual void calc_chi2( int vec_count, int vec_size, const float** vecs, + const float* another, float* results ); + virtual void calc_linear( int vec_count, int vec_size, const float** vecs, + const float* another, float* results ); + virtual void calc_rbf( int vec_count, int vec_size, const float** vecs, + const float* another, float* results ); + virtual void calc_poly( int vec_count, int vec_size, const float** vecs, + const float* another, float* results ); + virtual void calc_sigmoid( int vec_count, int vec_size, const float** vecs, + const float* another, float* results ); +}; + + +struct CvSVMKernelRow +{ + CvSVMKernelRow* prev; + CvSVMKernelRow* next; + float* data; +}; + + +struct CvSVMSolutionInfo +{ + double obj; + double rho; + double upper_bound_p; + double upper_bound_n; + double r; // for Solver_NU +}; + +class CvSVMSolver +{ +public: + typedef bool (CvSVMSolver::*SelectWorkingSet)( int& i, int& j ); + typedef float* (CvSVMSolver::*GetRow)( int i, float* row, float* dst, bool existed ); + typedef void (CvSVMSolver::*CalcRho)( double& rho, double& r ); + + CvSVMSolver(); + + CvSVMSolver( int count, int var_count, const float** samples, schar* y, + int alpha_count, double* alpha, double Cp, double Cn, + CvMemStorage* storage, CvSVMKernel* kernel, GetRow get_row, + SelectWorkingSet select_working_set, CalcRho calc_rho ); + virtual bool create( int count, int var_count, const float** samples, schar* y, + int alpha_count, double* alpha, double Cp, double Cn, + CvMemStorage* storage, CvSVMKernel* kernel, GetRow get_row, + SelectWorkingSet select_working_set, CalcRho calc_rho ); + virtual ~CvSVMSolver(); + + virtual void clear(); + virtual bool solve_generic( CvSVMSolutionInfo& si ); + + virtual bool solve_c_svc( int count, int var_count, const float** samples, schar* y, + double Cp, double Cn, CvMemStorage* storage, + CvSVMKernel* kernel, double* alpha, CvSVMSolutionInfo& si ); + virtual bool solve_nu_svc( int count, int var_count, const float** samples, schar* y, + CvMemStorage* storage, CvSVMKernel* kernel, + double* alpha, CvSVMSolutionInfo& si ); + virtual bool solve_one_class( int count, int var_count, const float** samples, + CvMemStorage* storage, CvSVMKernel* kernel, + double* alpha, CvSVMSolutionInfo& si ); + + virtual bool solve_eps_svr( int count, int var_count, const float** samples, const float* y, + CvMemStorage* storage, CvSVMKernel* kernel, + double* alpha, CvSVMSolutionInfo& si ); + + virtual bool solve_nu_svr( int count, int var_count, const float** samples, const float* y, + CvMemStorage* storage, CvSVMKernel* kernel, + double* alpha, CvSVMSolutionInfo& si ); + + virtual float* get_row_base( int i, bool* _existed ); + virtual float* get_row( int i, float* dst ); + + int sample_count; + int var_count; + int cache_size; + int cache_line_size; + const float** samples; + const CvSVMParams* params; + CvMemStorage* storage; + CvSVMKernelRow lru_list; + CvSVMKernelRow* rows; + + int alpha_count; + + double* G; + double* alpha; + + // -1 - lower bound, 0 - free, 1 - upper bound + schar* alpha_status; + + schar* y; + double* b; + float* buf[2]; + double eps; + int max_iter; + double C[2]; // C[0] == Cn, C[1] == Cp + CvSVMKernel* kernel; + + SelectWorkingSet select_working_set_func; + CalcRho calc_rho_func; + GetRow get_row_func; + + virtual bool select_working_set( int& i, int& j ); + virtual bool select_working_set_nu_svm( int& i, int& j ); + virtual void calc_rho( double& rho, double& r ); + virtual void calc_rho_nu_svm( double& rho, double& r ); + + virtual float* get_row_svc( int i, float* row, float* dst, bool existed ); + virtual float* get_row_one_class( int i, float* row, float* dst, bool existed ); + virtual float* get_row_svr( int i, float* row, float* dst, bool existed ); +}; + + +struct CvSVMDecisionFunc +{ + double rho; + int sv_count; + double* alpha; + int* sv_index; +}; + + +// SVM model +class CvSVM : public CvStatModel +{ +public: + // SVM type + enum { C_SVC=100, NU_SVC=101, ONE_CLASS=102, EPS_SVR=103, NU_SVR=104 }; + + // SVM kernel type + enum { LINEAR=0, POLY=1, RBF=2, SIGMOID=3, CHI2=4, INTER=5 }; + + // SVM params type + enum { C=0, GAMMA=1, P=2, NU=3, COEF=4, DEGREE=5 }; + + CV_WRAP CvSVM(); + virtual ~CvSVM(); + + CvSVM( const CvMat* trainData, const CvMat* responses, + const CvMat* varIdx=0, const CvMat* sampleIdx=0, + CvSVMParams params=CvSVMParams() ); + + virtual bool train( const CvMat* trainData, const CvMat* responses, + const CvMat* varIdx=0, const CvMat* sampleIdx=0, + CvSVMParams params=CvSVMParams() ); + + virtual bool train_auto( const CvMat* trainData, const CvMat* responses, + const CvMat* varIdx, const CvMat* sampleIdx, CvSVMParams params, + int kfold = 10, + CvParamGrid Cgrid = get_default_grid(CvSVM::C), + CvParamGrid gammaGrid = get_default_grid(CvSVM::GAMMA), + CvParamGrid pGrid = get_default_grid(CvSVM::P), + CvParamGrid nuGrid = get_default_grid(CvSVM::NU), + CvParamGrid coeffGrid = get_default_grid(CvSVM::COEF), + CvParamGrid degreeGrid = get_default_grid(CvSVM::DEGREE), + bool balanced=false ); + + virtual float predict( const CvMat* sample, bool returnDFVal=false ) const; + virtual float predict( const CvMat* samples, CV_OUT CvMat* results, bool returnDFVal=false ) const; + + CV_WRAP CvSVM( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& varIdx=cv::Mat(), const cv::Mat& sampleIdx=cv::Mat(), + CvSVMParams params=CvSVMParams() ); + + CV_WRAP virtual bool train( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& varIdx=cv::Mat(), const cv::Mat& sampleIdx=cv::Mat(), + CvSVMParams params=CvSVMParams() ); + + CV_WRAP virtual bool train_auto( const cv::Mat& trainData, const cv::Mat& responses, + const cv::Mat& varIdx, const cv::Mat& sampleIdx, CvSVMParams params, + int k_fold = 10, + CvParamGrid Cgrid = CvSVM::get_default_grid(CvSVM::C), + CvParamGrid gammaGrid = CvSVM::get_default_grid(CvSVM::GAMMA), + CvParamGrid pGrid = CvSVM::get_default_grid(CvSVM::P), + CvParamGrid nuGrid = CvSVM::get_default_grid(CvSVM::NU), + CvParamGrid coeffGrid = CvSVM::get_default_grid(CvSVM::COEF), + CvParamGrid degreeGrid = CvSVM::get_default_grid(CvSVM::DEGREE), + bool balanced=false); + CV_WRAP virtual float predict( const cv::Mat& sample, bool returnDFVal=false ) const; + CV_WRAP_AS(predict_all) virtual void predict( cv::InputArray samples, cv::OutputArray results ) const; + + CV_WRAP virtual int get_support_vector_count() const; + virtual const float* get_support_vector(int i) const; + virtual CvSVMParams get_params() const { return params; } + CV_WRAP virtual void clear(); + + virtual const CvSVMDecisionFunc* get_decision_function() const { return decision_func; } + + static CvParamGrid get_default_grid( int param_id ); + + virtual void write( CvFileStorage* storage, const char* name ) const; + virtual void read( CvFileStorage* storage, CvFileNode* node ); + CV_WRAP int get_var_count() const { return var_idx ? var_idx->cols : var_all; } + +protected: + + virtual bool set_params( const CvSVMParams& params ); + virtual bool train1( int sample_count, int var_count, const float** samples, + const void* responses, double Cp, double Cn, + CvMemStorage* _storage, double* alpha, double& rho ); + virtual bool do_train( int svm_type, int sample_count, int var_count, const float** samples, + const CvMat* responses, CvMemStorage* _storage, double* alpha ); + virtual void create_kernel(); + virtual void create_solver(); + + virtual float predict( const float* row_sample, int row_len, bool returnDFVal=false ) const; + + virtual void write_params( CvFileStorage* fs ) const; + virtual void read_params( CvFileStorage* fs, CvFileNode* node ); + + void optimize_linear_svm(); + + CvSVMParams params; + CvMat* class_labels; + int var_all; + float** sv; + int sv_total; + CvMat* var_idx; + CvMat* class_weights; + CvSVMDecisionFunc* decision_func; + CvMemStorage* storage; + + CvSVMSolver* solver; + CvSVMKernel* kernel; + +private: + CvSVM(const CvSVM&); + CvSVM& operator = (const CvSVM&); +}; + +/****************************************************************************************\ +* Expectation - Maximization * +\****************************************************************************************/ +namespace cv +{ +class EM : public Algorithm +{ +public: + // Type of covariation matrices + enum {COV_MAT_SPHERICAL=0, COV_MAT_DIAGONAL=1, COV_MAT_GENERIC=2, COV_MAT_DEFAULT=COV_MAT_DIAGONAL}; + + // Default parameters + enum {DEFAULT_NCLUSTERS=5, DEFAULT_MAX_ITERS=100}; + + // The initial step + enum {START_E_STEP=1, START_M_STEP=2, START_AUTO_STEP=0}; + + CV_WRAP EM(int nclusters=EM::DEFAULT_NCLUSTERS, int covMatType=EM::COV_MAT_DIAGONAL, + const TermCriteria& termCrit=TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, + EM::DEFAULT_MAX_ITERS, FLT_EPSILON)); + + virtual ~EM(); + CV_WRAP virtual void clear(); + + CV_WRAP virtual bool train(InputArray samples, + OutputArray logLikelihoods=noArray(), + OutputArray labels=noArray(), + OutputArray probs=noArray()); + + CV_WRAP virtual bool trainE(InputArray samples, + InputArray means0, + InputArray covs0=noArray(), + InputArray weights0=noArray(), + OutputArray logLikelihoods=noArray(), + OutputArray labels=noArray(), + OutputArray probs=noArray()); + + CV_WRAP virtual bool trainM(InputArray samples, + InputArray probs0, + OutputArray logLikelihoods=noArray(), + OutputArray labels=noArray(), + OutputArray probs=noArray()); + + CV_WRAP Vec2d predict(InputArray sample, + OutputArray probs=noArray()) const; + + CV_WRAP bool isTrained() const; + + AlgorithmInfo* info() const; + virtual void read(const FileNode& fn); + +protected: + + virtual void setTrainData(int startStep, const Mat& samples, + const Mat* probs0, + const Mat* means0, + const std::vector* covs0, + const Mat* weights0); + + bool doTrain(int startStep, + OutputArray logLikelihoods, + OutputArray labels, + OutputArray probs); + virtual void eStep(); + virtual void mStep(); + + void clusterTrainSamples(); + void decomposeCovs(); + void computeLogWeightDivDet(); + + Vec2d computeProbabilities(const Mat& sample, Mat* probs) const; + + // all inner matrices have type CV_64FC1 + CV_PROP_RW int nclusters; + CV_PROP_RW int covMatType; + CV_PROP_RW int maxIters; + CV_PROP_RW double epsilon; + + Mat trainSamples; + Mat trainProbs; + Mat trainLogLikelihoods; + Mat trainLabels; + + CV_PROP Mat weights; + CV_PROP Mat means; + CV_PROP std::vector covs; + + std::vector covsEigenValues; + std::vector covsRotateMats; + std::vector invCovsEigenValues; + Mat logWeightDivDet; +}; +} // namespace cv + +/****************************************************************************************\ +* Decision Tree * +\****************************************************************************************/\ +struct CvPair16u32s +{ + unsigned short* u; + int* i; +}; + + +#define CV_DTREE_CAT_DIR(idx,subset) \ + (2*((subset[(idx)>>5]&(1 << ((idx) & 31)))==0)-1) + +struct CvDTreeSplit +{ + int var_idx; + int condensed_idx; + int inversed; + float quality; + CvDTreeSplit* next; + union + { + int subset[2]; + struct + { + float c; + int split_point; + } + ord; + }; +}; + +struct CvDTreeNode +{ + int class_idx; + int Tn; + double value; + + CvDTreeNode* parent; + CvDTreeNode* left; + CvDTreeNode* right; + + CvDTreeSplit* split; + + int sample_count; + int depth; + int* num_valid; + int offset; + int buf_idx; + double maxlr; + + // global pruning data + int complexity; + double alpha; + double node_risk, tree_risk, tree_error; + + // cross-validation pruning data + int* cv_Tn; + double* cv_node_risk; + double* cv_node_error; + + int get_num_valid(int vi) { return num_valid ? num_valid[vi] : sample_count; } + void set_num_valid(int vi, int n) { if( num_valid ) num_valid[vi] = n; } +}; + + +struct CvDTreeParams +{ + CV_PROP_RW int max_categories; + CV_PROP_RW int max_depth; + CV_PROP_RW int min_sample_count; + CV_PROP_RW int cv_folds; + CV_PROP_RW bool use_surrogates; + CV_PROP_RW bool use_1se_rule; + CV_PROP_RW bool truncate_pruned_tree; + CV_PROP_RW float regression_accuracy; + const float* priors; + + CvDTreeParams(); + CvDTreeParams( int max_depth, int min_sample_count, + float regression_accuracy, bool use_surrogates, + int max_categories, int cv_folds, + bool use_1se_rule, bool truncate_pruned_tree, + const float* priors ); +}; + + +struct CvDTreeTrainData +{ + CvDTreeTrainData(); + CvDTreeTrainData( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + const CvDTreeParams& params=CvDTreeParams(), + bool _shared=false, bool _add_labels=false ); + virtual ~CvDTreeTrainData(); + + virtual void set_data( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + const CvDTreeParams& params=CvDTreeParams(), + bool _shared=false, bool _add_labels=false, + bool _update_data=false ); + virtual void do_responses_copy(); + + virtual void get_vectors( const CvMat* _subsample_idx, + float* values, uchar* missing, float* responses, bool get_class_idx=false ); + + virtual CvDTreeNode* subsample_data( const CvMat* _subsample_idx ); + + virtual void write_params( CvFileStorage* fs ) const; + virtual void read_params( CvFileStorage* fs, CvFileNode* node ); + + // release all the data + virtual void clear(); + + int get_num_classes() const; + int get_var_type(int vi) const; + int get_work_var_count() const {return work_var_count;} + + virtual const float* get_ord_responses( CvDTreeNode* n, float* values_buf, int* sample_indices_buf ); + virtual const int* get_class_labels( CvDTreeNode* n, int* labels_buf ); + virtual const int* get_cv_labels( CvDTreeNode* n, int* labels_buf ); + virtual const int* get_sample_indices( CvDTreeNode* n, int* indices_buf ); + virtual const int* get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf ); + virtual void get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* sorted_indices_buf, + const float** ord_values, const int** sorted_indices, int* sample_indices_buf ); + virtual int get_child_buf_idx( CvDTreeNode* n ); + + //////////////////////////////////// + + virtual bool set_params( const CvDTreeParams& params ); + virtual CvDTreeNode* new_node( CvDTreeNode* parent, int count, + int storage_idx, int offset ); + + virtual CvDTreeSplit* new_split_ord( int vi, float cmp_val, + int split_point, int inversed, float quality ); + virtual CvDTreeSplit* new_split_cat( int vi, float quality ); + virtual void free_node_data( CvDTreeNode* node ); + virtual void free_train_data(); + virtual void free_node( CvDTreeNode* node ); + + int sample_count, var_all, var_count, max_c_count; + int ord_var_count, cat_var_count, work_var_count; + bool have_labels, have_priors; + bool is_classifier; + int tflag; + + const CvMat* train_data; + const CvMat* responses; + CvMat* responses_copy; // used in Boosting + + int buf_count, buf_size; // buf_size is obsolete, please do not use it, use expression ((int64)buf->rows * (int64)buf->cols / buf_count) instead + bool shared; + int is_buf_16u; + + CvMat* cat_count; + CvMat* cat_ofs; + CvMat* cat_map; + + CvMat* counts; + CvMat* buf; + inline size_t get_length_subbuf() const + { + size_t res = (size_t)(work_var_count + 1) * (size_t)sample_count; + return res; + } + + CvMat* direction; + CvMat* split_buf; + + CvMat* var_idx; + CvMat* var_type; // i-th element = + // k<0 - ordered + // k>=0 - categorical, see k-th element of cat_* arrays + CvMat* priors; + CvMat* priors_mult; + + CvDTreeParams params; + + CvMemStorage* tree_storage; + CvMemStorage* temp_storage; + + CvDTreeNode* data_root; + + CvSet* node_heap; + CvSet* split_heap; + CvSet* cv_heap; + CvSet* nv_heap; + + cv::RNG* rng; +}; + +class CvDTree; +class CvForestTree; + +namespace cv +{ + struct DTreeBestSplitFinder; + struct ForestTreeBestSplitFinder; +} + +class CvDTree : public CvStatModel +{ +public: + CV_WRAP CvDTree(); + virtual ~CvDTree(); + + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvDTreeParams params=CvDTreeParams() ); + + virtual bool train( CvMLData* trainData, CvDTreeParams params=CvDTreeParams() ); + + // type in {CV_TRAIN_ERROR, CV_TEST_ERROR} + virtual float calc_error( CvMLData* trainData, int type, std::vector *resp = 0 ); + + virtual bool train( CvDTreeTrainData* trainData, const CvMat* subsampleIdx ); + + virtual CvDTreeNode* predict( const CvMat* sample, const CvMat* missingDataMask=0, + bool preprocessedInput=false ) const; + + CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvDTreeParams params=CvDTreeParams() ); + + CV_WRAP virtual CvDTreeNode* predict( const cv::Mat& sample, const cv::Mat& missingDataMask=cv::Mat(), + bool preprocessedInput=false ) const; + CV_WRAP virtual cv::Mat getVarImportance(); + + virtual const CvMat* get_var_importance(); + CV_WRAP virtual void clear(); + + virtual void read( CvFileStorage* fs, CvFileNode* node ); + virtual void write( CvFileStorage* fs, const char* name ) const; + + // special read & write methods for trees in the tree ensembles + virtual void read( CvFileStorage* fs, CvFileNode* node, + CvDTreeTrainData* data ); + virtual void write( CvFileStorage* fs ) const; + + const CvDTreeNode* get_root() const; + int get_pruned_tree_idx() const; + CvDTreeTrainData* get_data(); + +protected: + friend struct cv::DTreeBestSplitFinder; + + virtual bool do_train( const CvMat* _subsample_idx ); + + virtual void try_split_node( CvDTreeNode* n ); + virtual void split_node_data( CvDTreeNode* n ); + virtual CvDTreeSplit* find_best_split( CvDTreeNode* n ); + virtual CvDTreeSplit* find_split_ord_class( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_cat_class( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_ord_reg( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_cat_reg( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_surrogate_split_ord( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_surrogate_split_cat( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); + virtual double calc_node_dir( CvDTreeNode* node ); + virtual void complete_node_dir( CvDTreeNode* node ); + virtual void cluster_categories( const int* vectors, int vector_count, + int var_count, int* sums, int k, int* cluster_labels ); + + virtual void calc_node_value( CvDTreeNode* node ); + + virtual void prune_cv(); + virtual double update_tree_rnc( int T, int fold ); + virtual int cut_tree( int T, int fold, double min_alpha ); + virtual void free_prune_data(bool cut_tree); + virtual void free_tree(); + + virtual void write_node( CvFileStorage* fs, CvDTreeNode* node ) const; + virtual void write_split( CvFileStorage* fs, CvDTreeSplit* split ) const; + virtual CvDTreeNode* read_node( CvFileStorage* fs, CvFileNode* node, CvDTreeNode* parent ); + virtual CvDTreeSplit* read_split( CvFileStorage* fs, CvFileNode* node ); + virtual void write_tree_nodes( CvFileStorage* fs ) const; + virtual void read_tree_nodes( CvFileStorage* fs, CvFileNode* node ); + + CvDTreeNode* root; + CvMat* var_importance; + CvDTreeTrainData* data; + CvMat train_data_hdr, responses_hdr; + cv::Mat train_data_mat, responses_mat; + +public: + int pruned_tree_idx; +}; + + +/****************************************************************************************\ +* Random Trees Classifier * +\****************************************************************************************/ + +class CvRTrees; + +class CvForestTree: public CvDTree +{ +public: + CvForestTree(); + virtual ~CvForestTree(); + + virtual bool train( CvDTreeTrainData* trainData, const CvMat* _subsample_idx, CvRTrees* forest ); + + virtual int get_var_count() const {return data ? data->var_count : 0;} + virtual void read( CvFileStorage* fs, CvFileNode* node, CvRTrees* forest, CvDTreeTrainData* _data ); + + /* dummy methods to avoid warnings: BEGIN */ + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvDTreeParams params=CvDTreeParams() ); + + virtual bool train( CvDTreeTrainData* trainData, const CvMat* _subsample_idx ); + virtual void read( CvFileStorage* fs, CvFileNode* node ); + virtual void read( CvFileStorage* fs, CvFileNode* node, + CvDTreeTrainData* data ); + /* dummy methods to avoid warnings: END */ + +protected: + friend struct cv::ForestTreeBestSplitFinder; + + virtual CvDTreeSplit* find_best_split( CvDTreeNode* n ); + CvRTrees* forest; +}; + + +struct CvRTParams : public CvDTreeParams +{ + //Parameters for the forest + CV_PROP_RW bool calc_var_importance; // true <=> RF processes variable importance + CV_PROP_RW int nactive_vars; + CV_PROP_RW CvTermCriteria term_crit; + + CvRTParams(); + CvRTParams( int max_depth, int min_sample_count, + float regression_accuracy, bool use_surrogates, + int max_categories, const float* priors, bool calc_var_importance, + int nactive_vars, int max_num_of_trees_in_the_forest, + float forest_accuracy, int termcrit_type ); +}; + + +class CvRTrees : public CvStatModel +{ +public: + CV_WRAP CvRTrees(); + virtual ~CvRTrees(); + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvRTParams params=CvRTParams() ); + + virtual bool train( CvMLData* data, CvRTParams params=CvRTParams() ); + virtual float predict( const CvMat* sample, const CvMat* missing = 0 ) const; + virtual float predict_prob( const CvMat* sample, const CvMat* missing = 0 ) const; + + CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvRTParams params=CvRTParams() ); + CV_WRAP virtual float predict( const cv::Mat& sample, const cv::Mat& missing = cv::Mat() ) const; + CV_WRAP virtual float predict_prob( const cv::Mat& sample, const cv::Mat& missing = cv::Mat() ) const; + CV_WRAP virtual cv::Mat getVarImportance(); + + CV_WRAP virtual void clear(); + + virtual const CvMat* get_var_importance(); + virtual float get_proximity( const CvMat* sample1, const CvMat* sample2, + const CvMat* missing1 = 0, const CvMat* missing2 = 0 ) const; + + virtual float calc_error( CvMLData* data, int type , std::vector* resp = 0 ); // type in {CV_TRAIN_ERROR, CV_TEST_ERROR} + + virtual float get_train_error(); + + virtual void read( CvFileStorage* fs, CvFileNode* node ); + virtual void write( CvFileStorage* fs, const char* name ) const; + + CvMat* get_active_var_mask(); + CvRNG* get_rng(); + + int get_tree_count() const; + CvForestTree* get_tree(int i) const; + +protected: + virtual cv::String getName() const; + + virtual bool grow_forest( const CvTermCriteria term_crit ); + + // array of the trees of the forest + CvForestTree** trees; + CvDTreeTrainData* data; + CvMat train_data_hdr, responses_hdr; + cv::Mat train_data_mat, responses_mat; + int ntrees; + int nclasses; + double oob_error; + CvMat* var_importance; + int nsamples; + + cv::RNG* rng; + CvMat* active_var_mask; +}; + +/****************************************************************************************\ +* Extremely randomized trees Classifier * +\****************************************************************************************/ +struct CvERTreeTrainData : public CvDTreeTrainData +{ + virtual void set_data( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + const CvDTreeParams& params=CvDTreeParams(), + bool _shared=false, bool _add_labels=false, + bool _update_data=false ); + virtual void get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* missing_buf, + const float** ord_values, const int** missing, int* sample_buf = 0 ); + virtual const int* get_sample_indices( CvDTreeNode* n, int* indices_buf ); + virtual const int* get_cv_labels( CvDTreeNode* n, int* labels_buf ); + virtual const int* get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf ); + virtual void get_vectors( const CvMat* _subsample_idx, float* values, uchar* missing, + float* responses, bool get_class_idx=false ); + virtual CvDTreeNode* subsample_data( const CvMat* _subsample_idx ); + const CvMat* missing_mask; +}; + +class CvForestERTree : public CvForestTree +{ +protected: + virtual double calc_node_dir( CvDTreeNode* node ); + virtual CvDTreeSplit* find_split_ord_class( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_cat_class( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_ord_reg( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_cat_reg( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual void split_node_data( CvDTreeNode* n ); +}; + +class CvERTrees : public CvRTrees +{ +public: + CV_WRAP CvERTrees(); + virtual ~CvERTrees(); + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvRTParams params=CvRTParams()); + CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvRTParams params=CvRTParams()); + virtual bool train( CvMLData* data, CvRTParams params=CvRTParams() ); +protected: + virtual cv::String getName() const; + virtual bool grow_forest( const CvTermCriteria term_crit ); +}; + + +/****************************************************************************************\ +* Boosted tree classifier * +\****************************************************************************************/ + +struct CvBoostParams : public CvDTreeParams +{ + CV_PROP_RW int boost_type; + CV_PROP_RW int weak_count; + CV_PROP_RW int split_criteria; + CV_PROP_RW double weight_trim_rate; + + CvBoostParams(); + CvBoostParams( int boost_type, int weak_count, double weight_trim_rate, + int max_depth, bool use_surrogates, const float* priors ); +}; + + +class CvBoost; + +class CvBoostTree: public CvDTree +{ +public: + CvBoostTree(); + virtual ~CvBoostTree(); + + virtual bool train( CvDTreeTrainData* trainData, + const CvMat* subsample_idx, CvBoost* ensemble ); + + virtual void scale( double s ); + virtual void read( CvFileStorage* fs, CvFileNode* node, + CvBoost* ensemble, CvDTreeTrainData* _data ); + virtual void clear(); + + /* dummy methods to avoid warnings: BEGIN */ + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvDTreeParams params=CvDTreeParams() ); + virtual bool train( CvDTreeTrainData* trainData, const CvMat* _subsample_idx ); + + virtual void read( CvFileStorage* fs, CvFileNode* node ); + virtual void read( CvFileStorage* fs, CvFileNode* node, + CvDTreeTrainData* data ); + /* dummy methods to avoid warnings: END */ + +protected: + + virtual void try_split_node( CvDTreeNode* n ); + virtual CvDTreeSplit* find_surrogate_split_ord( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_surrogate_split_cat( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_ord_class( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_cat_class( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_ord_reg( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual CvDTreeSplit* find_split_cat_reg( CvDTreeNode* n, int vi, + float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); + virtual void calc_node_value( CvDTreeNode* n ); + virtual double calc_node_dir( CvDTreeNode* n ); + + CvBoost* ensemble; +}; + + +class CvBoost : public CvStatModel +{ +public: + // Boosting type + enum { DISCRETE=0, REAL=1, LOGIT=2, GENTLE=3 }; + + // Splitting criteria + enum { DEFAULT=0, GINI=1, MISCLASS=3, SQERR=4 }; + + CV_WRAP CvBoost(); + virtual ~CvBoost(); + + CvBoost( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvBoostParams params=CvBoostParams() ); + + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvBoostParams params=CvBoostParams(), + bool update=false ); + + virtual bool train( CvMLData* data, + CvBoostParams params=CvBoostParams(), + bool update=false ); + + virtual float predict( const CvMat* sample, const CvMat* missing=0, + CvMat* weak_responses=0, CvSlice slice=CV_WHOLE_SEQ, + bool raw_mode=false, bool return_sum=false ) const; + + CV_WRAP CvBoost( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvBoostParams params=CvBoostParams() ); + + CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvBoostParams params=CvBoostParams(), + bool update=false ); + + CV_WRAP virtual float predict( const cv::Mat& sample, const cv::Mat& missing=cv::Mat(), + const cv::Range& slice=cv::Range::all(), bool rawMode=false, + bool returnSum=false ) const; + + virtual float calc_error( CvMLData* _data, int type , std::vector *resp = 0 ); // type in {CV_TRAIN_ERROR, CV_TEST_ERROR} + + CV_WRAP virtual void prune( CvSlice slice ); + + CV_WRAP virtual void clear(); + + virtual void write( CvFileStorage* storage, const char* name ) const; + virtual void read( CvFileStorage* storage, CvFileNode* node ); + virtual const CvMat* get_active_vars(bool absolute_idx=true); + + CvSeq* get_weak_predictors(); + + CvMat* get_weights(); + CvMat* get_subtree_weights(); + CvMat* get_weak_response(); + const CvBoostParams& get_params() const; + const CvDTreeTrainData* get_data() const; + +protected: + + virtual bool set_params( const CvBoostParams& params ); + virtual void update_weights( CvBoostTree* tree ); + virtual void trim_weights(); + virtual void write_params( CvFileStorage* fs ) const; + virtual void read_params( CvFileStorage* fs, CvFileNode* node ); + + virtual void initialize_weights(double (&p)[2]); + + CvDTreeTrainData* data; + CvMat train_data_hdr, responses_hdr; + cv::Mat train_data_mat, responses_mat; + CvBoostParams params; + CvSeq* weak; + + CvMat* active_vars; + CvMat* active_vars_abs; + bool have_active_cat_vars; + + CvMat* orig_response; + CvMat* sum_response; + CvMat* weak_eval; + CvMat* subsample_mask; + CvMat* weights; + CvMat* subtree_weights; + bool have_subsample; +}; + + +/****************************************************************************************\ +* Gradient Boosted Trees * +\****************************************************************************************/ + +// DataType: STRUCT CvGBTreesParams +// Parameters of GBT (Gradient Boosted trees model), including single +// tree settings and ensemble parameters. +// +// weak_count - count of trees in the ensemble +// loss_function_type - loss function used for ensemble training +// subsample_portion - portion of whole training set used for +// every single tree training. +// subsample_portion value is in (0.0, 1.0]. +// subsample_portion == 1.0 when whole dataset is +// used on each step. Count of sample used on each +// step is computed as +// int(total_samples_count * subsample_portion). +// shrinkage - regularization parameter. +// Each tree prediction is multiplied on shrinkage value. + + +struct CvGBTreesParams : public CvDTreeParams +{ + CV_PROP_RW int weak_count; + CV_PROP_RW int loss_function_type; + CV_PROP_RW float subsample_portion; + CV_PROP_RW float shrinkage; + + CvGBTreesParams(); + CvGBTreesParams( int loss_function_type, int weak_count, float shrinkage, + float subsample_portion, int max_depth, bool use_surrogates ); +}; + +// DataType: CLASS CvGBTrees +// Gradient Boosting Trees (GBT) algorithm implementation. +// +// data - training dataset +// params - parameters of the CvGBTrees +// weak - array[0..(class_count-1)] of CvSeq +// for storing tree ensembles +// orig_response - original responses of the training set samples +// sum_response - predicitons of the current model on the training dataset. +// this matrix is updated on every iteration. +// sum_response_tmp - predicitons of the model on the training set on the next +// step. On every iteration values of sum_responses_tmp are +// computed via sum_responses values. When the current +// step is complete sum_response values become equal to +// sum_responses_tmp. +// sampleIdx - indices of samples used for training the ensemble. +// CvGBTrees training procedure takes a set of samples +// (train_data) and a set of responses (responses). +// Only pairs (train_data[i], responses[i]), where i is +// in sample_idx are used for training the ensemble. +// subsample_train - indices of samples used for training a single decision +// tree on the current step. This indices are countered +// relatively to the sample_idx, so that pairs +// (train_data[sample_idx[i]], responses[sample_idx[i]]) +// are used for training a decision tree. +// Training set is randomly splited +// in two parts (subsample_train and subsample_test) +// on every iteration accordingly to the portion parameter. +// subsample_test - relative indices of samples from the training set, +// which are not used for training a tree on the current +// step. +// missing - mask of the missing values in the training set. This +// matrix has the same size as train_data. 1 - missing +// value, 0 - not a missing value. +// class_labels - output class labels map. +// rng - random number generator. Used for spliting the +// training set. +// class_count - count of output classes. +// class_count == 1 in the case of regression, +// and > 1 in the case of classification. +// delta - Huber loss function parameter. +// base_value - start point of the gradient descent procedure. +// model prediction is +// f(x) = f_0 + sum_{i=1..weak_count-1}(f_i(x)), where +// f_0 is the base value. + + + +class CvGBTrees : public CvStatModel +{ +public: + + /* + // DataType: ENUM + // Loss functions implemented in CvGBTrees. + // + // SQUARED_LOSS + // problem: regression + // loss = (x - x')^2 + // + // ABSOLUTE_LOSS + // problem: regression + // loss = abs(x - x') + // + // HUBER_LOSS + // problem: regression + // loss = delta*( abs(x - x') - delta/2), if abs(x - x') > delta + // 1/2*(x - x')^2, if abs(x - x') <= delta, + // where delta is the alpha-quantile of pseudo responses from + // the training set. + // + // DEVIANCE_LOSS + // problem: classification + // + */ + enum {SQUARED_LOSS=0, ABSOLUTE_LOSS, HUBER_LOSS=3, DEVIANCE_LOSS}; + + + /* + // Default constructor. Creates a model only (without training). + // Should be followed by one form of the train(...) function. + // + // API + // CvGBTrees(); + + // INPUT + // OUTPUT + // RESULT + */ + CV_WRAP CvGBTrees(); + + + /* + // Full form constructor. Creates a gradient boosting model and does the + // train. + // + // API + // CvGBTrees( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvGBTreesParams params=CvGBTreesParams() ); + + // INPUT + // trainData - a set of input feature vectors. + // size of matrix is + // x + // or x + // depending on the tflag parameter. + // matrix values are float. + // tflag - a flag showing how do samples stored in the + // trainData matrix row by row (tflag=CV_ROW_SAMPLE) + // or column by column (tflag=CV_COL_SAMPLE). + // responses - a vector of responses corresponding to the samples + // in trainData. + // varIdx - indices of used variables. zero value means that all + // variables are active. + // sampleIdx - indices of used samples. zero value means that all + // samples from trainData are in the training set. + // varType - vector of length. gives every + // variable type CV_VAR_CATEGORICAL or CV_VAR_ORDERED. + // varType = 0 means all variables are numerical. + // missingDataMask - a mask of misiing values in trainData. + // missingDataMask = 0 means that there are no missing + // values. + // params - parameters of GTB algorithm. + // OUTPUT + // RESULT + */ + CvGBTrees( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvGBTreesParams params=CvGBTreesParams() ); + + + /* + // Destructor. + */ + virtual ~CvGBTrees(); + + + /* + // Gradient tree boosting model training + // + // API + // virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvGBTreesParams params=CvGBTreesParams(), + bool update=false ); + + // INPUT + // trainData - a set of input feature vectors. + // size of matrix is + // x + // or x + // depending on the tflag parameter. + // matrix values are float. + // tflag - a flag showing how do samples stored in the + // trainData matrix row by row (tflag=CV_ROW_SAMPLE) + // or column by column (tflag=CV_COL_SAMPLE). + // responses - a vector of responses corresponding to the samples + // in trainData. + // varIdx - indices of used variables. zero value means that all + // variables are active. + // sampleIdx - indices of used samples. zero value means that all + // samples from trainData are in the training set. + // varType - vector of length. gives every + // variable type CV_VAR_CATEGORICAL or CV_VAR_ORDERED. + // varType = 0 means all variables are numerical. + // missingDataMask - a mask of misiing values in trainData. + // missingDataMask = 0 means that there are no missing + // values. + // params - parameters of GTB algorithm. + // update - is not supported now. (!) + // OUTPUT + // RESULT + // Error state. + */ + virtual bool train( const CvMat* trainData, int tflag, + const CvMat* responses, const CvMat* varIdx=0, + const CvMat* sampleIdx=0, const CvMat* varType=0, + const CvMat* missingDataMask=0, + CvGBTreesParams params=CvGBTreesParams(), + bool update=false ); + + + /* + // Gradient tree boosting model training + // + // API + // virtual bool train( CvMLData* data, + CvGBTreesParams params=CvGBTreesParams(), + bool update=false ) {return false;} + + // INPUT + // data - training set. + // params - parameters of GTB algorithm. + // update - is not supported now. (!) + // OUTPUT + // RESULT + // Error state. + */ + virtual bool train( CvMLData* data, + CvGBTreesParams params=CvGBTreesParams(), + bool update=false ); + + + /* + // Response value prediction + // + // API + // virtual float predict_serial( const CvMat* sample, const CvMat* missing=0, + CvMat* weak_responses=0, CvSlice slice = CV_WHOLE_SEQ, + int k=-1 ) const; + + // INPUT + // sample - input sample of the same type as in the training set. + // missing - missing values mask. missing=0 if there are no + // missing values in sample vector. + // weak_responses - predictions of all of the trees. + // not implemented (!) + // slice - part of the ensemble used for prediction. + // slice = CV_WHOLE_SEQ when all trees are used. + // k - number of ensemble used. + // k is in {-1,0,1,..,}. + // in the case of classification problem + // ensembles are built. + // If k = -1 ordinary prediction is the result, + // otherwise function gives the prediction of the + // k-th ensemble only. + // OUTPUT + // RESULT + // Predicted value. + */ + virtual float predict_serial( const CvMat* sample, const CvMat* missing=0, + CvMat* weakResponses=0, CvSlice slice = CV_WHOLE_SEQ, + int k=-1 ) const; + + /* + // Response value prediction. + // Parallel version (in the case of TBB existence) + // + // API + // virtual float predict( const CvMat* sample, const CvMat* missing=0, + CvMat* weak_responses=0, CvSlice slice = CV_WHOLE_SEQ, + int k=-1 ) const; + + // INPUT + // sample - input sample of the same type as in the training set. + // missing - missing values mask. missing=0 if there are no + // missing values in sample vector. + // weak_responses - predictions of all of the trees. + // not implemented (!) + // slice - part of the ensemble used for prediction. + // slice = CV_WHOLE_SEQ when all trees are used. + // k - number of ensemble used. + // k is in {-1,0,1,..,}. + // in the case of classification problem + // ensembles are built. + // If k = -1 ordinary prediction is the result, + // otherwise function gives the prediction of the + // k-th ensemble only. + // OUTPUT + // RESULT + // Predicted value. + */ + virtual float predict( const CvMat* sample, const CvMat* missing=0, + CvMat* weakResponses=0, CvSlice slice = CV_WHOLE_SEQ, + int k=-1 ) const; + + /* + // Deletes all the data. + // + // API + // virtual void clear(); + + // INPUT + // OUTPUT + // delete data, weak, orig_response, sum_response, + // weak_eval, subsample_train, subsample_test, + // sample_idx, missing, lass_labels + // delta = 0.0 + // RESULT + */ + CV_WRAP virtual void clear(); + + /* + // Compute error on the train/test set. + // + // API + // virtual float calc_error( CvMLData* _data, int type, + // std::vector *resp = 0 ); + // + // INPUT + // data - dataset + // type - defines which error is to compute: train (CV_TRAIN_ERROR) or + // test (CV_TEST_ERROR). + // OUTPUT + // resp - vector of predicitons + // RESULT + // Error value. + */ + virtual float calc_error( CvMLData* _data, int type, + std::vector *resp = 0 ); + + /* + // + // Write parameters of the gtb model and data. Write learned model. + // + // API + // virtual void write( CvFileStorage* fs, const char* name ) const; + // + // INPUT + // fs - file storage to read parameters from. + // name - model name. + // OUTPUT + // RESULT + */ + virtual void write( CvFileStorage* fs, const char* name ) const; + + + /* + // + // Read parameters of the gtb model and data. Read learned model. + // + // API + // virtual void read( CvFileStorage* fs, CvFileNode* node ); + // + // INPUT + // fs - file storage to read parameters from. + // node - file node. + // OUTPUT + // RESULT + */ + virtual void read( CvFileStorage* fs, CvFileNode* node ); + + + // new-style C++ interface + CV_WRAP CvGBTrees( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvGBTreesParams params=CvGBTreesParams() ); + + CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, + const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), + const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), + const cv::Mat& missingDataMask=cv::Mat(), + CvGBTreesParams params=CvGBTreesParams(), + bool update=false ); + + CV_WRAP virtual float predict( const cv::Mat& sample, const cv::Mat& missing=cv::Mat(), + const cv::Range& slice = cv::Range::all(), + int k=-1 ) const; + +protected: + + /* + // Compute the gradient vector components. + // + // API + // virtual void find_gradient( const int k = 0); + + // INPUT + // k - used for classification problem, determining current + // tree ensemble. + // OUTPUT + // changes components of data->responses + // which correspond to samples used for training + // on the current step. + // RESULT + */ + virtual void find_gradient( const int k = 0); + + + /* + // + // Change values in tree leaves according to the used loss function. + // + // API + // virtual void change_values(CvDTree* tree, const int k = 0); + // + // INPUT + // tree - decision tree to change. + // k - used for classification problem, determining current + // tree ensemble. + // OUTPUT + // changes 'value' fields of the trees' leaves. + // changes sum_response_tmp. + // RESULT + */ + virtual void change_values(CvDTree* tree, const int k = 0); + + + /* + // + // Find optimal constant prediction value according to the used loss + // function. + // The goal is to find a constant which gives the minimal summary loss + // on the _Idx samples. + // + // API + // virtual float find_optimal_value( const CvMat* _Idx ); + // + // INPUT + // _Idx - indices of the samples from the training set. + // OUTPUT + // RESULT + // optimal constant value. + */ + virtual float find_optimal_value( const CvMat* _Idx ); + + + /* + // + // Randomly split the whole training set in two parts according + // to params.portion. + // + // API + // virtual void do_subsample(); + // + // INPUT + // OUTPUT + // subsample_train - indices of samples used for training + // subsample_test - indices of samples used for test + // RESULT + */ + virtual void do_subsample(); + + + /* + // + // Internal recursive function giving an array of subtree tree leaves. + // + // API + // void leaves_get( CvDTreeNode** leaves, int& count, CvDTreeNode* node ); + // + // INPUT + // node - current leaf. + // OUTPUT + // count - count of leaves in the subtree. + // leaves - array of pointers to leaves. + // RESULT + */ + void leaves_get( CvDTreeNode** leaves, int& count, CvDTreeNode* node ); + + + /* + // + // Get leaves of the tree. + // + // API + // CvDTreeNode** GetLeaves( const CvDTree* dtree, int& len ); + // + // INPUT + // dtree - decision tree. + // OUTPUT + // len - count of the leaves. + // RESULT + // CvDTreeNode** - array of pointers to leaves. + */ + CvDTreeNode** GetLeaves( const CvDTree* dtree, int& len ); + + + /* + // + // Is it a regression or a classification. + // + // API + // bool problem_type(); + // + // INPUT + // OUTPUT + // RESULT + // false if it is a classification problem, + // true - if regression. + */ + virtual bool problem_type() const; + + + /* + // + // Write parameters of the gtb model. + // + // API + // virtual void write_params( CvFileStorage* fs ) const; + // + // INPUT + // fs - file storage to write parameters to. + // OUTPUT + // RESULT + */ + virtual void write_params( CvFileStorage* fs ) const; + + + /* + // + // Read parameters of the gtb model and data. + // + // API + // virtual void read_params( CvFileStorage* fs ); + // + // INPUT + // fs - file storage to read parameters from. + // OUTPUT + // params - parameters of the gtb model. + // data - contains information about the structure + // of the data set (count of variables, + // their types, etc.). + // class_labels - output class labels map. + // RESULT + */ + virtual void read_params( CvFileStorage* fs, CvFileNode* fnode ); + int get_len(const CvMat* mat) const; + + + CvDTreeTrainData* data; + CvGBTreesParams params; + + CvSeq** weak; + CvMat* orig_response; + CvMat* sum_response; + CvMat* sum_response_tmp; + CvMat* sample_idx; + CvMat* subsample_train; + CvMat* subsample_test; + CvMat* missing; + CvMat* class_labels; + + cv::RNG* rng; + + int class_count; + float delta; + float base_value; + +}; + + + +/****************************************************************************************\ +* Artificial Neural Networks (ANN) * +\****************************************************************************************/ + +/////////////////////////////////// Multi-Layer Perceptrons ////////////////////////////// + +struct CvANN_MLP_TrainParams +{ + CvANN_MLP_TrainParams(); + CvANN_MLP_TrainParams( CvTermCriteria term_crit, int train_method, + double param1, double param2=0 ); + ~CvANN_MLP_TrainParams(); + + enum { BACKPROP=0, RPROP=1 }; + + CV_PROP_RW CvTermCriteria term_crit; + CV_PROP_RW int train_method; + + // backpropagation parameters + CV_PROP_RW double bp_dw_scale, bp_moment_scale; + + // rprop parameters + CV_PROP_RW double rp_dw0, rp_dw_plus, rp_dw_minus, rp_dw_min, rp_dw_max; +}; + + +class CvANN_MLP : public CvStatModel +{ +public: + CV_WRAP CvANN_MLP(); + CvANN_MLP( const CvMat* layerSizes, + int activateFunc=CvANN_MLP::SIGMOID_SYM, + double fparam1=0, double fparam2=0 ); + + virtual ~CvANN_MLP(); + + virtual void create( const CvMat* layerSizes, + int activateFunc=CvANN_MLP::SIGMOID_SYM, + double fparam1=0, double fparam2=0 ); + + virtual int train( const CvMat* inputs, const CvMat* outputs, + const CvMat* sampleWeights, const CvMat* sampleIdx=0, + CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(), + int flags=0 ); + virtual float predict( const CvMat* inputs, CV_OUT CvMat* outputs ) const; + + CV_WRAP CvANN_MLP( const cv::Mat& layerSizes, + int activateFunc=CvANN_MLP::SIGMOID_SYM, + double fparam1=0, double fparam2=0 ); + + CV_WRAP virtual void create( const cv::Mat& layerSizes, + int activateFunc=CvANN_MLP::SIGMOID_SYM, + double fparam1=0, double fparam2=0 ); + + CV_WRAP virtual int train( const cv::Mat& inputs, const cv::Mat& outputs, + const cv::Mat& sampleWeights, const cv::Mat& sampleIdx=cv::Mat(), + CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(), + int flags=0 ); + + CV_WRAP virtual float predict( const cv::Mat& inputs, CV_OUT cv::Mat& outputs ) const; + + CV_WRAP virtual void clear(); + + // possible activation functions + enum { IDENTITY = 0, SIGMOID_SYM = 1, GAUSSIAN = 2 }; + + // available training flags + enum { UPDATE_WEIGHTS = 1, NO_INPUT_SCALE = 2, NO_OUTPUT_SCALE = 4 }; + + virtual void read( CvFileStorage* fs, CvFileNode* node ); + virtual void write( CvFileStorage* storage, const char* name ) const; + + int get_layer_count() { return layer_sizes ? layer_sizes->cols : 0; } + const CvMat* get_layer_sizes() { return layer_sizes; } + double* get_weights(int layer) + { + return layer_sizes && weights && + (unsigned)layer <= (unsigned)layer_sizes->cols ? weights[layer] : 0; + } + + virtual void calc_activ_func_deriv( CvMat* xf, CvMat* deriv, const double* bias ) const; + +protected: + + virtual bool prepare_to_train( const CvMat* _inputs, const CvMat* _outputs, + const CvMat* _sample_weights, const CvMat* sampleIdx, + CvVectors* _ivecs, CvVectors* _ovecs, double** _sw, int _flags ); + + // sequential random backpropagation + virtual int train_backprop( CvVectors _ivecs, CvVectors _ovecs, const double* _sw ); + + // RPROP algorithm + virtual int train_rprop( CvVectors _ivecs, CvVectors _ovecs, const double* _sw ); + + virtual void calc_activ_func( CvMat* xf, const double* bias ) const; + virtual void set_activ_func( int _activ_func=SIGMOID_SYM, + double _f_param1=0, double _f_param2=0 ); + virtual void init_weights(); + virtual void scale_input( const CvMat* _src, CvMat* _dst ) const; + virtual void scale_output( const CvMat* _src, CvMat* _dst ) const; + virtual void calc_input_scale( const CvVectors* vecs, int flags ); + virtual void calc_output_scale( const CvVectors* vecs, int flags ); + + virtual void write_params( CvFileStorage* fs ) const; + virtual void read_params( CvFileStorage* fs, CvFileNode* node ); + + CvMat* layer_sizes; + CvMat* wbuf; + CvMat* sample_weights; + double** weights; + double f_param1, f_param2; + double min_val, max_val, min_val1, max_val1; + int activ_func; + int max_count, max_buf_sz; + CvANN_MLP_TrainParams params; + cv::RNG* rng; +}; + +/****************************************************************************************\ +* Auxilary functions declarations * +\****************************************************************************************/ + +/* Generates from multivariate normal distribution, where - is an + average row vector, - symmetric covariation matrix */ +CVAPI(void) cvRandMVNormal( CvMat* mean, CvMat* cov, CvMat* sample, + CvRNG* rng CV_DEFAULT(0) ); + +/* Generates sample from gaussian mixture distribution */ +CVAPI(void) cvRandGaussMixture( CvMat* means[], + CvMat* covs[], + float weights[], + int clsnum, + CvMat* sample, + CvMat* sampClasses CV_DEFAULT(0) ); + +#define CV_TS_CONCENTRIC_SPHERES 0 + +/* creates test set */ +CVAPI(void) cvCreateTestSet( int type, CvMat** samples, + int num_samples, + int num_features, + CvMat** responses, + int num_classes, ... ); + +/****************************************************************************************\ +* Data * +\****************************************************************************************/ + +#define CV_COUNT 0 +#define CV_PORTION 1 + +struct CvTrainTestSplit +{ + CvTrainTestSplit(); + CvTrainTestSplit( int train_sample_count, bool mix = true); + CvTrainTestSplit( float train_sample_portion, bool mix = true); + + union + { + int count; + float portion; + } train_sample_part; + int train_sample_part_mode; + + bool mix; +}; + +class CvMLData +{ +public: + CvMLData(); + virtual ~CvMLData(); + + // returns: + // 0 - OK + // -1 - file can not be opened or is not correct + int read_csv( const char* filename ); + + const CvMat* get_values() const; + const CvMat* get_responses(); + const CvMat* get_missing() const; + + void set_header_lines_number( int n ); + int get_header_lines_number() const; + + void set_response_idx( int idx ); // old response become predictors, new response_idx = idx + // if idx < 0 there will be no response + int get_response_idx() const; + + void set_train_test_split( const CvTrainTestSplit * spl ); + const CvMat* get_train_sample_idx() const; + const CvMat* get_test_sample_idx() const; + void mix_train_and_test_idx(); + + const CvMat* get_var_idx(); + void chahge_var_idx( int vi, bool state ); // misspelled (saved for back compitability), + // use change_var_idx + void change_var_idx( int vi, bool state ); // state == true to set vi-variable as predictor + + const CvMat* get_var_types(); + int get_var_type( int var_idx ) const; + // following 2 methods enable to change vars type + // use these methods to assign CV_VAR_CATEGORICAL type for categorical variable + // with numerical labels; in the other cases var types are correctly determined automatically + void set_var_types( const char* str ); // str examples: + // "ord[0-17],cat[18]", "ord[0,2,4,10-12], cat[1,3,5-9,13,14]", + // "cat", "ord" (all vars are categorical/ordered) + void change_var_type( int var_idx, int type); // type in { CV_VAR_ORDERED, CV_VAR_CATEGORICAL } + + void set_delimiter( char ch ); + char get_delimiter() const; + + void set_miss_ch( char ch ); + char get_miss_ch() const; + + const std::map& get_class_labels_map() const; + +protected: + virtual void clear(); + + void str_to_flt_elem( const char* token, float& flt_elem, int& type); + void free_train_test_idx(); + + char delimiter; + char miss_ch; + //char flt_separator; + + CvMat* values; + CvMat* missing; + CvMat* var_types; + CvMat* var_idx_mask; + + CvMat* response_out; // header + CvMat* var_idx_out; // mat + CvMat* var_types_out; // mat + + int header_lines_number; + + int response_idx; + + int train_sample_count; + bool mix; + + int total_class_count; + std::map class_map; + + CvMat* train_sample_idx; + CvMat* test_sample_idx; + int* sample_idx; // data of train_sample_idx and test_sample_idx + + cv::RNG* rng; +}; + + +namespace cv +{ + +typedef CvStatModel StatModel; +typedef CvParamGrid ParamGrid; +typedef CvNormalBayesClassifier NormalBayesClassifier; +typedef CvKNearest KNearest; +typedef CvSVMParams SVMParams; +typedef CvSVMKernel SVMKernel; +typedef CvSVMSolver SVMSolver; +typedef CvSVM SVM; +typedef CvDTreeParams DTreeParams; +typedef CvMLData TrainData; +typedef CvDTree DecisionTree; +typedef CvForestTree ForestTree; +typedef CvRTParams RandomTreeParams; +typedef CvRTrees RandomTrees; +typedef CvERTreeTrainData ERTreeTRainData; +typedef CvForestERTree ERTree; +typedef CvERTrees ERTrees; +typedef CvBoostParams BoostParams; +typedef CvBoostTree BoostTree; +typedef CvBoost Boost; +typedef CvANN_MLP_TrainParams ANN_MLP_TrainParams; +typedef CvANN_MLP NeuralNet_MLP; +typedef CvGBTreesParams GradientBoostingTreeParams; +typedef CvGBTrees GradientBoostingTrees; + +template<> void DefaultDeleter::operator ()(CvDTreeSplit* obj) const; + +bool initModule_ml(void); +} + +#endif // __cplusplus +#endif // __OPENCV_ML_HPP__ + +/* End of file. */ diff --git a/apps/traincascade/old_ml_boost.cpp b/apps/traincascade/old_ml_boost.cpp new file mode 100644 index 0000000000000000000000000000000000000000..be4cd81f04bd113c37eeb5d182de0b9975274216 --- /dev/null +++ b/apps/traincascade/old_ml_boost.cpp @@ -0,0 +1,2162 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "old_ml_precomp.hpp" + +static inline double +log_ratio( double val ) +{ + const double eps = 1e-5; + + val = MAX( val, eps ); + val = MIN( val, 1. - eps ); + return log( val/(1. - val) ); +} + + +CvBoostParams::CvBoostParams() +{ + boost_type = CvBoost::REAL; + weak_count = 100; + weight_trim_rate = 0.95; + cv_folds = 0; + max_depth = 1; +} + + +CvBoostParams::CvBoostParams( int _boost_type, int _weak_count, + double _weight_trim_rate, int _max_depth, + bool _use_surrogates, const float* _priors ) +{ + boost_type = _boost_type; + weak_count = _weak_count; + weight_trim_rate = _weight_trim_rate; + split_criteria = CvBoost::DEFAULT; + cv_folds = 0; + max_depth = _max_depth; + use_surrogates = _use_surrogates; + priors = _priors; +} + + + +///////////////////////////////// CvBoostTree /////////////////////////////////// + +CvBoostTree::CvBoostTree() +{ + ensemble = 0; +} + + +CvBoostTree::~CvBoostTree() +{ + clear(); +} + + +void +CvBoostTree::clear() +{ + CvDTree::clear(); + ensemble = 0; +} + + +bool +CvBoostTree::train( CvDTreeTrainData* _train_data, + const CvMat* _subsample_idx, CvBoost* _ensemble ) +{ + clear(); + ensemble = _ensemble; + data = _train_data; + data->shared = true; + return do_train( _subsample_idx ); +} + + +bool +CvBoostTree::train( const CvMat*, int, const CvMat*, const CvMat*, + const CvMat*, const CvMat*, const CvMat*, CvDTreeParams ) +{ + assert(0); + return false; +} + + +bool +CvBoostTree::train( CvDTreeTrainData*, const CvMat* ) +{ + assert(0); + return false; +} + + +void +CvBoostTree::scale( double _scale ) +{ + CvDTreeNode* node = root; + + // traverse the tree and scale all the node values + for(;;) + { + CvDTreeNode* parent; + for(;;) + { + node->value *= _scale; + if( !node->left ) + break; + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + ; + + if( !parent ) + break; + + node = parent->right; + } +} + + +void +CvBoostTree::try_split_node( CvDTreeNode* node ) +{ + CvDTree::try_split_node( node ); + + if( !node->left ) + { + // if the node has not been split, + // store the responses for the corresponding training samples + double* weak_eval = ensemble->get_weak_response()->data.db; + cv::AutoBuffer inn_buf(node->sample_count); + const int* labels = data->get_cv_labels( node, (int*)inn_buf ); + int i, count = node->sample_count; + double value = node->value; + + for( i = 0; i < count; i++ ) + weak_eval[labels[i]] = value; + } +} + + +double +CvBoostTree::calc_node_dir( CvDTreeNode* node ) +{ + char* dir = (char*)data->direction->data.ptr; + const double* weights = ensemble->get_subtree_weights()->data.db; + int i, n = node->sample_count, vi = node->split->var_idx; + double L, R; + + assert( !node->split->inversed ); + + if( data->get_var_type(vi) >= 0 ) // split on categorical var + { + cv::AutoBuffer inn_buf(n); + const int* cat_labels = data->get_cat_var_data( node, vi, (int*)inn_buf ); + const int* subset = node->split->subset; + double sum = 0, sum_abs = 0; + + for( i = 0; i < n; i++ ) + { + int idx = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; + double w = weights[i]; + int d = idx >= 0 ? CV_DTREE_CAT_DIR(idx,subset) : 0; + sum += d*w; sum_abs += (d & 1)*w; + dir[i] = (char)d; + } + + R = (sum_abs + sum) * 0.5; + L = (sum_abs - sum) * 0.5; + } + else // split on ordered var + { + cv::AutoBuffer inn_buf(2*n*sizeof(int)+n*sizeof(float)); + float* values_buf = (float*)(uchar*)inn_buf; + int* sorted_indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = sorted_indices_buf + n; + const float* values = 0; + const int* sorted_indices = 0; + data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); + int split_point = node->split->ord.split_point; + int n1 = node->get_num_valid(vi); + + assert( 0 <= split_point && split_point < n1-1 ); + L = R = 0; + + for( i = 0; i <= split_point; i++ ) + { + int idx = sorted_indices[i]; + double w = weights[idx]; + dir[idx] = (char)-1; + L += w; + } + + for( ; i < n1; i++ ) + { + int idx = sorted_indices[i]; + double w = weights[idx]; + dir[idx] = (char)1; + R += w; + } + + for( ; i < n; i++ ) + dir[sorted_indices[i]] = (char)0; + } + + node->maxlr = MAX( L, R ); + return node->split->quality/(L + R); +} + + +CvDTreeSplit* +CvBoostTree::find_split_ord_class( CvDTreeNode* node, int vi, float init_quality, + CvDTreeSplit* _split, uchar* _ext_buf ) +{ + const float epsilon = FLT_EPSILON*2; + + const double* weights = ensemble->get_subtree_weights()->data.db; + int n = node->sample_count; + int n1 = node->get_num_valid(vi); + + cv::AutoBuffer inn_buf; + if( !_ext_buf ) + inn_buf.allocate(n*(3*sizeof(int)+sizeof(float))); + uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; + float* values_buf = (float*)ext_buf; + int* sorted_indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = sorted_indices_buf + n; + const float* values = 0; + const int* sorted_indices = 0; + data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); + int* responses_buf = sorted_indices_buf + n; + const int* responses = data->get_class_labels( node, responses_buf ); + const double* rcw0 = weights + n; + double lcw[2] = {0,0}, rcw[2]; + int i, best_i = -1; + double best_val = init_quality; + int boost_type = ensemble->get_params().boost_type; + int split_criteria = ensemble->get_params().split_criteria; + + rcw[0] = rcw0[0]; rcw[1] = rcw0[1]; + for( i = n1; i < n; i++ ) + { + int idx = sorted_indices[i]; + double w = weights[idx]; + rcw[responses[idx]] -= w; + } + + if( split_criteria != CvBoost::GINI && split_criteria != CvBoost::MISCLASS ) + split_criteria = boost_type == CvBoost::DISCRETE ? CvBoost::MISCLASS : CvBoost::GINI; + + if( split_criteria == CvBoost::GINI ) + { + double L = 0, R = rcw[0] + rcw[1]; + double lsum2 = 0, rsum2 = rcw[0]*rcw[0] + rcw[1]*rcw[1]; + + for( i = 0; i < n1 - 1; i++ ) + { + int idx = sorted_indices[i]; + double w = weights[idx], w2 = w*w; + double lv, rv; + idx = responses[idx]; + L += w; R -= w; + lv = lcw[idx]; rv = rcw[idx]; + lsum2 += 2*lv*w + w2; + rsum2 -= 2*rv*w - w2; + lcw[idx] = lv + w; rcw[idx] = rv - w; + + if( values[i] + epsilon < values[i+1] ) + { + double val = (lsum2*R + rsum2*L)/(L*R); + if( best_val < val ) + { + best_val = val; + best_i = i; + } + } + } + } + else + { + for( i = 0; i < n1 - 1; i++ ) + { + int idx = sorted_indices[i]; + double w = weights[idx]; + idx = responses[idx]; + lcw[idx] += w; + rcw[idx] -= w; + + if( values[i] + epsilon < values[i+1] ) + { + double val = lcw[0] + rcw[1], val2 = lcw[1] + rcw[0]; + val = MAX(val, val2); + if( best_val < val ) + { + best_val = val; + best_i = i; + } + } + } + } + + CvDTreeSplit* split = 0; + if( best_i >= 0 ) + { + split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); + split->var_idx = vi; + split->ord.c = (values[best_i] + values[best_i+1])*0.5f; + split->ord.split_point = best_i; + split->inversed = 0; + split->quality = (float)best_val; + } + return split; +} + +template +class LessThanPtr +{ +public: + bool operator()(T* a, T* b) const { return *a < *b; } +}; + +CvDTreeSplit* +CvBoostTree::find_split_cat_class( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) +{ + int ci = data->get_var_type(vi); + int n = node->sample_count; + int mi = data->cat_count->data.i[ci]; + + int base_size = (2*mi+3)*sizeof(double) + mi*sizeof(double*); + cv::AutoBuffer inn_buf((2*mi+3)*sizeof(double) + mi*sizeof(double*)); + if( !_ext_buf) + inn_buf.allocate( base_size + 2*n*sizeof(int) ); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; + + int* cat_labels_buf = (int*)ext_buf; + const int* cat_labels = data->get_cat_var_data(node, vi, cat_labels_buf); + int* responses_buf = cat_labels_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + double lcw[2]={0,0}, rcw[2]={0,0}; + + double* cjk = (double*)cv::alignPtr(base_buf,sizeof(double))+2; + const double* weights = ensemble->get_subtree_weights()->data.db; + double** dbl_ptr = (double**)(cjk + 2*mi); + int i, j, k, idx; + double L = 0, R; + double best_val = init_quality; + int best_subset = -1, subset_i; + int boost_type = ensemble->get_params().boost_type; + int split_criteria = ensemble->get_params().split_criteria; + + // init array of counters: + // c_{jk} - number of samples that have vi-th input variable = j and response = k. + for( j = -1; j < mi; j++ ) + cjk[j*2] = cjk[j*2+1] = 0; + + for( i = 0; i < n; i++ ) + { + double w = weights[i]; + j = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; + k = responses[i]; + cjk[j*2 + k] += w; + } + + for( j = 0; j < mi; j++ ) + { + rcw[0] += cjk[j*2]; + rcw[1] += cjk[j*2+1]; + dbl_ptr[j] = cjk + j*2 + 1; + } + + R = rcw[0] + rcw[1]; + + if( split_criteria != CvBoost::GINI && split_criteria != CvBoost::MISCLASS ) + split_criteria = boost_type == CvBoost::DISCRETE ? CvBoost::MISCLASS : CvBoost::GINI; + + // sort rows of c_jk by increasing c_j,1 + // (i.e. by the weight of samples in j-th category that belong to class 1) + std::sort(dbl_ptr, dbl_ptr + mi, LessThanPtr()); + + for( subset_i = 0; subset_i < mi-1; subset_i++ ) + { + idx = (int)(dbl_ptr[subset_i] - cjk)/2; + const double* crow = cjk + idx*2; + double w0 = crow[0], w1 = crow[1]; + double weight = w0 + w1; + + if( weight < FLT_EPSILON ) + continue; + + lcw[0] += w0; rcw[0] -= w0; + lcw[1] += w1; rcw[1] -= w1; + + if( split_criteria == CvBoost::GINI ) + { + double lsum2 = lcw[0]*lcw[0] + lcw[1]*lcw[1]; + double rsum2 = rcw[0]*rcw[0] + rcw[1]*rcw[1]; + + L += weight; + R -= weight; + + if( L > FLT_EPSILON && R > FLT_EPSILON ) + { + double val = (lsum2*R + rsum2*L)/(L*R); + if( best_val < val ) + { + best_val = val; + best_subset = subset_i; + } + } + } + else + { + double val = lcw[0] + rcw[1]; + double val2 = lcw[1] + rcw[0]; + + val = MAX(val, val2); + if( best_val < val ) + { + best_val = val; + best_subset = subset_i; + } + } + } + + CvDTreeSplit* split = 0; + if( best_subset >= 0 ) + { + split = _split ? _split : data->new_split_cat( 0, -1.0f); + split->var_idx = vi; + split->quality = (float)best_val; + memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); + for( i = 0; i <= best_subset; i++ ) + { + idx = (int)(dbl_ptr[i] - cjk) >> 1; + split->subset[idx >> 5] |= 1 << (idx & 31); + } + } + return split; +} + + +CvDTreeSplit* +CvBoostTree::find_split_ord_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) +{ + const float epsilon = FLT_EPSILON*2; + const double* weights = ensemble->get_subtree_weights()->data.db; + int n = node->sample_count; + int n1 = node->get_num_valid(vi); + + cv::AutoBuffer inn_buf; + if( !_ext_buf ) + inn_buf.allocate(2*n*(sizeof(int)+sizeof(float))); + uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; + + float* values_buf = (float*)ext_buf; + int* indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = indices_buf + n; + const float* values = 0; + const int* indices = 0; + data->get_ord_var_data( node, vi, values_buf, indices_buf, &values, &indices, sample_indices_buf ); + float* responses_buf = (float*)(indices_buf + n); + const float* responses = data->get_ord_responses( node, responses_buf, sample_indices_buf ); + + int i, best_i = -1; + double L = 0, R = weights[n]; + double best_val = init_quality, lsum = 0, rsum = node->value*R; + + // compensate for missing values + for( i = n1; i < n; i++ ) + { + int idx = indices[i]; + double w = weights[idx]; + rsum -= responses[idx]*w; + R -= w; + } + + // find the optimal split + for( i = 0; i < n1 - 1; i++ ) + { + int idx = indices[i]; + double w = weights[idx]; + double t = responses[idx]*w; + L += w; R -= w; + lsum += t; rsum -= t; + + if( values[i] + epsilon < values[i+1] ) + { + double val = (lsum*lsum*R + rsum*rsum*L)/(L*R); + if( best_val < val ) + { + best_val = val; + best_i = i; + } + } + } + + CvDTreeSplit* split = 0; + if( best_i >= 0 ) + { + split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); + split->var_idx = vi; + split->ord.c = (values[best_i] + values[best_i+1])*0.5f; + split->ord.split_point = best_i; + split->inversed = 0; + split->quality = (float)best_val; + } + return split; +} + + +CvDTreeSplit* +CvBoostTree::find_split_cat_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) +{ + const double* weights = ensemble->get_subtree_weights()->data.db; + int ci = data->get_var_type(vi); + int n = node->sample_count; + int mi = data->cat_count->data.i[ci]; + int base_size = (2*mi+3)*sizeof(double) + mi*sizeof(double*); + cv::AutoBuffer inn_buf(base_size); + if( !_ext_buf ) + inn_buf.allocate(base_size + n*(2*sizeof(int) + sizeof(float))); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; + + int* cat_labels_buf = (int*)ext_buf; + const int* cat_labels = data->get_cat_var_data(node, vi, cat_labels_buf); + float* responses_buf = (float*)(cat_labels_buf + n); + int* sample_indices_buf = (int*)(responses_buf + n); + const float* responses = data->get_ord_responses(node, responses_buf, sample_indices_buf); + + double* sum = (double*)cv::alignPtr(base_buf,sizeof(double)) + 1; + double* counts = sum + mi + 1; + double** sum_ptr = (double**)(counts + mi); + double L = 0, R = 0, best_val = init_quality, lsum = 0, rsum = 0; + int i, best_subset = -1, subset_i; + + for( i = -1; i < mi; i++ ) + sum[i] = counts[i] = 0; + + // calculate sum response and weight of each category of the input var + for( i = 0; i < n; i++ ) + { + int idx = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; + double w = weights[i]; + double s = sum[idx] + responses[i]*w; + double nc = counts[idx] + w; + sum[idx] = s; + counts[idx] = nc; + } + + // calculate average response in each category + for( i = 0; i < mi; i++ ) + { + R += counts[i]; + rsum += sum[i]; + sum[i] = fabs(counts[i]) > DBL_EPSILON ? sum[i]/counts[i] : 0; + sum_ptr[i] = sum + i; + } + + std::sort(sum_ptr, sum_ptr + mi, LessThanPtr()); + + // revert back to unnormalized sums + // (there should be a very little loss in accuracy) + for( i = 0; i < mi; i++ ) + sum[i] *= counts[i]; + + for( subset_i = 0; subset_i < mi-1; subset_i++ ) + { + int idx = (int)(sum_ptr[subset_i] - sum); + double ni = counts[idx]; + + if( ni > FLT_EPSILON ) + { + double s = sum[idx]; + lsum += s; L += ni; + rsum -= s; R -= ni; + + if( L > FLT_EPSILON && R > FLT_EPSILON ) + { + double val = (lsum*lsum*R + rsum*rsum*L)/(L*R); + if( best_val < val ) + { + best_val = val; + best_subset = subset_i; + } + } + } + } + + CvDTreeSplit* split = 0; + if( best_subset >= 0 ) + { + split = _split ? _split : data->new_split_cat( 0, -1.0f); + split->var_idx = vi; + split->quality = (float)best_val; + memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); + for( i = 0; i <= best_subset; i++ ) + { + int idx = (int)(sum_ptr[i] - sum); + split->subset[idx >> 5] |= 1 << (idx & 31); + } + } + return split; +} + + +CvDTreeSplit* +CvBoostTree::find_surrogate_split_ord( CvDTreeNode* node, int vi, uchar* _ext_buf ) +{ + const float epsilon = FLT_EPSILON*2; + int n = node->sample_count; + cv::AutoBuffer inn_buf; + if( !_ext_buf ) + inn_buf.allocate(n*(2*sizeof(int)+sizeof(float))); + uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; + float* values_buf = (float*)ext_buf; + int* indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = indices_buf + n; + const float* values = 0; + const int* indices = 0; + data->get_ord_var_data( node, vi, values_buf, indices_buf, &values, &indices, sample_indices_buf ); + + const double* weights = ensemble->get_subtree_weights()->data.db; + const char* dir = (char*)data->direction->data.ptr; + int n1 = node->get_num_valid(vi); + // LL - number of samples that both the primary and the surrogate splits send to the left + // LR - ... primary split sends to the left and the surrogate split sends to the right + // RL - ... primary split sends to the right and the surrogate split sends to the left + // RR - ... both send to the right + int i, best_i = -1, best_inversed = 0; + double best_val; + double LL = 0, RL = 0, LR, RR; + double worst_val = node->maxlr; + double sum = 0, sum_abs = 0; + best_val = worst_val; + + for( i = 0; i < n1; i++ ) + { + int idx = indices[i]; + double w = weights[idx]; + int d = dir[idx]; + sum += d*w; sum_abs += (d & 1)*w; + } + + // sum_abs = R + L; sum = R - L + RR = (sum_abs + sum)*0.5; + LR = (sum_abs - sum)*0.5; + + // initially all the samples are sent to the right by the surrogate split, + // LR of them are sent to the left by primary split, and RR - to the right. + // now iteratively compute LL, LR, RL and RR for every possible surrogate split value. + for( i = 0; i < n1 - 1; i++ ) + { + int idx = indices[i]; + double w = weights[idx]; + int d = dir[idx]; + + if( d < 0 ) + { + LL += w; LR -= w; + if( LL + RR > best_val && values[i] + epsilon < values[i+1] ) + { + best_val = LL + RR; + best_i = i; best_inversed = 0; + } + } + else if( d > 0 ) + { + RL += w; RR -= w; + if( RL + LR > best_val && values[i] + epsilon < values[i+1] ) + { + best_val = RL + LR; + best_i = i; best_inversed = 1; + } + } + } + + return best_i >= 0 && best_val > node->maxlr ? data->new_split_ord( vi, + (values[best_i] + values[best_i+1])*0.5f, best_i, + best_inversed, (float)best_val ) : 0; +} + + +CvDTreeSplit* +CvBoostTree::find_surrogate_split_cat( CvDTreeNode* node, int vi, uchar* _ext_buf ) +{ + const char* dir = (char*)data->direction->data.ptr; + const double* weights = ensemble->get_subtree_weights()->data.db; + int n = node->sample_count; + int i, mi = data->cat_count->data.i[data->get_var_type(vi)]; + + int base_size = (2*mi+3)*sizeof(double); + cv::AutoBuffer inn_buf(base_size); + if( !_ext_buf ) + inn_buf.allocate(base_size + n*sizeof(int)); + uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; + int* cat_labels_buf = (int*)ext_buf; + const int* cat_labels = data->get_cat_var_data(node, vi, cat_labels_buf); + + // LL - number of samples that both the primary and the surrogate splits send to the left + // LR - ... primary split sends to the left and the surrogate split sends to the right + // RL - ... primary split sends to the right and the surrogate split sends to the left + // RR - ... both send to the right + CvDTreeSplit* split = data->new_split_cat( vi, 0 ); + double best_val = 0; + double* lc = (double*)cv::alignPtr(cat_labels_buf + n, sizeof(double)) + 1; + double* rc = lc + mi + 1; + + for( i = -1; i < mi; i++ ) + lc[i] = rc[i] = 0; + + // 1. for each category calculate the weight of samples + // sent to the left (lc) and to the right (rc) by the primary split + for( i = 0; i < n; i++ ) + { + int idx = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; + double w = weights[i]; + int d = dir[i]; + double sum = lc[idx] + d*w; + double sum_abs = rc[idx] + (d & 1)*w; + lc[idx] = sum; rc[idx] = sum_abs; + } + + for( i = 0; i < mi; i++ ) + { + double sum = lc[i]; + double sum_abs = rc[i]; + lc[i] = (sum_abs - sum) * 0.5; + rc[i] = (sum_abs + sum) * 0.5; + } + + // 2. now form the split. + // in each category send all the samples to the same direction as majority + for( i = 0; i < mi; i++ ) + { + double lval = lc[i], rval = rc[i]; + if( lval > rval ) + { + split->subset[i >> 5] |= 1 << (i & 31); + best_val += lval; + } + else + best_val += rval; + } + + split->quality = (float)best_val; + if( split->quality <= node->maxlr ) + cvSetRemoveByPtr( data->split_heap, split ), split = 0; + + return split; +} + + +void +CvBoostTree::calc_node_value( CvDTreeNode* node ) +{ + int i, n = node->sample_count; + const double* weights = ensemble->get_weights()->data.db; + cv::AutoBuffer inn_buf(n*(sizeof(int) + ( data->is_classifier ? sizeof(int) : sizeof(int) + sizeof(float)))); + int* labels_buf = (int*)(uchar*)inn_buf; + const int* labels = data->get_cv_labels(node, labels_buf); + double* subtree_weights = ensemble->get_subtree_weights()->data.db; + double rcw[2] = {0,0}; + int boost_type = ensemble->get_params().boost_type; + + if( data->is_classifier ) + { + int* _responses_buf = labels_buf + n; + const int* _responses = data->get_class_labels(node, _responses_buf); + int m = data->get_num_classes(); + int* cls_count = data->counts->data.i; + for( int k = 0; k < m; k++ ) + cls_count[k] = 0; + + for( i = 0; i < n; i++ ) + { + int idx = labels[i]; + double w = weights[idx]; + int r = _responses[i]; + rcw[r] += w; + cls_count[r]++; + subtree_weights[i] = w; + } + + node->class_idx = rcw[1] > rcw[0]; + + if( boost_type == CvBoost::DISCRETE ) + { + // ignore cat_map for responses, and use {-1,1}, + // as the whole ensemble response is computes as sign(sum_i(weak_response_i) + node->value = node->class_idx*2 - 1; + } + else + { + double p = rcw[1]/(rcw[0] + rcw[1]); + assert( boost_type == CvBoost::REAL ); + + // store log-ratio of the probability + node->value = 0.5*log_ratio(p); + } + } + else + { + // in case of regression tree: + // * node value is 1/n*sum_i(Y_i), where Y_i is i-th response, + // n is the number of samples in the node. + // * node risk is the sum of squared errors: sum_i((Y_i - )^2) + double sum = 0, sum2 = 0, iw; + float* values_buf = (float*)(labels_buf + n); + int* sample_indices_buf = (int*)(values_buf + n); + const float* values = data->get_ord_responses(node, values_buf, sample_indices_buf); + + for( i = 0; i < n; i++ ) + { + int idx = labels[i]; + double w = weights[idx]/*priors[values[i] > 0]*/; + double t = values[i]; + rcw[0] += w; + subtree_weights[i] = w; + sum += t*w; + sum2 += t*t*w; + } + + iw = 1./rcw[0]; + node->value = sum*iw; + node->node_risk = sum2 - (sum*iw)*sum; + + // renormalize the risk, as in try_split_node the unweighted formula + // sqrt(risk)/n is used, rather than sqrt(risk)/sum(weights_i) + node->node_risk *= n*iw*n*iw; + } + + // store summary weights + subtree_weights[n] = rcw[0]; + subtree_weights[n+1] = rcw[1]; +} + + +void CvBoostTree::read( CvFileStorage* fs, CvFileNode* fnode, CvBoost* _ensemble, CvDTreeTrainData* _data ) +{ + CvDTree::read( fs, fnode, _data ); + ensemble = _ensemble; +} + +void CvBoostTree::read( CvFileStorage*, CvFileNode* ) +{ + assert(0); +} + +void CvBoostTree::read( CvFileStorage* _fs, CvFileNode* _node, + CvDTreeTrainData* _data ) +{ + CvDTree::read( _fs, _node, _data ); +} + + +/////////////////////////////////// CvBoost ///////////////////////////////////// + +CvBoost::CvBoost() +{ + data = 0; + weak = 0; + default_model_name = "my_boost_tree"; + + active_vars = active_vars_abs = orig_response = sum_response = weak_eval = + subsample_mask = weights = subtree_weights = 0; + have_active_cat_vars = have_subsample = false; + + clear(); +} + + +void CvBoost::prune( CvSlice slice ) +{ + if( weak && weak->total > 0 ) + { + CvSeqReader reader; + int i, count = cvSliceLength( slice, weak ); + + cvStartReadSeq( weak, &reader ); + cvSetSeqReaderPos( &reader, slice.start_index ); + + for( i = 0; i < count; i++ ) + { + CvBoostTree* w; + CV_READ_SEQ_ELEM( w, reader ); + delete w; + } + + cvSeqRemoveSlice( weak, slice ); + } +} + + +void CvBoost::clear() +{ + if( weak ) + { + prune( CV_WHOLE_SEQ ); + cvReleaseMemStorage( &weak->storage ); + } + if( data ) + delete data; + weak = 0; + data = 0; + cvReleaseMat( &active_vars ); + cvReleaseMat( &active_vars_abs ); + cvReleaseMat( &orig_response ); + cvReleaseMat( &sum_response ); + cvReleaseMat( &weak_eval ); + cvReleaseMat( &subsample_mask ); + cvReleaseMat( &weights ); + cvReleaseMat( &subtree_weights ); + + have_subsample = false; +} + + +CvBoost::~CvBoost() +{ + clear(); +} + + +CvBoost::CvBoost( const CvMat* _train_data, int _tflag, + const CvMat* _responses, const CvMat* _var_idx, + const CvMat* _sample_idx, const CvMat* _var_type, + const CvMat* _missing_mask, CvBoostParams _params ) +{ + weak = 0; + data = 0; + default_model_name = "my_boost_tree"; + + active_vars = active_vars_abs = orig_response = sum_response = weak_eval = + subsample_mask = weights = subtree_weights = 0; + + train( _train_data, _tflag, _responses, _var_idx, _sample_idx, + _var_type, _missing_mask, _params ); +} + + +bool +CvBoost::set_params( const CvBoostParams& _params ) +{ + bool ok = false; + + CV_FUNCNAME( "CvBoost::set_params" ); + + __BEGIN__; + + params = _params; + if( params.boost_type != DISCRETE && params.boost_type != REAL && + params.boost_type != LOGIT && params.boost_type != GENTLE ) + CV_ERROR( CV_StsBadArg, "Unknown/unsupported boosting type" ); + + params.weak_count = MAX( params.weak_count, 1 ); + params.weight_trim_rate = MAX( params.weight_trim_rate, 0. ); + params.weight_trim_rate = MIN( params.weight_trim_rate, 1. ); + if( params.weight_trim_rate < FLT_EPSILON ) + params.weight_trim_rate = 1.f; + + if( params.boost_type == DISCRETE && + params.split_criteria != GINI && params.split_criteria != MISCLASS ) + params.split_criteria = MISCLASS; + if( params.boost_type == REAL && + params.split_criteria != GINI && params.split_criteria != MISCLASS ) + params.split_criteria = GINI; + if( (params.boost_type == LOGIT || params.boost_type == GENTLE) && + params.split_criteria != SQERR ) + params.split_criteria = SQERR; + + ok = true; + + __END__; + + return ok; +} + + +bool +CvBoost::train( const CvMat* _train_data, int _tflag, + const CvMat* _responses, const CvMat* _var_idx, + const CvMat* _sample_idx, const CvMat* _var_type, + const CvMat* _missing_mask, + CvBoostParams _params, bool _update ) +{ + bool ok = false; + CvMemStorage* storage = 0; + + CV_FUNCNAME( "CvBoost::train" ); + + __BEGIN__; + + int i; + + set_params( _params ); + + cvReleaseMat( &active_vars ); + cvReleaseMat( &active_vars_abs ); + + if( !_update || !data ) + { + clear(); + data = new CvDTreeTrainData( _train_data, _tflag, _responses, _var_idx, + _sample_idx, _var_type, _missing_mask, _params, true, true ); + + if( data->get_num_classes() != 2 ) + CV_ERROR( CV_StsNotImplemented, + "Boosted trees can only be used for 2-class classification." ); + CV_CALL( storage = cvCreateMemStorage() ); + weak = cvCreateSeq( 0, sizeof(CvSeq), sizeof(CvBoostTree*), storage ); + storage = 0; + } + else + { + data->set_data( _train_data, _tflag, _responses, _var_idx, + _sample_idx, _var_type, _missing_mask, _params, true, true, true ); + } + + if ( (_params.boost_type == LOGIT) || (_params.boost_type == GENTLE) ) + data->do_responses_copy(); + + update_weights( 0 ); + + for( i = 0; i < params.weak_count; i++ ) + { + CvBoostTree* tree = new CvBoostTree; + if( !tree->train( data, subsample_mask, this ) ) + { + delete tree; + break; + } + //cvCheckArr( get_weak_response()); + cvSeqPush( weak, &tree ); + update_weights( tree ); + trim_weights(); + if( cvCountNonZero(subsample_mask) == 0 ) + break; + } + + if(weak->total > 0) + { + get_active_vars(); // recompute active_vars* maps and condensed_idx's in the splits. + data->is_classifier = true; + data->free_train_data(); + ok = true; + } + else + clear(); + + __END__; + + return ok; +} + +bool CvBoost::train( CvMLData* _data, + CvBoostParams _params, + bool update ) +{ + bool result = false; + + CV_FUNCNAME( "CvBoost::train" ); + + __BEGIN__; + + const CvMat* values = _data->get_values(); + const CvMat* response = _data->get_responses(); + const CvMat* missing = _data->get_missing(); + const CvMat* var_types = _data->get_var_types(); + const CvMat* train_sidx = _data->get_train_sample_idx(); + const CvMat* var_idx = _data->get_var_idx(); + + CV_CALL( result = train( values, CV_ROW_SAMPLE, response, var_idx, + train_sidx, var_types, missing, _params, update ) ); + + __END__; + + return result; +} + +void CvBoost::initialize_weights(double (&p)[2]) +{ + p[0] = 1.; + p[1] = 1.; +} + +void +CvBoost::update_weights( CvBoostTree* tree ) +{ + CV_FUNCNAME( "CvBoost::update_weights" ); + + __BEGIN__; + + int i, n = data->sample_count; + double sumw = 0.; + int step = 0; + float* fdata = 0; + int *sample_idx_buf; + const int* sample_idx = 0; + cv::AutoBuffer inn_buf; + size_t _buf_size = (params.boost_type == LOGIT) || (params.boost_type == GENTLE) ? (size_t)(data->sample_count)*sizeof(int) : 0; + if( !tree ) + _buf_size += n*sizeof(int); + else + { + if( have_subsample ) + _buf_size += data->get_length_subbuf()*(sizeof(float)+sizeof(uchar)); + } + inn_buf.allocate(_buf_size); + uchar* cur_buf_pos = (uchar*)inn_buf; + + if ( (params.boost_type == LOGIT) || (params.boost_type == GENTLE) ) + { + step = CV_IS_MAT_CONT(data->responses_copy->type) ? + 1 : data->responses_copy->step / CV_ELEM_SIZE(data->responses_copy->type); + fdata = data->responses_copy->data.fl; + sample_idx_buf = (int*)cur_buf_pos; + cur_buf_pos = (uchar*)(sample_idx_buf + data->sample_count); + sample_idx = data->get_sample_indices( data->data_root, sample_idx_buf ); + } + CvMat* dtree_data_buf = data->buf; + size_t length_buf_row = data->get_length_subbuf(); + if( !tree ) // before training the first tree, initialize weights and other parameters + { + int* class_labels_buf = (int*)cur_buf_pos; + cur_buf_pos = (uchar*)(class_labels_buf + n); + const int* class_labels = data->get_class_labels(data->data_root, class_labels_buf); + // in case of logitboost and gentle adaboost each weak tree is a regression tree, + // so we need to convert class labels to floating-point values + + double w0 = 1./ n; + double p[2] = { 1., 1. }; + initialize_weights(p); + + cvReleaseMat( &orig_response ); + cvReleaseMat( &sum_response ); + cvReleaseMat( &weak_eval ); + cvReleaseMat( &subsample_mask ); + cvReleaseMat( &weights ); + cvReleaseMat( &subtree_weights ); + + CV_CALL( orig_response = cvCreateMat( 1, n, CV_32S )); + CV_CALL( weak_eval = cvCreateMat( 1, n, CV_64F )); + CV_CALL( subsample_mask = cvCreateMat( 1, n, CV_8U )); + CV_CALL( weights = cvCreateMat( 1, n, CV_64F )); + CV_CALL( subtree_weights = cvCreateMat( 1, n + 2, CV_64F )); + + if( data->have_priors ) + { + // compute weight scale for each class from their prior probabilities + int c1 = 0; + for( i = 0; i < n; i++ ) + c1 += class_labels[i]; + p[0] = data->priors->data.db[0]*(c1 < n ? 1./(n - c1) : 0.); + p[1] = data->priors->data.db[1]*(c1 > 0 ? 1./c1 : 0.); + p[0] /= p[0] + p[1]; + p[1] = 1. - p[0]; + } + + if (data->is_buf_16u) + { + unsigned short* labels = (unsigned short*)(dtree_data_buf->data.s + data->data_root->buf_idx*length_buf_row + + data->data_root->offset + (data->work_var_count-1)*data->sample_count); + for( i = 0; i < n; i++ ) + { + // save original categorical responses {0,1}, convert them to {-1,1} + orig_response->data.i[i] = class_labels[i]*2 - 1; + // make all the samples active at start. + // later, in trim_weights() deactivate/reactive again some, if need + subsample_mask->data.ptr[i] = (uchar)1; + // make all the initial weights the same. + weights->data.db[i] = w0*p[class_labels[i]]; + // set the labels to find (from within weak tree learning proc) + // the particular sample weight, and where to store the response. + labels[i] = (unsigned short)i; + } + } + else + { + int* labels = dtree_data_buf->data.i + data->data_root->buf_idx*length_buf_row + + data->data_root->offset + (data->work_var_count-1)*data->sample_count; + + for( i = 0; i < n; i++ ) + { + // save original categorical responses {0,1}, convert them to {-1,1} + orig_response->data.i[i] = class_labels[i]*2 - 1; + // make all the samples active at start. + // later, in trim_weights() deactivate/reactive again some, if need + subsample_mask->data.ptr[i] = (uchar)1; + // make all the initial weights the same. + weights->data.db[i] = w0*p[class_labels[i]]; + // set the labels to find (from within weak tree learning proc) + // the particular sample weight, and where to store the response. + labels[i] = i; + } + } + + if( params.boost_type == LOGIT ) + { + CV_CALL( sum_response = cvCreateMat( 1, n, CV_64F )); + + for( i = 0; i < n; i++ ) + { + sum_response->data.db[i] = 0; + fdata[sample_idx[i]*step] = orig_response->data.i[i] > 0 ? 2.f : -2.f; + } + + // in case of logitboost each weak tree is a regression tree. + // the target function values are recalculated for each of the trees + data->is_classifier = false; + } + else if( params.boost_type == GENTLE ) + { + for( i = 0; i < n; i++ ) + fdata[sample_idx[i]*step] = (float)orig_response->data.i[i]; + + data->is_classifier = false; + } + } + else + { + // at this moment, for all the samples that participated in the training of the most + // recent weak classifier we know the responses. For other samples we need to compute them + if( have_subsample ) + { + float* values = (float*)cur_buf_pos; + cur_buf_pos = (uchar*)(values + data->get_length_subbuf()); + uchar* missing = cur_buf_pos; + cur_buf_pos = missing + data->get_length_subbuf() * (size_t)CV_ELEM_SIZE(data->buf->type); + + CvMat _sample, _mask; + + // invert the subsample mask + cvXorS( subsample_mask, cvScalar(1.), subsample_mask ); + data->get_vectors( subsample_mask, values, missing, 0 ); + + _sample = cvMat( 1, data->var_count, CV_32F ); + _mask = cvMat( 1, data->var_count, CV_8U ); + + // run tree through all the non-processed samples + for( i = 0; i < n; i++ ) + if( subsample_mask->data.ptr[i] ) + { + _sample.data.fl = values; + _mask.data.ptr = missing; + values += _sample.cols; + missing += _mask.cols; + weak_eval->data.db[i] = tree->predict( &_sample, &_mask, true )->value; + } + } + + // now update weights and other parameters for each type of boosting + if( params.boost_type == DISCRETE ) + { + // Discrete AdaBoost: + // weak_eval[i] (=f(x_i)) is in {-1,1} + // err = sum(w_i*(f(x_i) != y_i))/sum(w_i) + // C = log((1-err)/err) + // w_i *= exp(C*(f(x_i) != y_i)) + + double C, err = 0.; + double scale[] = { 1., 0. }; + + for( i = 0; i < n; i++ ) + { + double w = weights->data.db[i]; + sumw += w; + err += w*(weak_eval->data.db[i] != orig_response->data.i[i]); + } + + if( sumw != 0 ) + err /= sumw; + C = err = -log_ratio( err ); + scale[1] = exp(err); + + sumw = 0; + for( i = 0; i < n; i++ ) + { + double w = weights->data.db[i]* + scale[weak_eval->data.db[i] != orig_response->data.i[i]]; + sumw += w; + weights->data.db[i] = w; + } + + tree->scale( C ); + } + else if( params.boost_type == REAL ) + { + // Real AdaBoost: + // weak_eval[i] = f(x_i) = 0.5*log(p(x_i)/(1-p(x_i))), p(x_i)=P(y=1|x_i) + // w_i *= exp(-y_i*f(x_i)) + + for( i = 0; i < n; i++ ) + weak_eval->data.db[i] *= -orig_response->data.i[i]; + + cvExp( weak_eval, weak_eval ); + + for( i = 0; i < n; i++ ) + { + double w = weights->data.db[i]*weak_eval->data.db[i]; + sumw += w; + weights->data.db[i] = w; + } + } + else if( params.boost_type == LOGIT ) + { + // LogitBoost: + // weak_eval[i] = f(x_i) in [-z_max,z_max] + // sum_response = F(x_i). + // F(x_i) += 0.5*f(x_i) + // p(x_i) = exp(F(x_i))/(exp(F(x_i)) + exp(-F(x_i))=1/(1+exp(-2*F(x_i))) + // reuse weak_eval: weak_eval[i] <- p(x_i) + // w_i = p(x_i)*1(1 - p(x_i)) + // z_i = ((y_i+1)/2 - p(x_i))/(p(x_i)*(1 - p(x_i))) + // store z_i to the data->data_root as the new target responses + + const double lb_weight_thresh = FLT_EPSILON; + const double lb_z_max = 10.; + /*float* responses_buf = data->get_resp_float_buf(); + const float* responses = 0; + data->get_ord_responses(data->data_root, responses_buf, &responses);*/ + + /*if( weak->total == 7 ) + putchar('*');*/ + + for( i = 0; i < n; i++ ) + { + double s = sum_response->data.db[i] + 0.5*weak_eval->data.db[i]; + sum_response->data.db[i] = s; + weak_eval->data.db[i] = -2*s; + } + + cvExp( weak_eval, weak_eval ); + + for( i = 0; i < n; i++ ) + { + double p = 1./(1. + weak_eval->data.db[i]); + double w = p*(1 - p), z; + w = MAX( w, lb_weight_thresh ); + weights->data.db[i] = w; + sumw += w; + if( orig_response->data.i[i] > 0 ) + { + z = 1./p; + fdata[sample_idx[i]*step] = (float)MIN(z, lb_z_max); + } + else + { + z = 1./(1-p); + fdata[sample_idx[i]*step] = (float)-MIN(z, lb_z_max); + } + } + } + else + { + // Gentle AdaBoost: + // weak_eval[i] = f(x_i) in [-1,1] + // w_i *= exp(-y_i*f(x_i)) + assert( params.boost_type == GENTLE ); + + for( i = 0; i < n; i++ ) + weak_eval->data.db[i] *= -orig_response->data.i[i]; + + cvExp( weak_eval, weak_eval ); + + for( i = 0; i < n; i++ ) + { + double w = weights->data.db[i] * weak_eval->data.db[i]; + weights->data.db[i] = w; + sumw += w; + } + } + } + + // renormalize weights + if( sumw > FLT_EPSILON ) + { + sumw = 1./sumw; + for( i = 0; i < n; ++i ) + weights->data.db[i] *= sumw; + } + + __END__; +} + + +void +CvBoost::trim_weights() +{ + //CV_FUNCNAME( "CvBoost::trim_weights" ); + + __BEGIN__; + + int i, count = data->sample_count, nz_count = 0; + double sum, threshold; + + if( params.weight_trim_rate <= 0. || params.weight_trim_rate >= 1. ) + EXIT; + + // use weak_eval as temporary buffer for sorted weights + cvCopy( weights, weak_eval ); + + std::sort(weak_eval->data.db, weak_eval->data.db + count); + + // as weight trimming occurs immediately after updating the weights, + // where they are renormalized, we assume that the weight sum = 1. + sum = 1. - params.weight_trim_rate; + + for( i = 0; i < count; i++ ) + { + double w = weak_eval->data.db[i]; + if( sum <= 0 ) + break; + sum -= w; + } + + threshold = i < count ? weak_eval->data.db[i] : DBL_MAX; + + for( i = 0; i < count; i++ ) + { + double w = weights->data.db[i]; + int f = w >= threshold; + subsample_mask->data.ptr[i] = (uchar)f; + nz_count += f; + } + + have_subsample = nz_count < count; + + __END__; +} + + +const CvMat* +CvBoost::get_active_vars( bool absolute_idx ) +{ + CvMat* mask = 0; + CvMat* inv_map = 0; + CvMat* result = 0; + + CV_FUNCNAME( "CvBoost::get_active_vars" ); + + __BEGIN__; + + if( !weak ) + CV_ERROR( CV_StsError, "The boosted tree ensemble has not been trained yet" ); + + if( !active_vars || !active_vars_abs ) + { + CvSeqReader reader; + int i, j, nactive_vars; + CvBoostTree* wtree; + const CvDTreeNode* node; + + assert(!active_vars && !active_vars_abs); + mask = cvCreateMat( 1, data->var_count, CV_8U ); + inv_map = cvCreateMat( 1, data->var_count, CV_32S ); + cvZero( mask ); + cvSet( inv_map, cvScalar(-1) ); + + // first pass: compute the mask of used variables + cvStartReadSeq( weak, &reader ); + for( i = 0; i < weak->total; i++ ) + { + CV_READ_SEQ_ELEM(wtree, reader); + + node = wtree->get_root(); + assert( node != 0 ); + for(;;) + { + const CvDTreeNode* parent; + for(;;) + { + CvDTreeSplit* split = node->split; + for( ; split != 0; split = split->next ) + mask->data.ptr[split->var_idx] = 1; + if( !node->left ) + break; + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + ; + + if( !parent ) + break; + + node = parent->right; + } + } + + nactive_vars = cvCountNonZero(mask); + + //if ( nactive_vars > 0 ) + { + active_vars = cvCreateMat( 1, nactive_vars, CV_32S ); + active_vars_abs = cvCreateMat( 1, nactive_vars, CV_32S ); + + have_active_cat_vars = false; + + for( i = j = 0; i < data->var_count; i++ ) + { + if( mask->data.ptr[i] ) + { + active_vars->data.i[j] = i; + active_vars_abs->data.i[j] = data->var_idx ? data->var_idx->data.i[i] : i; + inv_map->data.i[i] = j; + if( data->var_type->data.i[i] >= 0 ) + have_active_cat_vars = true; + j++; + } + } + + + // second pass: now compute the condensed indices + cvStartReadSeq( weak, &reader ); + for( i = 0; i < weak->total; i++ ) + { + CV_READ_SEQ_ELEM(wtree, reader); + node = wtree->get_root(); + for(;;) + { + const CvDTreeNode* parent; + for(;;) + { + CvDTreeSplit* split = node->split; + for( ; split != 0; split = split->next ) + { + split->condensed_idx = inv_map->data.i[split->var_idx]; + assert( split->condensed_idx >= 0 ); + } + + if( !node->left ) + break; + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + ; + + if( !parent ) + break; + + node = parent->right; + } + } + } + } + + result = absolute_idx ? active_vars_abs : active_vars; + + __END__; + + cvReleaseMat( &mask ); + cvReleaseMat( &inv_map ); + + return result; +} + + +float +CvBoost::predict( const CvMat* _sample, const CvMat* _missing, + CvMat* weak_responses, CvSlice slice, + bool raw_mode, bool return_sum ) const +{ + float value = -FLT_MAX; + + CvSeqReader reader; + double sum = 0; + int wstep = 0; + const float* sample_data; + + if( !weak ) + CV_Error( CV_StsError, "The boosted tree ensemble has not been trained yet" ); + + if( !CV_IS_MAT(_sample) || CV_MAT_TYPE(_sample->type) != CV_32FC1 || + (_sample->cols != 1 && _sample->rows != 1) || + (_sample->cols + _sample->rows - 1 != data->var_all && !raw_mode) || + (active_vars && _sample->cols + _sample->rows - 1 != active_vars->cols && raw_mode) ) + CV_Error( CV_StsBadArg, + "the input sample must be 1d floating-point vector with the same " + "number of elements as the total number of variables or " + "as the number of variables used for training" ); + + if( _missing ) + { + if( !CV_IS_MAT(_missing) || !CV_IS_MASK_ARR(_missing) || + !CV_ARE_SIZES_EQ(_missing, _sample) ) + CV_Error( CV_StsBadArg, + "the missing data mask must be 8-bit vector of the same size as input sample" ); + } + + int i, weak_count = cvSliceLength( slice, weak ); + if( weak_count >= weak->total ) + { + weak_count = weak->total; + slice.start_index = 0; + } + + if( weak_responses ) + { + if( !CV_IS_MAT(weak_responses) || + CV_MAT_TYPE(weak_responses->type) != CV_32FC1 || + (weak_responses->cols != 1 && weak_responses->rows != 1) || + weak_responses->cols + weak_responses->rows - 1 != weak_count ) + CV_Error( CV_StsBadArg, + "The output matrix of weak classifier responses must be valid " + "floating-point vector of the same number of components as the length of input slice" ); + wstep = CV_IS_MAT_CONT(weak_responses->type) ? 1 : weak_responses->step/sizeof(float); + } + + int var_count = active_vars->cols; + const int* vtype = data->var_type->data.i; + const int* cmap = data->cat_map->data.i; + const int* cofs = data->cat_ofs->data.i; + + cv::Mat sample = cv::cvarrToMat(_sample); + cv::Mat missing; + if(!_missing) + missing = cv::cvarrToMat(_missing); + + // if need, preprocess the input vector + if( !raw_mode ) + { + int sstep, mstep = 0; + const float* src_sample; + const uchar* src_mask = 0; + float* dst_sample; + uchar* dst_mask; + const int* vidx = active_vars->data.i; + const int* vidx_abs = active_vars_abs->data.i; + bool have_mask = _missing != 0; + + sample = cv::Mat(1, var_count, CV_32FC1); + missing = cv::Mat(1, var_count, CV_8UC1); + + dst_sample = sample.ptr(); + dst_mask = missing.ptr(); + + src_sample = _sample->data.fl; + sstep = CV_IS_MAT_CONT(_sample->type) ? 1 : _sample->step/sizeof(src_sample[0]); + + if( _missing ) + { + src_mask = _missing->data.ptr; + mstep = CV_IS_MAT_CONT(_missing->type) ? 1 : _missing->step; + } + + for( i = 0; i < var_count; i++ ) + { + int idx = vidx[i], idx_abs = vidx_abs[i]; + float val = src_sample[idx_abs*sstep]; + int ci = vtype[idx]; + uchar m = src_mask ? src_mask[idx_abs*mstep] : (uchar)0; + + if( ci >= 0 ) + { + int a = cofs[ci], b = (ci+1 >= data->cat_ofs->cols) ? data->cat_map->cols : cofs[ci+1], + c = a; + int ival = cvRound(val); + if ( (ival != val) && (!m) ) + CV_Error( CV_StsBadArg, + "one of input categorical variable is not an integer" ); + + while( a < b ) + { + c = (a + b) >> 1; + if( ival < cmap[c] ) + b = c; + else if( ival > cmap[c] ) + a = c+1; + else + break; + } + + if( c < 0 || ival != cmap[c] ) + { + m = 1; + have_mask = true; + } + else + { + val = (float)(c - cofs[ci]); + } + } + + dst_sample[i] = val; + dst_mask[i] = m; + } + + if( !have_mask ) + missing.release(); + } + else + { + if( !CV_IS_MAT_CONT(_sample->type & (_missing ? _missing->type : -1)) ) + CV_Error( CV_StsBadArg, "In raw mode the input vectors must be continuous" ); + } + + cvStartReadSeq( weak, &reader ); + cvSetSeqReaderPos( &reader, slice.start_index ); + + sample_data = sample.ptr(); + + if( !have_active_cat_vars && missing.empty() && !weak_responses ) + { + for( i = 0; i < weak_count; i++ ) + { + CvBoostTree* wtree; + const CvDTreeNode* node; + CV_READ_SEQ_ELEM( wtree, reader ); + + node = wtree->get_root(); + while( node->left ) + { + CvDTreeSplit* split = node->split; + int vi = split->condensed_idx; + float val = sample_data[vi]; + int dir = val <= split->ord.c ? -1 : 1; + if( split->inversed ) + dir = -dir; + node = dir < 0 ? node->left : node->right; + } + sum += node->value; + } + } + else + { + const int* avars = active_vars->data.i; + const uchar* m = !missing.empty() ? missing.ptr() : 0; + + // full-featured version + for( i = 0; i < weak_count; i++ ) + { + CvBoostTree* wtree; + const CvDTreeNode* node; + CV_READ_SEQ_ELEM( wtree, reader ); + + node = wtree->get_root(); + while( node->left ) + { + const CvDTreeSplit* split = node->split; + int dir = 0; + for( ; !dir && split != 0; split = split->next ) + { + int vi = split->condensed_idx; + int ci = vtype[avars[vi]]; + float val = sample_data[vi]; + if( m && m[vi] ) + continue; + if( ci < 0 ) // ordered + dir = val <= split->ord.c ? -1 : 1; + else // categorical + { + int c = cvRound(val); + dir = CV_DTREE_CAT_DIR(c, split->subset); + } + if( split->inversed ) + dir = -dir; + } + + if( !dir ) + { + int diff = node->right->sample_count - node->left->sample_count; + dir = diff < 0 ? -1 : 1; + } + node = dir < 0 ? node->left : node->right; + } + if( weak_responses ) + weak_responses->data.fl[i*wstep] = (float)node->value; + sum += node->value; + } + } + + if( return_sum ) + value = (float)sum; + else + { + int cls_idx = sum >= 0; + if( raw_mode ) + value = (float)cls_idx; + else + value = (float)cmap[cofs[vtype[data->var_count]] + cls_idx]; + } + + return value; +} + +float CvBoost::calc_error( CvMLData* _data, int type, std::vector *resp ) +{ + float err = 0; + const CvMat* values = _data->get_values(); + const CvMat* response = _data->get_responses(); + const CvMat* missing = _data->get_missing(); + const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); + const CvMat* var_types = _data->get_var_types(); + int* sidx = sample_idx ? sample_idx->data.i : 0; + int r_step = CV_IS_MAT_CONT(response->type) ? + 1 : response->step / CV_ELEM_SIZE(response->type); + bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; + int sample_count = sample_idx ? sample_idx->cols : 0; + sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; + float* pred_resp = 0; + if( resp && (sample_count > 0) ) + { + resp->resize( sample_count ); + pred_resp = &((*resp)[0]); + } + if ( is_classifier ) + { + for( int i = 0; i < sample_count; i++ ) + { + CvMat sample, miss; + int si = sidx ? sidx[i] : i; + cvGetRow( values, &sample, si ); + if( missing ) + cvGetRow( missing, &miss, si ); + float r = (float)predict( &sample, missing ? &miss : 0 ); + if( pred_resp ) + pred_resp[i] = r; + int d = fabs((double)r - response->data.fl[si*r_step]) <= FLT_EPSILON ? 0 : 1; + err += d; + } + err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; + } + else + { + for( int i = 0; i < sample_count; i++ ) + { + CvMat sample, miss; + int si = sidx ? sidx[i] : i; + cvGetRow( values, &sample, si ); + if( missing ) + cvGetRow( missing, &miss, si ); + float r = (float)predict( &sample, missing ? &miss : 0 ); + if( pred_resp ) + pred_resp[i] = r; + float d = r - response->data.fl[si*r_step]; + err += d*d; + } + err = sample_count ? err / (float)sample_count : -FLT_MAX; + } + return err; +} + +void CvBoost::write_params( CvFileStorage* fs ) const +{ + const char* boost_type_str = + params.boost_type == DISCRETE ? "DiscreteAdaboost" : + params.boost_type == REAL ? "RealAdaboost" : + params.boost_type == LOGIT ? "LogitBoost" : + params.boost_type == GENTLE ? "GentleAdaboost" : 0; + + const char* split_crit_str = + params.split_criteria == DEFAULT ? "Default" : + params.split_criteria == GINI ? "Gini" : + params.boost_type == MISCLASS ? "Misclassification" : + params.boost_type == SQERR ? "SquaredErr" : 0; + + if( boost_type_str ) + cvWriteString( fs, "boosting_type", boost_type_str ); + else + cvWriteInt( fs, "boosting_type", params.boost_type ); + + if( split_crit_str ) + cvWriteString( fs, "splitting_criteria", split_crit_str ); + else + cvWriteInt( fs, "splitting_criteria", params.split_criteria ); + + cvWriteInt( fs, "ntrees", weak->total ); + cvWriteReal( fs, "weight_trimming_rate", params.weight_trim_rate ); + + data->write_params( fs ); +} + + +void CvBoost::read_params( CvFileStorage* fs, CvFileNode* fnode ) +{ + CV_FUNCNAME( "CvBoost::read_params" ); + + __BEGIN__; + + CvFileNode* temp; + + if( !fnode || !CV_NODE_IS_MAP(fnode->tag) ) + return; + + data = new CvDTreeTrainData(); + CV_CALL( data->read_params(fs, fnode)); + data->shared = true; + + params.max_depth = data->params.max_depth; + params.min_sample_count = data->params.min_sample_count; + params.max_categories = data->params.max_categories; + params.priors = data->params.priors; + params.regression_accuracy = data->params.regression_accuracy; + params.use_surrogates = data->params.use_surrogates; + + temp = cvGetFileNodeByName( fs, fnode, "boosting_type" ); + if( !temp ) + return; + + if( temp && CV_NODE_IS_STRING(temp->tag) ) + { + const char* boost_type_str = cvReadString( temp, "" ); + params.boost_type = strcmp( boost_type_str, "DiscreteAdaboost" ) == 0 ? DISCRETE : + strcmp( boost_type_str, "RealAdaboost" ) == 0 ? REAL : + strcmp( boost_type_str, "LogitBoost" ) == 0 ? LOGIT : + strcmp( boost_type_str, "GentleAdaboost" ) == 0 ? GENTLE : -1; + } + else + params.boost_type = cvReadInt( temp, -1 ); + + if( params.boost_type < DISCRETE || params.boost_type > GENTLE ) + CV_ERROR( CV_StsBadArg, "Unknown boosting type" ); + + temp = cvGetFileNodeByName( fs, fnode, "splitting_criteria" ); + if( temp && CV_NODE_IS_STRING(temp->tag) ) + { + const char* split_crit_str = cvReadString( temp, "" ); + params.split_criteria = strcmp( split_crit_str, "Default" ) == 0 ? DEFAULT : + strcmp( split_crit_str, "Gini" ) == 0 ? GINI : + strcmp( split_crit_str, "Misclassification" ) == 0 ? MISCLASS : + strcmp( split_crit_str, "SquaredErr" ) == 0 ? SQERR : -1; + } + else + params.split_criteria = cvReadInt( temp, -1 ); + + if( params.split_criteria < DEFAULT || params.boost_type > SQERR ) + CV_ERROR( CV_StsBadArg, "Unknown boosting type" ); + + params.weak_count = cvReadIntByName( fs, fnode, "ntrees" ); + params.weight_trim_rate = cvReadRealByName( fs, fnode, "weight_trimming_rate", 0. ); + + __END__; +} + + + +void +CvBoost::read( CvFileStorage* fs, CvFileNode* node ) +{ + CV_FUNCNAME( "CvBoost::read" ); + + __BEGIN__; + + CvSeqReader reader; + CvFileNode* trees_fnode; + CvMemStorage* storage; + int i, ntrees; + + clear(); + read_params( fs, node ); + + if( !data ) + EXIT; + + trees_fnode = cvGetFileNodeByName( fs, node, "trees" ); + if( !trees_fnode || !CV_NODE_IS_SEQ(trees_fnode->tag) ) + CV_ERROR( CV_StsParseError, " tag is missing" ); + + cvStartReadSeq( trees_fnode->data.seq, &reader ); + ntrees = trees_fnode->data.seq->total; + + if( ntrees != params.weak_count ) + CV_ERROR( CV_StsUnmatchedSizes, + "The number of trees stored does not match tag value" ); + + CV_CALL( storage = cvCreateMemStorage() ); + weak = cvCreateSeq( 0, sizeof(CvSeq), sizeof(CvBoostTree*), storage ); + + for( i = 0; i < ntrees; i++ ) + { + CvBoostTree* tree = new CvBoostTree(); + CV_CALL(tree->read( fs, (CvFileNode*)reader.ptr, this, data )); + CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + cvSeqPush( weak, &tree ); + } + get_active_vars(); + + __END__; +} + + +void +CvBoost::write( CvFileStorage* fs, const char* name ) const +{ + CV_FUNCNAME( "CvBoost::write" ); + + __BEGIN__; + + CvSeqReader reader; + int i; + + cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_BOOSTING ); + + if( !weak ) + CV_ERROR( CV_StsBadArg, "The classifier has not been trained yet" ); + + write_params( fs ); + cvStartWriteStruct( fs, "trees", CV_NODE_SEQ ); + + cvStartReadSeq( weak, &reader ); + + for( i = 0; i < weak->total; i++ ) + { + CvBoostTree* tree; + CV_READ_SEQ_ELEM( tree, reader ); + cvStartWriteStruct( fs, 0, CV_NODE_MAP ); + tree->write( fs ); + cvEndWriteStruct( fs ); + } + + cvEndWriteStruct( fs ); + cvEndWriteStruct( fs ); + + __END__; +} + + +CvMat* +CvBoost::get_weights() +{ + return weights; +} + + +CvMat* +CvBoost::get_subtree_weights() +{ + return subtree_weights; +} + + +CvMat* +CvBoost::get_weak_response() +{ + return weak_eval; +} + + +const CvBoostParams& +CvBoost::get_params() const +{ + return params; +} + +CvSeq* CvBoost::get_weak_predictors() +{ + return weak; +} + +const CvDTreeTrainData* CvBoost::get_data() const +{ + return data; +} + +using namespace cv; + +CvBoost::CvBoost( const Mat& _train_data, int _tflag, + const Mat& _responses, const Mat& _var_idx, + const Mat& _sample_idx, const Mat& _var_type, + const Mat& _missing_mask, + CvBoostParams _params ) +{ + weak = 0; + data = 0; + default_model_name = "my_boost_tree"; + active_vars = active_vars_abs = orig_response = sum_response = weak_eval = + subsample_mask = weights = subtree_weights = 0; + + train( _train_data, _tflag, _responses, _var_idx, _sample_idx, + _var_type, _missing_mask, _params ); +} + + +bool +CvBoost::train( const Mat& _train_data, int _tflag, + const Mat& _responses, const Mat& _var_idx, + const Mat& _sample_idx, const Mat& _var_type, + const Mat& _missing_mask, + CvBoostParams _params, bool _update ) +{ + train_data_hdr = _train_data; + train_data_mat = _train_data; + responses_hdr = _responses; + responses_mat = _responses; + + CvMat vidx = _var_idx, sidx = _sample_idx, vtype = _var_type, mmask = _missing_mask; + + return train(&train_data_hdr, _tflag, &responses_hdr, vidx.data.ptr ? &vidx : 0, + sidx.data.ptr ? &sidx : 0, vtype.data.ptr ? &vtype : 0, + mmask.data.ptr ? &mmask : 0, _params, _update); +} + +float +CvBoost::predict( const Mat& _sample, const Mat& _missing, + const Range& slice, bool raw_mode, bool return_sum ) const +{ + CvMat sample = _sample, mmask = _missing; + /*if( weak_responses ) + { + int weak_count = cvSliceLength( slice, weak ); + if( weak_count >= weak->total ) + { + weak_count = weak->total; + slice.start_index = 0; + } + + if( !(weak_responses->data && weak_responses->type() == CV_32FC1 && + (weak_responses->cols == 1 || weak_responses->rows == 1) && + weak_responses->cols + weak_responses->rows - 1 == weak_count) ) + weak_responses->create(weak_count, 1, CV_32FC1); + pwr = &(wr = *weak_responses); + }*/ + return predict(&sample, _missing.empty() ? 0 : &mmask, 0, + slice == Range::all() ? CV_WHOLE_SEQ : cvSlice(slice.start, slice.end), + raw_mode, return_sum); +} + +/* End of file. */ diff --git a/apps/traincascade/old_ml_data.cpp b/apps/traincascade/old_ml_data.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d221dcbf0f7be2763d32401bc7d58496ef9611a6 --- /dev/null +++ b/apps/traincascade/old_ml_data.cpp @@ -0,0 +1,792 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "old_ml_precomp.hpp" +#include + +#define MISS_VAL FLT_MAX +#define CV_VAR_MISS 0 + +CvTrainTestSplit::CvTrainTestSplit() +{ + train_sample_part_mode = CV_COUNT; + train_sample_part.count = -1; + mix = false; +} + +CvTrainTestSplit::CvTrainTestSplit( int _train_sample_count, bool _mix ) +{ + train_sample_part_mode = CV_COUNT; + train_sample_part.count = _train_sample_count; + mix = _mix; +} + +CvTrainTestSplit::CvTrainTestSplit( float _train_sample_portion, bool _mix ) +{ + train_sample_part_mode = CV_PORTION; + train_sample_part.portion = _train_sample_portion; + mix = _mix; +} + +//////////////// + +CvMLData::CvMLData() +{ + values = missing = var_types = var_idx_mask = response_out = var_idx_out = var_types_out = 0; + train_sample_idx = test_sample_idx = 0; + header_lines_number = 0; + sample_idx = 0; + response_idx = -1; + + train_sample_count = -1; + + delimiter = ','; + miss_ch = '?'; + //flt_separator = '.'; + + rng = &cv::theRNG(); +} + +CvMLData::~CvMLData() +{ + clear(); +} + +void CvMLData::free_train_test_idx() +{ + cvReleaseMat( &train_sample_idx ); + cvReleaseMat( &test_sample_idx ); + sample_idx = 0; +} + +void CvMLData::clear() +{ + class_map.clear(); + + cvReleaseMat( &values ); + cvReleaseMat( &missing ); + cvReleaseMat( &var_types ); + cvReleaseMat( &var_idx_mask ); + + cvReleaseMat( &response_out ); + cvReleaseMat( &var_idx_out ); + cvReleaseMat( &var_types_out ); + + free_train_test_idx(); + + total_class_count = 0; + + response_idx = -1; + + train_sample_count = -1; +} + + +void CvMLData::set_header_lines_number( int idx ) +{ + header_lines_number = std::max(0, idx); +} + +int CvMLData::get_header_lines_number() const +{ + return header_lines_number; +} + +static char *fgets_chomp(char *str, int n, FILE *stream) +{ + char *head = fgets(str, n, stream); + if( head ) + { + for(char *tail = head + strlen(head) - 1; tail >= head; --tail) + { + if( *tail != '\r' && *tail != '\n' ) + break; + *tail = '\0'; + } + } + return head; +} + + +int CvMLData::read_csv(const char* filename) +{ + const int M = 1000000; + const char str_delimiter[3] = { ' ', delimiter, '\0' }; + FILE* file = 0; + CvMemStorage* storage; + CvSeq* seq; + char *ptr; + float* el_ptr; + CvSeqReader reader; + int cols_count = 0; + uchar *var_types_ptr = 0; + + clear(); + + file = fopen( filename, "rt" ); + + if( !file ) + return -1; + + std::vector _buf(M); + char* buf = &_buf[0]; + + // skip header lines + for( int i = 0; i < header_lines_number; i++ ) + { + if( fgets( buf, M, file ) == 0 ) + { + fclose(file); + return -1; + } + } + + // read the first data line and determine the number of variables + if( !fgets_chomp( buf, M, file )) + { + fclose(file); + return -1; + } + + ptr = buf; + while( *ptr == ' ' ) + ptr++; + for( ; *ptr != '\0'; ) + { + if(*ptr == delimiter || *ptr == ' ') + { + cols_count++; + ptr++; + while( *ptr == ' ' ) ptr++; + } + else + ptr++; + } + + cols_count++; + + if ( cols_count == 0) + { + fclose(file); + return -1; + } + + // create temporary memory storage to store the whole database + el_ptr = new float[cols_count]; + storage = cvCreateMemStorage(); + seq = cvCreateSeq( 0, sizeof(*seq), cols_count*sizeof(float), storage ); + + var_types = cvCreateMat( 1, cols_count, CV_8U ); + cvZero( var_types ); + var_types_ptr = var_types->data.ptr; + + for(;;) + { + char *token = NULL; + int type; + token = strtok(buf, str_delimiter); + if (!token) + break; + for (int i = 0; i < cols_count-1; i++) + { + str_to_flt_elem( token, el_ptr[i], type); + var_types_ptr[i] |= type; + token = strtok(NULL, str_delimiter); + if (!token) + { + fclose(file); + delete [] el_ptr; + return -1; + } + } + str_to_flt_elem( token, el_ptr[cols_count-1], type); + var_types_ptr[cols_count-1] |= type; + cvSeqPush( seq, el_ptr ); + if( !fgets_chomp( buf, M, file ) ) + break; + } + fclose(file); + + values = cvCreateMat( seq->total, cols_count, CV_32FC1 ); + missing = cvCreateMat( seq->total, cols_count, CV_8U ); + var_idx_mask = cvCreateMat( 1, values->cols, CV_8UC1 ); + cvSet( var_idx_mask, cvRealScalar(1) ); + train_sample_count = seq->total; + + cvStartReadSeq( seq, &reader ); + for(int i = 0; i < seq->total; i++ ) + { + const float* sdata = (float*)reader.ptr; + float* ddata = values->data.fl + cols_count*i; + uchar* dm = missing->data.ptr + cols_count*i; + + for( int j = 0; j < cols_count; j++ ) + { + ddata[j] = sdata[j]; + dm[j] = ( fabs( MISS_VAL - sdata[j] ) <= FLT_EPSILON ); + } + CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); + } + + if ( cvNorm( missing, 0, CV_L1 ) <= FLT_EPSILON ) + cvReleaseMat( &missing ); + + cvReleaseMemStorage( &storage ); + delete []el_ptr; + return 0; +} + +const CvMat* CvMLData::get_values() const +{ + return values; +} + +const CvMat* CvMLData::get_missing() const +{ + CV_FUNCNAME( "CvMLData::get_missing" ); + __BEGIN__; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + __END__; + + return missing; +} + +const std::map& CvMLData::get_class_labels_map() const +{ + return class_map; +} + +void CvMLData::str_to_flt_elem( const char* token, float& flt_elem, int& type) +{ + + char* stopstring = NULL; + flt_elem = (float)strtod( token, &stopstring ); + assert( stopstring ); + type = CV_VAR_ORDERED; + if ( *stopstring == miss_ch && strlen(stopstring) == 1 ) // missed value + { + flt_elem = MISS_VAL; + type = CV_VAR_MISS; + } + else + { + if ( (*stopstring != 0) && (*stopstring != '\n') && (strcmp(stopstring, "\r\n") != 0) ) // class label + { + int idx = class_map[token]; + if ( idx == 0) + { + total_class_count++; + idx = total_class_count; + class_map[token] = idx; + } + flt_elem = (float)idx; + type = CV_VAR_CATEGORICAL; + } + } +} + +void CvMLData::set_delimiter(char ch) +{ + CV_FUNCNAME( "CvMLData::set_delimited" ); + __BEGIN__; + + if (ch == miss_ch /*|| ch == flt_separator*/) + CV_ERROR(CV_StsBadArg, "delimited, miss_character and flt_separator must be different"); + + delimiter = ch; + + __END__; +} + +char CvMLData::get_delimiter() const +{ + return delimiter; +} + +void CvMLData::set_miss_ch(char ch) +{ + CV_FUNCNAME( "CvMLData::set_miss_ch" ); + __BEGIN__; + + if (ch == delimiter/* || ch == flt_separator*/) + CV_ERROR(CV_StsBadArg, "delimited, miss_character and flt_separator must be different"); + + miss_ch = ch; + + __END__; +} + +char CvMLData::get_miss_ch() const +{ + return miss_ch; +} + +void CvMLData::set_response_idx( int idx ) +{ + CV_FUNCNAME( "CvMLData::set_response_idx" ); + __BEGIN__; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + if ( idx >= values->cols) + CV_ERROR( CV_StsBadArg, "idx value is not correct" ); + + if ( response_idx >= 0 ) + chahge_var_idx( response_idx, true ); + if ( idx >= 0 ) + chahge_var_idx( idx, false ); + response_idx = idx; + + __END__; +} + +int CvMLData::get_response_idx() const +{ + CV_FUNCNAME( "CvMLData::get_response_idx" ); + __BEGIN__; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + __END__; + return response_idx; +} + +void CvMLData::change_var_type( int var_idx, int type ) +{ + CV_FUNCNAME( "CvMLData::change_var_type" ); + __BEGIN__; + + int var_count = 0; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + var_count = values->cols; + + if ( var_idx < 0 || var_idx >= var_count) + CV_ERROR( CV_StsBadArg, "var_idx is not correct" ); + + if ( type != CV_VAR_ORDERED && type != CV_VAR_CATEGORICAL) + CV_ERROR( CV_StsBadArg, "type is not correct" ); + + assert( var_types ); + if ( var_types->data.ptr[var_idx] == CV_VAR_CATEGORICAL && type == CV_VAR_ORDERED) + CV_ERROR( CV_StsBadArg, "it`s impossible to assign CV_VAR_ORDERED type to categorical variable" ); + var_types->data.ptr[var_idx] = (uchar)type; + + __END__; + + return; +} + +void CvMLData::set_var_types( const char* str ) +{ + CV_FUNCNAME( "CvMLData::set_var_types" ); + __BEGIN__; + + const char* ord = 0, *cat = 0; + int var_count = 0, set_var_type_count = 0; + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + var_count = values->cols; + + assert( var_types ); + + ord = strstr( str, "ord" ); + cat = strstr( str, "cat" ); + if ( !ord && !cat ) + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + + if ( !ord && strlen(cat) == 3 ) // str == "cat" + { + cvSet( var_types, cvScalarAll(CV_VAR_CATEGORICAL) ); + return; + } + + if ( !cat && strlen(ord) == 3 ) // str == "ord" + { + cvSet( var_types, cvScalarAll(CV_VAR_ORDERED) ); + return; + } + + if ( ord ) // parse ord str + { + char* stopstring = NULL; + if ( ord[3] != '[') + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + + ord += 4; // pass "ord[" + do + { + int b1 = (int)strtod( ord, &stopstring ); + if ( *stopstring == 0 || (*stopstring != ',' && *stopstring != ']' && *stopstring != '-') ) + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + ord = stopstring + 1; + if ( (stopstring[0] == ',') || (stopstring[0] == ']')) + { + if ( var_types->data.ptr[b1] == CV_VAR_CATEGORICAL) + CV_ERROR( CV_StsBadArg, "it`s impossible to assign CV_VAR_ORDERED type to categorical variable" ); + var_types->data.ptr[b1] = CV_VAR_ORDERED; + set_var_type_count++; + } + else + { + if ( stopstring[0] == '-') + { + int b2 = (int)strtod( ord, &stopstring); + if ( (*stopstring == 0) || (*stopstring != ',' && *stopstring != ']') ) + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + ord = stopstring + 1; + for (int i = b1; i <= b2; i++) + { + if ( var_types->data.ptr[i] == CV_VAR_CATEGORICAL) + CV_ERROR( CV_StsBadArg, "it`s impossible to assign CV_VAR_ORDERED type to categorical variable" ); + var_types->data.ptr[i] = CV_VAR_ORDERED; + } + set_var_type_count += b2 - b1 + 1; + } + else + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + + } + } + while (*stopstring != ']'); + + if ( stopstring[1] != '\0' && stopstring[1] != ',') + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + } + + if ( cat ) // parse cat str + { + char* stopstring = NULL; + if ( cat[3] != '[') + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + + cat += 4; // pass "cat[" + do + { + int b1 = (int)strtod( cat, &stopstring ); + if ( *stopstring == 0 || (*stopstring != ',' && *stopstring != ']' && *stopstring != '-') ) + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + cat = stopstring + 1; + if ( (stopstring[0] == ',') || (stopstring[0] == ']')) + { + var_types->data.ptr[b1] = CV_VAR_CATEGORICAL; + set_var_type_count++; + } + else + { + if ( stopstring[0] == '-') + { + int b2 = (int)strtod( cat, &stopstring); + if ( (*stopstring == 0) || (*stopstring != ',' && *stopstring != ']') ) + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + cat = stopstring + 1; + for (int i = b1; i <= b2; i++) + var_types->data.ptr[i] = CV_VAR_CATEGORICAL; + set_var_type_count += b2 - b1 + 1; + } + else + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + + } + } + while (*stopstring != ']'); + + if ( stopstring[1] != '\0' && stopstring[1] != ',') + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + } + + if (set_var_type_count != var_count) + CV_ERROR( CV_StsBadArg, "types string is not correct" ); + + __END__; +} + +const CvMat* CvMLData::get_var_types() +{ + CV_FUNCNAME( "CvMLData::get_var_types" ); + __BEGIN__; + + uchar *var_types_out_ptr = 0; + int avcount, vt_size; + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + assert( var_idx_mask ); + + avcount = cvFloor( cvNorm( var_idx_mask, 0, CV_L1 ) ); + vt_size = avcount + (response_idx >= 0); + + if ( avcount == values->cols || (avcount == values->cols-1 && response_idx == values->cols-1) ) + return var_types; + + if ( !var_types_out || ( var_types_out && var_types_out->cols != vt_size ) ) + { + cvReleaseMat( &var_types_out ); + var_types_out = cvCreateMat( 1, vt_size, CV_8UC1 ); + } + + var_types_out_ptr = var_types_out->data.ptr; + for( int i = 0; i < var_types->cols; i++) + { + if (i == response_idx || !var_idx_mask->data.ptr[i]) continue; + *var_types_out_ptr = var_types->data.ptr[i]; + var_types_out_ptr++; + } + if ( response_idx >= 0 ) + *var_types_out_ptr = var_types->data.ptr[response_idx]; + + __END__; + + return var_types_out; +} + +int CvMLData::get_var_type( int var_idx ) const +{ + return var_types->data.ptr[var_idx]; +} + +const CvMat* CvMLData::get_responses() +{ + CV_FUNCNAME( "CvMLData::get_responses_ptr" ); + __BEGIN__; + + int var_count = 0; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + var_count = values->cols; + + if ( response_idx < 0 || response_idx >= var_count ) + return 0; + if ( !response_out ) + response_out = cvCreateMatHeader( values->rows, 1, CV_32FC1 ); + else + cvInitMatHeader( response_out, values->rows, 1, CV_32FC1); + cvGetCol( values, response_out, response_idx ); + + __END__; + + return response_out; +} + +void CvMLData::set_train_test_split( const CvTrainTestSplit * spl) +{ + CV_FUNCNAME( "CvMLData::set_division" ); + __BEGIN__; + + int sample_count = 0; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + sample_count = values->rows; + + float train_sample_portion; + + if (spl->train_sample_part_mode == CV_COUNT) + { + train_sample_count = spl->train_sample_part.count; + if (train_sample_count > sample_count) + CV_ERROR( CV_StsBadArg, "train samples count is not correct" ); + train_sample_count = train_sample_count<=0 ? sample_count : train_sample_count; + } + else // dtype.train_sample_part_mode == CV_PORTION + { + train_sample_portion = spl->train_sample_part.portion; + if ( train_sample_portion > 1) + CV_ERROR( CV_StsBadArg, "train samples count is not correct" ); + train_sample_portion = train_sample_portion <= FLT_EPSILON || + 1 - train_sample_portion <= FLT_EPSILON ? 1 : train_sample_portion; + train_sample_count = std::max(1, cvFloor( train_sample_portion * sample_count )); + } + + if ( train_sample_count == sample_count ) + { + free_train_test_idx(); + return; + } + + if ( train_sample_idx && train_sample_idx->cols != train_sample_count ) + free_train_test_idx(); + + if ( !sample_idx) + { + int test_sample_count = sample_count- train_sample_count; + sample_idx = (int*)cvAlloc( sample_count * sizeof(sample_idx[0]) ); + for (int i = 0; i < sample_count; i++ ) + sample_idx[i] = i; + train_sample_idx = cvCreateMatHeader( 1, train_sample_count, CV_32SC1 ); + *train_sample_idx = cvMat( 1, train_sample_count, CV_32SC1, &sample_idx[0] ); + + CV_Assert(test_sample_count > 0); + test_sample_idx = cvCreateMatHeader( 1, test_sample_count, CV_32SC1 ); + *test_sample_idx = cvMat( 1, test_sample_count, CV_32SC1, &sample_idx[train_sample_count] ); + } + + mix = spl->mix; + if ( mix ) + mix_train_and_test_idx(); + + __END__; +} + +const CvMat* CvMLData::get_train_sample_idx() const +{ + CV_FUNCNAME( "CvMLData::get_train_sample_idx" ); + __BEGIN__; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + __END__; + + return train_sample_idx; +} + +const CvMat* CvMLData::get_test_sample_idx() const +{ + CV_FUNCNAME( "CvMLData::get_test_sample_idx" ); + __BEGIN__; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + __END__; + + return test_sample_idx; +} + +void CvMLData::mix_train_and_test_idx() +{ + CV_FUNCNAME( "CvMLData::mix_train_and_test_idx" ); + __BEGIN__; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + __END__; + + if ( !sample_idx) + return; + + if ( train_sample_count > 0 && train_sample_count < values->rows ) + { + int n = values->rows; + for (int i = 0; i < n; i++) + { + int a = (*rng)(n); + int b = (*rng)(n); + int t; + CV_SWAP( sample_idx[a], sample_idx[b], t ); + } + } +} + +const CvMat* CvMLData::get_var_idx() +{ + CV_FUNCNAME( "CvMLData::get_var_idx" ); + __BEGIN__; + + int avcount = 0; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + assert( var_idx_mask ); + + avcount = cvFloor( cvNorm( var_idx_mask, 0, CV_L1 ) ); + int* vidx; + + if ( avcount == values->cols ) + return 0; + + if ( !var_idx_out || ( var_idx_out && var_idx_out->cols != avcount ) ) + { + cvReleaseMat( &var_idx_out ); + var_idx_out = cvCreateMat( 1, avcount, CV_32SC1); + if ( response_idx >=0 ) + var_idx_mask->data.ptr[response_idx] = 0; + } + + vidx = var_idx_out->data.i; + + for(int i = 0; i < var_idx_mask->cols; i++) + if ( var_idx_mask->data.ptr[i] ) + { + *vidx = i; + vidx++; + } + + __END__; + + return var_idx_out; +} + +void CvMLData::chahge_var_idx( int vi, bool state ) +{ + change_var_idx( vi, state ); +} + +void CvMLData::change_var_idx( int vi, bool state ) +{ + CV_FUNCNAME( "CvMLData::change_var_idx" ); + __BEGIN__; + + int var_count = 0; + + if ( !values ) + CV_ERROR( CV_StsInternal, "data is empty" ); + + var_count = values->cols; + + if ( vi < 0 || vi >= var_count) + CV_ERROR( CV_StsBadArg, "variable index is not correct" ); + + assert( var_idx_mask ); + var_idx_mask->data.ptr[vi] = state; + + __END__; +} + +/* End of file. */ diff --git a/apps/traincascade/old_ml_inner_functions.cpp b/apps/traincascade/old_ml_inner_functions.cpp new file mode 100644 index 0000000000000000000000000000000000000000..10b43f93fe3a34efdca92e89434643ae97838b3d --- /dev/null +++ b/apps/traincascade/old_ml_inner_functions.cpp @@ -0,0 +1,1879 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "old_ml_precomp.hpp" + + +CvStatModel::CvStatModel() +{ + default_model_name = "my_stat_model"; +} + + +CvStatModel::~CvStatModel() +{ + clear(); +} + + +void CvStatModel::clear() +{ +} + + +void CvStatModel::save( const char* filename, const char* name ) const +{ + CvFileStorage* fs = 0; + + CV_FUNCNAME( "CvStatModel::save" ); + + __BEGIN__; + + CV_CALL( fs = cvOpenFileStorage( filename, 0, CV_STORAGE_WRITE )); + if( !fs ) + CV_ERROR( CV_StsError, "Could not open the file storage. Check the path and permissions" ); + + write( fs, name ? name : default_model_name ); + + __END__; + + cvReleaseFileStorage( &fs ); +} + + +void CvStatModel::load( const char* filename, const char* name ) +{ + CvFileStorage* fs = 0; + + CV_FUNCNAME( "CvStatModel::load" ); + + __BEGIN__; + + CvFileNode* model_node = 0; + + CV_CALL( fs = cvOpenFileStorage( filename, 0, CV_STORAGE_READ )); + if( !fs ) + EXIT; + + if( name ) + model_node = cvGetFileNodeByName( fs, 0, name ); + else + { + CvFileNode* root = cvGetRootFileNode( fs ); + if( root->data.seq->total > 0 ) + model_node = (CvFileNode*)cvGetSeqElem( root->data.seq, 0 ); + } + + read( fs, model_node ); + + __END__; + + cvReleaseFileStorage( &fs ); +} + + +void CvStatModel::write( CvFileStorage*, const char* ) const +{ + OPENCV_ERROR( CV_StsNotImplemented, "CvStatModel::write", "" ); +} + + +void CvStatModel::read( CvFileStorage*, CvFileNode* ) +{ + OPENCV_ERROR( CV_StsNotImplemented, "CvStatModel::read", "" ); +} + + +/* Calculates upper triangular matrix S, where A is a symmetrical matrix A=S'*S */ +static void cvChol( CvMat* A, CvMat* S ) +{ + int dim = A->rows; + + int i, j, k; + float sum; + + for( i = 0; i < dim; i++ ) + { + for( j = 0; j < i; j++ ) + CV_MAT_ELEM(*S, float, i, j) = 0; + + sum = 0; + for( k = 0; k < i; k++ ) + sum += CV_MAT_ELEM(*S, float, k, i) * CV_MAT_ELEM(*S, float, k, i); + + CV_MAT_ELEM(*S, float, i, i) = (float)sqrt(CV_MAT_ELEM(*A, float, i, i) - sum); + + for( j = i + 1; j < dim; j++ ) + { + sum = 0; + for( k = 0; k < i; k++ ) + sum += CV_MAT_ELEM(*S, float, k, i) * CV_MAT_ELEM(*S, float, k, j); + + CV_MAT_ELEM(*S, float, i, j) = + (CV_MAT_ELEM(*A, float, i, j) - sum) / CV_MAT_ELEM(*S, float, i, i); + + } + } +} + +/* Generates from multivariate normal distribution, where - is an + average row vector, - symmetric covariation matrix */ +CV_IMPL void cvRandMVNormal( CvMat* mean, CvMat* cov, CvMat* sample, CvRNG* rng ) +{ + int dim = sample->cols; + int amount = sample->rows; + + CvRNG state = rng ? *rng : cvRNG( cvGetTickCount() ); + cvRandArr(&state, sample, CV_RAND_NORMAL, cvScalarAll(0), cvScalarAll(1) ); + + CvMat* utmat = cvCreateMat(dim, dim, sample->type); + CvMat* vect = cvCreateMatHeader(1, dim, sample->type); + + cvChol(cov, utmat); + + int i; + for( i = 0; i < amount; i++ ) + { + cvGetRow(sample, vect, i); + cvMatMulAdd(vect, utmat, mean, vect); + } + + cvReleaseMat(&vect); + cvReleaseMat(&utmat); +} + + +/* Generates of points from a discrete variate xi, + where Pr{xi = k} == probs[k], 0 < k < len - 1. */ +static void cvRandSeries( float probs[], int len, int sample[], int amount ) +{ + CvMat* univals = cvCreateMat(1, amount, CV_32FC1); + float* knots = (float*)cvAlloc( len * sizeof(float) ); + + int i, j; + + CvRNG state = cvRNG(-1); + cvRandArr(&state, univals, CV_RAND_UNI, cvScalarAll(0), cvScalarAll(1) ); + + knots[0] = probs[0]; + for( i = 1; i < len; i++ ) + knots[i] = knots[i - 1] + probs[i]; + + for( i = 0; i < amount; i++ ) + for( j = 0; j < len; j++ ) + { + if ( CV_MAT_ELEM(*univals, float, 0, i) <= knots[j] ) + { + sample[i] = j; + break; + } + } + + cvFree(&knots); +} + +/* Generates from gaussian mixture distribution */ +CV_IMPL void cvRandGaussMixture( CvMat* means[], + CvMat* covs[], + float weights[], + int clsnum, + CvMat* sample, + CvMat* sampClasses ) +{ + int dim = sample->cols; + int amount = sample->rows; + + int i, clss; + + int* sample_clsnum = (int*)cvAlloc( amount * sizeof(int) ); + CvMat** utmats = (CvMat**)cvAlloc( clsnum * sizeof(CvMat*) ); + CvMat* vect = cvCreateMatHeader(1, dim, CV_32FC1); + + CvMat* classes; + if( sampClasses ) + classes = sampClasses; + else + classes = cvCreateMat(1, amount, CV_32FC1); + + CvRNG state = cvRNG(-1); + cvRandArr(&state, sample, CV_RAND_NORMAL, cvScalarAll(0), cvScalarAll(1)); + + cvRandSeries(weights, clsnum, sample_clsnum, amount); + + for( i = 0; i < clsnum; i++ ) + { + utmats[i] = cvCreateMat(dim, dim, CV_32FC1); + cvChol(covs[i], utmats[i]); + } + + for( i = 0; i < amount; i++ ) + { + CV_MAT_ELEM(*classes, float, 0, i) = (float)sample_clsnum[i]; + cvGetRow(sample, vect, i); + clss = sample_clsnum[i]; + cvMatMulAdd(vect, utmats[clss], means[clss], vect); + } + + if( !sampClasses ) + cvReleaseMat(&classes); + for( i = 0; i < clsnum; i++ ) + cvReleaseMat(&utmats[i]); + cvFree(&utmats); + cvFree(&sample_clsnum); + cvReleaseMat(&vect); +} + + +CvMat* icvGenerateRandomClusterCenters ( int seed, const CvMat* data, + int num_of_clusters, CvMat* _centers ) +{ + CvMat* centers = _centers; + + CV_FUNCNAME("icvGenerateRandomClusterCenters"); + __BEGIN__; + + CvRNG rng; + CvMat data_comp, centers_comp; + CvPoint minLoc, maxLoc; // Not used, just for function "cvMinMaxLoc" + double minVal, maxVal; + int i; + int dim = data ? data->cols : 0; + + if( ICV_IS_MAT_OF_TYPE(data, CV_32FC1) ) + { + if( _centers && !ICV_IS_MAT_OF_TYPE (_centers, CV_32FC1) ) + { + CV_ERROR(CV_StsBadArg,""); + } + else if( !_centers ) + CV_CALL(centers = cvCreateMat (num_of_clusters, dim, CV_32FC1)); + } + else if( ICV_IS_MAT_OF_TYPE(data, CV_64FC1) ) + { + if( _centers && !ICV_IS_MAT_OF_TYPE (_centers, CV_64FC1) ) + { + CV_ERROR(CV_StsBadArg,""); + } + else if( !_centers ) + CV_CALL(centers = cvCreateMat (num_of_clusters, dim, CV_64FC1)); + } + else + CV_ERROR (CV_StsBadArg,""); + + if( num_of_clusters < 1 ) + CV_ERROR (CV_StsBadArg,""); + + rng = cvRNG(seed); + for (i = 0; i < dim; i++) + { + CV_CALL(cvGetCol (data, &data_comp, i)); + CV_CALL(cvMinMaxLoc (&data_comp, &minVal, &maxVal, &minLoc, &maxLoc)); + CV_CALL(cvGetCol (centers, ¢ers_comp, i)); + CV_CALL(cvRandArr (&rng, ¢ers_comp, CV_RAND_UNI, cvScalarAll(minVal), cvScalarAll(maxVal))); + } + + __END__; + + if( (cvGetErrStatus () < 0) || (centers != _centers) ) + cvReleaseMat (¢ers); + + return _centers ? _centers : centers; +} // end of icvGenerateRandomClusterCenters + +// By S. Dilman - begin - + +#define ICV_RAND_MAX 4294967296 // == 2^32 + +// static void cvRandRoundUni (CvMat* center, +// float radius_small, +// float radius_large, +// CvMat* desired_matrix, +// CvRNG* rng_state_ptr) +// { +// float rad, norm, coefficient; +// int dim, size, i, j; +// CvMat *cov, sample; +// CvRNG rng_local; + +// CV_FUNCNAME("cvRandRoundUni"); +// __BEGIN__ + +// rng_local = *rng_state_ptr; + +// CV_ASSERT ((radius_small >= 0) && +// (radius_large > 0) && +// (radius_small <= radius_large)); +// CV_ASSERT (center && desired_matrix && rng_state_ptr); +// CV_ASSERT (center->rows == 1); +// CV_ASSERT (center->cols == desired_matrix->cols); + +// dim = desired_matrix->cols; +// size = desired_matrix->rows; +// cov = cvCreateMat (dim, dim, CV_32FC1); +// cvSetIdentity (cov); +// cvRandMVNormal (center, cov, desired_matrix, &rng_local); + +// for (i = 0; i < size; i++) +// { +// rad = (float)(cvRandReal(&rng_local)*(radius_large - radius_small) + radius_small); +// cvGetRow (desired_matrix, &sample, i); +// norm = (float) cvNorm (&sample, 0, CV_L2); +// coefficient = rad / norm; +// for (j = 0; j < dim; j++) +// CV_MAT_ELEM (sample, float, 0, j) *= coefficient; +// } + +// __END__ + +// } + +// By S. Dilman - end - + +static int CV_CDECL +icvCmpIntegers( const void* a, const void* b ) +{ + return *(const int*)a - *(const int*)b; +} + + +static int CV_CDECL +icvCmpIntegersPtr( const void* _a, const void* _b ) +{ + int a = **(const int**)_a; + int b = **(const int**)_b; + return (a < b ? -1 : 0)|(a > b); +} + + +static int icvCmpSparseVecElems( const void* a, const void* b ) +{ + return ((CvSparseVecElem32f*)a)->idx - ((CvSparseVecElem32f*)b)->idx; +} + + +CvMat* +cvPreprocessIndexArray( const CvMat* idx_arr, int data_arr_size, bool check_for_duplicates ) +{ + CvMat* idx = 0; + + CV_FUNCNAME( "cvPreprocessIndexArray" ); + + __BEGIN__; + + int i, idx_total, idx_selected = 0, step, type, prev = INT_MIN, is_sorted = 1; + uchar* srcb = 0; + int* srci = 0; + int* dsti; + + if( !CV_IS_MAT(idx_arr) ) + CV_ERROR( CV_StsBadArg, "Invalid index array" ); + + if( idx_arr->rows != 1 && idx_arr->cols != 1 ) + CV_ERROR( CV_StsBadSize, "the index array must be 1-dimensional" ); + + idx_total = idx_arr->rows + idx_arr->cols - 1; + srcb = idx_arr->data.ptr; + srci = idx_arr->data.i; + + type = CV_MAT_TYPE(idx_arr->type); + step = CV_IS_MAT_CONT(idx_arr->type) ? 1 : idx_arr->step/CV_ELEM_SIZE(type); + + switch( type ) + { + case CV_8UC1: + case CV_8SC1: + // idx_arr is array of 1's and 0's - + // i.e. it is a mask of the selected components + if( idx_total != data_arr_size ) + CV_ERROR( CV_StsUnmatchedSizes, + "Component mask should contain as many elements as the total number of input variables" ); + + for( i = 0; i < idx_total; i++ ) + idx_selected += srcb[i*step] != 0; + + if( idx_selected == 0 ) + CV_ERROR( CV_StsOutOfRange, "No components/input_variables is selected!" ); + + break; + case CV_32SC1: + // idx_arr is array of integer indices of selected components + if( idx_total > data_arr_size ) + CV_ERROR( CV_StsOutOfRange, + "index array may not contain more elements than the total number of input variables" ); + idx_selected = idx_total; + // check if sorted already + for( i = 0; i < idx_total; i++ ) + { + int val = srci[i*step]; + if( val >= prev ) + { + is_sorted = 0; + break; + } + prev = val; + } + break; + default: + CV_ERROR( CV_StsUnsupportedFormat, "Unsupported index array data type " + "(it should be 8uC1, 8sC1 or 32sC1)" ); + } + + CV_CALL( idx = cvCreateMat( 1, idx_selected, CV_32SC1 )); + dsti = idx->data.i; + + if( type < CV_32SC1 ) + { + for( i = 0; i < idx_total; i++ ) + if( srcb[i*step] ) + *dsti++ = i; + } + else + { + for( i = 0; i < idx_total; i++ ) + dsti[i] = srci[i*step]; + + if( !is_sorted ) + qsort( dsti, idx_total, sizeof(dsti[0]), icvCmpIntegers ); + + if( dsti[0] < 0 || dsti[idx_total-1] >= data_arr_size ) + CV_ERROR( CV_StsOutOfRange, "the index array elements are out of range" ); + + if( check_for_duplicates ) + { + for( i = 1; i < idx_total; i++ ) + if( dsti[i] <= dsti[i-1] ) + CV_ERROR( CV_StsBadArg, "There are duplicated index array elements" ); + } + } + + __END__; + + if( cvGetErrStatus() < 0 ) + cvReleaseMat( &idx ); + + return idx; +} + + +CvMat* +cvPreprocessVarType( const CvMat* var_type, const CvMat* var_idx, + int var_count, int* response_type ) +{ + CvMat* out_var_type = 0; + CV_FUNCNAME( "cvPreprocessVarType" ); + + if( response_type ) + *response_type = -1; + + __BEGIN__; + + int i, tm_size, tm_step; + //int* map = 0; + const uchar* src; + uchar* dst; + + if( !CV_IS_MAT(var_type) ) + CV_ERROR( var_type ? CV_StsBadArg : CV_StsNullPtr, "Invalid or absent var_type array" ); + + if( var_type->rows != 1 && var_type->cols != 1 ) + CV_ERROR( CV_StsBadSize, "var_type array must be 1-dimensional" ); + + if( !CV_IS_MASK_ARR(var_type)) + CV_ERROR( CV_StsUnsupportedFormat, "type mask must be 8uC1 or 8sC1 array" ); + + tm_size = var_type->rows + var_type->cols - 1; + tm_step = var_type->rows == 1 ? 1 : var_type->step/CV_ELEM_SIZE(var_type->type); + + if( /*tm_size != var_count &&*/ tm_size != var_count + 1 ) + CV_ERROR( CV_StsBadArg, + "type mask must be of + 1 size" ); + + if( response_type && tm_size > var_count ) + *response_type = var_type->data.ptr[var_count*tm_step] != 0; + + if( var_idx ) + { + if( !CV_IS_MAT(var_idx) || CV_MAT_TYPE(var_idx->type) != CV_32SC1 || + (var_idx->rows != 1 && var_idx->cols != 1) || !CV_IS_MAT_CONT(var_idx->type) ) + CV_ERROR( CV_StsBadArg, "var index array should be continuous 1-dimensional integer vector" ); + if( var_idx->rows + var_idx->cols - 1 > var_count ) + CV_ERROR( CV_StsBadSize, "var index array is too large" ); + //map = var_idx->data.i; + var_count = var_idx->rows + var_idx->cols - 1; + } + + CV_CALL( out_var_type = cvCreateMat( 1, var_count, CV_8UC1 )); + src = var_type->data.ptr; + dst = out_var_type->data.ptr; + + for( i = 0; i < var_count; i++ ) + { + //int idx = map ? map[i] : i; + assert( (unsigned)/*idx*/i < (unsigned)tm_size ); + dst[i] = (uchar)(src[/*idx*/i*tm_step] != 0); + } + + __END__; + + return out_var_type; +} + + +CvMat* +cvPreprocessOrderedResponses( const CvMat* responses, const CvMat* sample_idx, int sample_all ) +{ + CvMat* out_responses = 0; + + CV_FUNCNAME( "cvPreprocessOrderedResponses" ); + + __BEGIN__; + + int i, r_type, r_step; + const int* map = 0; + float* dst; + int sample_count = sample_all; + + if( !CV_IS_MAT(responses) ) + CV_ERROR( CV_StsBadArg, "Invalid response array" ); + + if( responses->rows != 1 && responses->cols != 1 ) + CV_ERROR( CV_StsBadSize, "Response array must be 1-dimensional" ); + + if( responses->rows + responses->cols - 1 != sample_count ) + CV_ERROR( CV_StsUnmatchedSizes, + "Response array must contain as many elements as the total number of samples" ); + + r_type = CV_MAT_TYPE(responses->type); + if( r_type != CV_32FC1 && r_type != CV_32SC1 ) + CV_ERROR( CV_StsUnsupportedFormat, "Unsupported response type" ); + + r_step = responses->step ? responses->step / CV_ELEM_SIZE(responses->type) : 1; + + if( r_type == CV_32FC1 && CV_IS_MAT_CONT(responses->type) && !sample_idx ) + { + out_responses = cvCloneMat( responses ); + EXIT; + } + + if( sample_idx ) + { + if( !CV_IS_MAT(sample_idx) || CV_MAT_TYPE(sample_idx->type) != CV_32SC1 || + (sample_idx->rows != 1 && sample_idx->cols != 1) || !CV_IS_MAT_CONT(sample_idx->type) ) + CV_ERROR( CV_StsBadArg, "sample index array should be continuous 1-dimensional integer vector" ); + if( sample_idx->rows + sample_idx->cols - 1 > sample_count ) + CV_ERROR( CV_StsBadSize, "sample index array is too large" ); + map = sample_idx->data.i; + sample_count = sample_idx->rows + sample_idx->cols - 1; + } + + CV_CALL( out_responses = cvCreateMat( 1, sample_count, CV_32FC1 )); + + dst = out_responses->data.fl; + if( r_type == CV_32FC1 ) + { + const float* src = responses->data.fl; + for( i = 0; i < sample_count; i++ ) + { + int idx = map ? map[i] : i; + assert( (unsigned)idx < (unsigned)sample_all ); + dst[i] = src[idx*r_step]; + } + } + else + { + const int* src = responses->data.i; + for( i = 0; i < sample_count; i++ ) + { + int idx = map ? map[i] : i; + assert( (unsigned)idx < (unsigned)sample_all ); + dst[i] = (float)src[idx*r_step]; + } + } + + __END__; + + return out_responses; +} + +CvMat* +cvPreprocessCategoricalResponses( const CvMat* responses, + const CvMat* sample_idx, int sample_all, + CvMat** out_response_map, CvMat** class_counts ) +{ + CvMat* out_responses = 0; + int** response_ptr = 0; + + CV_FUNCNAME( "cvPreprocessCategoricalResponses" ); + + if( out_response_map ) + *out_response_map = 0; + + if( class_counts ) + *class_counts = 0; + + __BEGIN__; + + int i, r_type, r_step; + int cls_count = 1, prev_cls, prev_i; + const int* map = 0; + const int* srci; + const float* srcfl; + int* dst; + int* cls_map; + int* cls_counts = 0; + int sample_count = sample_all; + + if( !CV_IS_MAT(responses) ) + CV_ERROR( CV_StsBadArg, "Invalid response array" ); + + if( responses->rows != 1 && responses->cols != 1 ) + CV_ERROR( CV_StsBadSize, "Response array must be 1-dimensional" ); + + if( responses->rows + responses->cols - 1 != sample_count ) + CV_ERROR( CV_StsUnmatchedSizes, + "Response array must contain as many elements as the total number of samples" ); + + r_type = CV_MAT_TYPE(responses->type); + if( r_type != CV_32FC1 && r_type != CV_32SC1 ) + CV_ERROR( CV_StsUnsupportedFormat, "Unsupported response type" ); + + r_step = responses->rows == 1 ? 1 : responses->step / CV_ELEM_SIZE(responses->type); + + if( sample_idx ) + { + if( !CV_IS_MAT(sample_idx) || CV_MAT_TYPE(sample_idx->type) != CV_32SC1 || + (sample_idx->rows != 1 && sample_idx->cols != 1) || !CV_IS_MAT_CONT(sample_idx->type) ) + CV_ERROR( CV_StsBadArg, "sample index array should be continuous 1-dimensional integer vector" ); + if( sample_idx->rows + sample_idx->cols - 1 > sample_count ) + CV_ERROR( CV_StsBadSize, "sample index array is too large" ); + map = sample_idx->data.i; + sample_count = sample_idx->rows + sample_idx->cols - 1; + } + + CV_CALL( out_responses = cvCreateMat( 1, sample_count, CV_32SC1 )); + + if( !out_response_map ) + CV_ERROR( CV_StsNullPtr, "out_response_map pointer is NULL" ); + + CV_CALL( response_ptr = (int**)cvAlloc( sample_count*sizeof(response_ptr[0]))); + + srci = responses->data.i; + srcfl = responses->data.fl; + dst = out_responses->data.i; + + for( i = 0; i < sample_count; i++ ) + { + int idx = map ? map[i] : i; + assert( (unsigned)idx < (unsigned)sample_all ); + if( r_type == CV_32SC1 ) + dst[i] = srci[idx*r_step]; + else + { + float rf = srcfl[idx*r_step]; + int ri = cvRound(rf); + if( ri != rf ) + { + char buf[100]; + sprintf( buf, "response #%d is not integral", idx ); + CV_ERROR( CV_StsBadArg, buf ); + } + dst[i] = ri; + } + response_ptr[i] = dst + i; + } + + qsort( response_ptr, sample_count, sizeof(int*), icvCmpIntegersPtr ); + + // count the classes + for( i = 1; i < sample_count; i++ ) + cls_count += *response_ptr[i] != *response_ptr[i-1]; + + if( cls_count < 2 ) + CV_ERROR( CV_StsBadArg, "There is only a single class" ); + + CV_CALL( *out_response_map = cvCreateMat( 1, cls_count, CV_32SC1 )); + + if( class_counts ) + { + CV_CALL( *class_counts = cvCreateMat( 1, cls_count, CV_32SC1 )); + cls_counts = (*class_counts)->data.i; + } + + // compact the class indices and build the map + prev_cls = ~*response_ptr[0]; + cls_count = -1; + cls_map = (*out_response_map)->data.i; + + for( i = 0, prev_i = -1; i < sample_count; i++ ) + { + int cur_cls = *response_ptr[i]; + if( cur_cls != prev_cls ) + { + if( cls_counts && cls_count >= 0 ) + cls_counts[cls_count] = i - prev_i; + cls_map[++cls_count] = prev_cls = cur_cls; + prev_i = i; + } + *response_ptr[i] = cls_count; + } + + if( cls_counts ) + cls_counts[cls_count] = i - prev_i; + + __END__; + + cvFree( &response_ptr ); + + return out_responses; +} + + +const float** +cvGetTrainSamples( const CvMat* train_data, int tflag, + const CvMat* var_idx, const CvMat* sample_idx, + int* _var_count, int* _sample_count, + bool always_copy_data ) +{ + float** samples = 0; + + CV_FUNCNAME( "cvGetTrainSamples" ); + + __BEGIN__; + + int i, j, var_count, sample_count, s_step, v_step; + bool copy_data; + const float* data; + const int *s_idx, *v_idx; + + if( !CV_IS_MAT(train_data) ) + CV_ERROR( CV_StsBadArg, "Invalid or NULL training data matrix" ); + + var_count = var_idx ? var_idx->cols + var_idx->rows - 1 : + tflag == CV_ROW_SAMPLE ? train_data->cols : train_data->rows; + sample_count = sample_idx ? sample_idx->cols + sample_idx->rows - 1 : + tflag == CV_ROW_SAMPLE ? train_data->rows : train_data->cols; + + if( _var_count ) + *_var_count = var_count; + + if( _sample_count ) + *_sample_count = sample_count; + + copy_data = tflag != CV_ROW_SAMPLE || var_idx || always_copy_data; + + CV_CALL( samples = (float**)cvAlloc(sample_count*sizeof(samples[0]) + + (copy_data ? 1 : 0)*var_count*sample_count*sizeof(samples[0][0])) ); + data = train_data->data.fl; + s_step = train_data->step / sizeof(samples[0][0]); + v_step = 1; + s_idx = sample_idx ? sample_idx->data.i : 0; + v_idx = var_idx ? var_idx->data.i : 0; + + if( !copy_data ) + { + for( i = 0; i < sample_count; i++ ) + samples[i] = (float*)(data + (s_idx ? s_idx[i] : i)*s_step); + } + else + { + samples[0] = (float*)(samples + sample_count); + if( tflag != CV_ROW_SAMPLE ) + CV_SWAP( s_step, v_step, i ); + + for( i = 0; i < sample_count; i++ ) + { + float* dst = samples[i] = samples[0] + i*var_count; + const float* src = data + (s_idx ? s_idx[i] : i)*s_step; + + if( !v_idx ) + for( j = 0; j < var_count; j++ ) + dst[j] = src[j*v_step]; + else + for( j = 0; j < var_count; j++ ) + dst[j] = src[v_idx[j]*v_step]; + } + } + + __END__; + + return (const float**)samples; +} + + +void +cvCheckTrainData( const CvMat* train_data, int tflag, + const CvMat* missing_mask, + int* var_all, int* sample_all ) +{ + CV_FUNCNAME( "cvCheckTrainData" ); + + if( var_all ) + *var_all = 0; + + if( sample_all ) + *sample_all = 0; + + __BEGIN__; + + // check parameter types and sizes + if( !CV_IS_MAT(train_data) || CV_MAT_TYPE(train_data->type) != CV_32FC1 ) + CV_ERROR( CV_StsBadArg, "train data must be floating-point matrix" ); + + if( missing_mask ) + { + if( !CV_IS_MAT(missing_mask) || !CV_IS_MASK_ARR(missing_mask) || + !CV_ARE_SIZES_EQ(train_data, missing_mask) ) + CV_ERROR( CV_StsBadArg, + "missing value mask must be 8-bit matrix of the same size as training data" ); + } + + if( tflag != CV_ROW_SAMPLE && tflag != CV_COL_SAMPLE ) + CV_ERROR( CV_StsBadArg, + "Unknown training data layout (must be CV_ROW_SAMPLE or CV_COL_SAMPLE)" ); + + if( var_all ) + *var_all = tflag == CV_ROW_SAMPLE ? train_data->cols : train_data->rows; + + if( sample_all ) + *sample_all = tflag == CV_ROW_SAMPLE ? train_data->rows : train_data->cols; + + __END__; +} + + +int +cvPrepareTrainData( const char* /*funcname*/, + const CvMat* train_data, int tflag, + const CvMat* responses, int response_type, + const CvMat* var_idx, + const CvMat* sample_idx, + bool always_copy_data, + const float*** out_train_samples, + int* _sample_count, + int* _var_count, + int* _var_all, + CvMat** out_responses, + CvMat** out_response_map, + CvMat** out_var_idx, + CvMat** out_sample_idx ) +{ + int ok = 0; + CvMat* _var_idx = 0; + CvMat* _sample_idx = 0; + CvMat* _responses = 0; + int sample_all = 0, sample_count = 0, var_all = 0, var_count = 0; + + CV_FUNCNAME( "cvPrepareTrainData" ); + + // step 0. clear all the output pointers to ensure we do not try + // to call free() with uninitialized pointers + if( out_responses ) + *out_responses = 0; + + if( out_response_map ) + *out_response_map = 0; + + if( out_var_idx ) + *out_var_idx = 0; + + if( out_sample_idx ) + *out_sample_idx = 0; + + if( out_train_samples ) + *out_train_samples = 0; + + if( _sample_count ) + *_sample_count = 0; + + if( _var_count ) + *_var_count = 0; + + if( _var_all ) + *_var_all = 0; + + __BEGIN__; + + if( !out_train_samples ) + CV_ERROR( CV_StsBadArg, "output pointer to train samples is NULL" ); + + CV_CALL( cvCheckTrainData( train_data, tflag, 0, &var_all, &sample_all )); + + if( sample_idx ) + CV_CALL( _sample_idx = cvPreprocessIndexArray( sample_idx, sample_all )); + if( var_idx ) + CV_CALL( _var_idx = cvPreprocessIndexArray( var_idx, var_all )); + + if( responses ) + { + if( !out_responses ) + CV_ERROR( CV_StsNullPtr, "output response pointer is NULL" ); + + if( response_type == CV_VAR_NUMERICAL ) + { + CV_CALL( _responses = cvPreprocessOrderedResponses( responses, + _sample_idx, sample_all )); + } + else + { + CV_CALL( _responses = cvPreprocessCategoricalResponses( responses, + _sample_idx, sample_all, out_response_map, 0 )); + } + } + + CV_CALL( *out_train_samples = + cvGetTrainSamples( train_data, tflag, _var_idx, _sample_idx, + &var_count, &sample_count, always_copy_data )); + + ok = 1; + + __END__; + + if( ok ) + { + if( out_responses ) + *out_responses = _responses, _responses = 0; + + if( out_var_idx ) + *out_var_idx = _var_idx, _var_idx = 0; + + if( out_sample_idx ) + *out_sample_idx = _sample_idx, _sample_idx = 0; + + if( _sample_count ) + *_sample_count = sample_count; + + if( _var_count ) + *_var_count = var_count; + + if( _var_all ) + *_var_all = var_all; + } + else + { + if( out_response_map ) + cvReleaseMat( out_response_map ); + cvFree( out_train_samples ); + } + + if( _responses != responses ) + cvReleaseMat( &_responses ); + cvReleaseMat( &_var_idx ); + cvReleaseMat( &_sample_idx ); + + return ok; +} + + +typedef struct CvSampleResponsePair +{ + const float* sample; + const uchar* mask; + int response; + int index; +} +CvSampleResponsePair; + + +static int +CV_CDECL icvCmpSampleResponsePairs( const void* a, const void* b ) +{ + int ra = ((const CvSampleResponsePair*)a)->response; + int rb = ((const CvSampleResponsePair*)b)->response; + int ia = ((const CvSampleResponsePair*)a)->index; + int ib = ((const CvSampleResponsePair*)b)->index; + + return ra < rb ? -1 : ra > rb ? 1 : ia - ib; + //return (ra > rb ? -1 : 0)|(ra < rb); +} + + +void +cvSortSamplesByClasses( const float** samples, const CvMat* classes, + int* class_ranges, const uchar** mask ) +{ + CvSampleResponsePair* pairs = 0; + CV_FUNCNAME( "cvSortSamplesByClasses" ); + + __BEGIN__; + + int i, k = 0, sample_count; + + if( !samples || !classes || !class_ranges ) + CV_ERROR( CV_StsNullPtr, "INTERNAL ERROR: some of the args are NULL pointers" ); + + if( classes->rows != 1 || CV_MAT_TYPE(classes->type) != CV_32SC1 ) + CV_ERROR( CV_StsBadArg, "classes array must be a single row of integers" ); + + sample_count = classes->cols; + CV_CALL( pairs = (CvSampleResponsePair*)cvAlloc( (sample_count+1)*sizeof(pairs[0]))); + + for( i = 0; i < sample_count; i++ ) + { + pairs[i].sample = samples[i]; + pairs[i].mask = (mask) ? (mask[i]) : 0; + pairs[i].response = classes->data.i[i]; + pairs[i].index = i; + assert( classes->data.i[i] >= 0 ); + } + + qsort( pairs, sample_count, sizeof(pairs[0]), icvCmpSampleResponsePairs ); + pairs[sample_count].response = -1; + class_ranges[0] = 0; + + for( i = 0; i < sample_count; i++ ) + { + samples[i] = pairs[i].sample; + if (mask) + mask[i] = pairs[i].mask; + classes->data.i[i] = pairs[i].response; + + if( pairs[i].response != pairs[i+1].response ) + class_ranges[++k] = i+1; + } + + __END__; + + cvFree( &pairs ); +} + + +void +cvPreparePredictData( const CvArr* _sample, int dims_all, + const CvMat* comp_idx, int class_count, + const CvMat* prob, float** _row_sample, + int as_sparse ) +{ + float* row_sample = 0; + int* inverse_comp_idx = 0; + + CV_FUNCNAME( "cvPreparePredictData" ); + + __BEGIN__; + + const CvMat* sample = (const CvMat*)_sample; + float* sample_data; + int sample_step; + int is_sparse = CV_IS_SPARSE_MAT(sample); + int d, sizes[CV_MAX_DIM]; + int i, dims_selected; + int vec_size; + + if( !is_sparse && !CV_IS_MAT(sample) ) + CV_ERROR( !sample ? CV_StsNullPtr : CV_StsBadArg, "The sample is not a valid vector" ); + + if( cvGetElemType( sample ) != CV_32FC1 ) + CV_ERROR( CV_StsUnsupportedFormat, "Input sample must have 32fC1 type" ); + + CV_CALL( d = cvGetDims( sample, sizes )); + + if( !((is_sparse && d == 1) || (!is_sparse && d == 2 && (sample->rows == 1 || sample->cols == 1))) ) + CV_ERROR( CV_StsBadSize, "Input sample must be 1-dimensional vector" ); + + if( d == 1 ) + sizes[1] = 1; + + if( sizes[0] + sizes[1] - 1 != dims_all ) + CV_ERROR( CV_StsUnmatchedSizes, + "The sample size is different from what has been used for training" ); + + if( !_row_sample ) + CV_ERROR( CV_StsNullPtr, "INTERNAL ERROR: The row_sample pointer is NULL" ); + + if( comp_idx && (!CV_IS_MAT(comp_idx) || comp_idx->rows != 1 || + CV_MAT_TYPE(comp_idx->type) != CV_32SC1) ) + CV_ERROR( CV_StsBadArg, "INTERNAL ERROR: invalid comp_idx" ); + + dims_selected = comp_idx ? comp_idx->cols : dims_all; + + if( prob ) + { + if( !CV_IS_MAT(prob) ) + CV_ERROR( CV_StsBadArg, "The output matrix of probabilities is invalid" ); + + if( (prob->rows != 1 && prob->cols != 1) || + (CV_MAT_TYPE(prob->type) != CV_32FC1 && + CV_MAT_TYPE(prob->type) != CV_64FC1) ) + CV_ERROR( CV_StsBadSize, + "The matrix of probabilities must be 1-dimensional vector of 32fC1 type" ); + + if( prob->rows + prob->cols - 1 != class_count ) + CV_ERROR( CV_StsUnmatchedSizes, + "The vector of probabilities must contain as many elements as " + "the number of classes in the training set" ); + } + + vec_size = !as_sparse ? dims_selected*sizeof(row_sample[0]) : + (dims_selected + 1)*sizeof(CvSparseVecElem32f); + + if( CV_IS_MAT(sample) ) + { + sample_data = sample->data.fl; + sample_step = CV_IS_MAT_CONT(sample->type) ? 1 : sample->step/sizeof(row_sample[0]); + + if( !comp_idx && CV_IS_MAT_CONT(sample->type) && !as_sparse ) + *_row_sample = sample_data; + else + { + CV_CALL( row_sample = (float*)cvAlloc( vec_size )); + + if( !comp_idx ) + for( i = 0; i < dims_selected; i++ ) + row_sample[i] = sample_data[sample_step*i]; + else + { + int* comp = comp_idx->data.i; + for( i = 0; i < dims_selected; i++ ) + row_sample[i] = sample_data[sample_step*comp[i]]; + } + + *_row_sample = row_sample; + } + + if( as_sparse ) + { + const float* src = (const float*)row_sample; + CvSparseVecElem32f* dst = (CvSparseVecElem32f*)row_sample; + + dst[dims_selected].idx = -1; + for( i = dims_selected - 1; i >= 0; i-- ) + { + dst[i].idx = i; + dst[i].val = src[i]; + } + } + } + else + { + CvSparseNode* node; + CvSparseMatIterator mat_iterator; + const CvSparseMat* sparse = (const CvSparseMat*)sample; + assert( is_sparse ); + + node = cvInitSparseMatIterator( sparse, &mat_iterator ); + CV_CALL( row_sample = (float*)cvAlloc( vec_size )); + + if( comp_idx ) + { + CV_CALL( inverse_comp_idx = (int*)cvAlloc( dims_all*sizeof(int) )); + memset( inverse_comp_idx, -1, dims_all*sizeof(int) ); + for( i = 0; i < dims_selected; i++ ) + inverse_comp_idx[comp_idx->data.i[i]] = i; + } + + if( !as_sparse ) + { + memset( row_sample, 0, vec_size ); + + for( ; node != 0; node = cvGetNextSparseNode(&mat_iterator) ) + { + int idx = *CV_NODE_IDX( sparse, node ); + if( inverse_comp_idx ) + { + idx = inverse_comp_idx[idx]; + if( idx < 0 ) + continue; + } + row_sample[idx] = *(float*)CV_NODE_VAL( sparse, node ); + } + } + else + { + CvSparseVecElem32f* ptr = (CvSparseVecElem32f*)row_sample; + + for( ; node != 0; node = cvGetNextSparseNode(&mat_iterator) ) + { + int idx = *CV_NODE_IDX( sparse, node ); + if( inverse_comp_idx ) + { + idx = inverse_comp_idx[idx]; + if( idx < 0 ) + continue; + } + ptr->idx = idx; + ptr->val = *(float*)CV_NODE_VAL( sparse, node ); + ptr++; + } + + qsort( row_sample, ptr - (CvSparseVecElem32f*)row_sample, + sizeof(ptr[0]), icvCmpSparseVecElems ); + ptr->idx = -1; + } + + *_row_sample = row_sample; + } + + __END__; + + if( inverse_comp_idx ) + cvFree( &inverse_comp_idx ); + + if( cvGetErrStatus() < 0 && _row_sample ) + { + cvFree( &row_sample ); + *_row_sample = 0; + } +} + + +static void +icvConvertDataToSparse( const uchar* src, int src_step, int src_type, + uchar* dst, int dst_step, int dst_type, + CvSize size, int* idx ) +{ + CV_FUNCNAME( "icvConvertDataToSparse" ); + + __BEGIN__; + + int i, j; + src_type = CV_MAT_TYPE(src_type); + dst_type = CV_MAT_TYPE(dst_type); + + if( CV_MAT_CN(src_type) != 1 || CV_MAT_CN(dst_type) != 1 ) + CV_ERROR( CV_StsUnsupportedFormat, "The function supports only single-channel arrays" ); + + if( src_step == 0 ) + src_step = CV_ELEM_SIZE(src_type); + + if( dst_step == 0 ) + dst_step = CV_ELEM_SIZE(dst_type); + + // if there is no "idx" and if both arrays are continuous, + // do the whole processing (copying or conversion) in a single loop + if( !idx && CV_ELEM_SIZE(src_type)*size.width == src_step && + CV_ELEM_SIZE(dst_type)*size.width == dst_step ) + { + size.width *= size.height; + size.height = 1; + } + + if( src_type == dst_type ) + { + int full_width = CV_ELEM_SIZE(dst_type)*size.width; + + if( full_width == sizeof(int) ) // another common case: copy int's or float's + for( i = 0; i < size.height; i++, src += src_step ) + *(int*)(dst + dst_step*(idx ? idx[i] : i)) = *(int*)src; + else + for( i = 0; i < size.height; i++, src += src_step ) + memcpy( dst + dst_step*(idx ? idx[i] : i), src, full_width ); + } + else if( src_type == CV_32SC1 && (dst_type == CV_32FC1 || dst_type == CV_64FC1) ) + for( i = 0; i < size.height; i++, src += src_step ) + { + uchar* _dst = dst + dst_step*(idx ? idx[i] : i); + if( dst_type == CV_32FC1 ) + for( j = 0; j < size.width; j++ ) + ((float*)_dst)[j] = (float)((int*)src)[j]; + else + for( j = 0; j < size.width; j++ ) + ((double*)_dst)[j] = ((int*)src)[j]; + } + else if( (src_type == CV_32FC1 || src_type == CV_64FC1) && dst_type == CV_32SC1 ) + for( i = 0; i < size.height; i++, src += src_step ) + { + uchar* _dst = dst + dst_step*(idx ? idx[i] : i); + if( src_type == CV_32FC1 ) + for( j = 0; j < size.width; j++ ) + ((int*)_dst)[j] = cvRound(((float*)src)[j]); + else + for( j = 0; j < size.width; j++ ) + ((int*)_dst)[j] = cvRound(((double*)src)[j]); + } + else if( (src_type == CV_32FC1 && dst_type == CV_64FC1) || + (src_type == CV_64FC1 && dst_type == CV_32FC1) ) + for( i = 0; i < size.height; i++, src += src_step ) + { + uchar* _dst = dst + dst_step*(idx ? idx[i] : i); + if( src_type == CV_32FC1 ) + for( j = 0; j < size.width; j++ ) + ((double*)_dst)[j] = ((float*)src)[j]; + else + for( j = 0; j < size.width; j++ ) + ((float*)_dst)[j] = (float)((double*)src)[j]; + } + else + CV_ERROR( CV_StsUnsupportedFormat, "Unsupported combination of input and output vectors" ); + + __END__; +} + + +void +cvWritebackLabels( const CvMat* labels, CvMat* dst_labels, + const CvMat* centers, CvMat* dst_centers, + const CvMat* probs, CvMat* dst_probs, + const CvMat* sample_idx, int samples_all, + const CvMat* comp_idx, int dims_all ) +{ + CV_FUNCNAME( "cvWritebackLabels" ); + + __BEGIN__; + + int samples_selected = samples_all, dims_selected = dims_all; + + if( dst_labels && !CV_IS_MAT(dst_labels) ) + CV_ERROR( CV_StsBadArg, "Array of output labels is not a valid matrix" ); + + if( dst_centers ) + if( !ICV_IS_MAT_OF_TYPE(dst_centers, CV_32FC1) && + !ICV_IS_MAT_OF_TYPE(dst_centers, CV_64FC1) ) + CV_ERROR( CV_StsBadArg, "Array of cluster centers is not a valid matrix" ); + + if( dst_probs && !CV_IS_MAT(dst_probs) ) + CV_ERROR( CV_StsBadArg, "Probability matrix is not valid" ); + + if( sample_idx ) + { + CV_ASSERT( sample_idx->rows == 1 && CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ); + samples_selected = sample_idx->cols; + } + + if( comp_idx ) + { + CV_ASSERT( comp_idx->rows == 1 && CV_MAT_TYPE(comp_idx->type) == CV_32SC1 ); + dims_selected = comp_idx->cols; + } + + if( dst_labels && (!labels || labels->data.ptr != dst_labels->data.ptr) ) + { + if( !labels ) + CV_ERROR( CV_StsNullPtr, "NULL labels" ); + + CV_ASSERT( labels->rows == 1 ); + + if( dst_labels->rows != 1 && dst_labels->cols != 1 ) + CV_ERROR( CV_StsBadSize, "Array of output labels should be 1d vector" ); + + if( dst_labels->rows + dst_labels->cols - 1 != samples_all ) + CV_ERROR( CV_StsUnmatchedSizes, + "Size of vector of output labels is not equal to the total number of input samples" ); + + CV_ASSERT( labels->cols == samples_selected ); + + CV_CALL( icvConvertDataToSparse( labels->data.ptr, labels->step, labels->type, + dst_labels->data.ptr, dst_labels->step, dst_labels->type, + cvSize( 1, samples_selected ), sample_idx ? sample_idx->data.i : 0 )); + } + + if( dst_centers && (!centers || centers->data.ptr != dst_centers->data.ptr) ) + { + int i; + + if( !centers ) + CV_ERROR( CV_StsNullPtr, "NULL centers" ); + + if( centers->rows != dst_centers->rows ) + CV_ERROR( CV_StsUnmatchedSizes, "Invalid number of rows in matrix of output centers" ); + + if( dst_centers->cols != dims_all ) + CV_ERROR( CV_StsUnmatchedSizes, + "Number of columns in matrix of output centers is " + "not equal to the total number of components in the input samples" ); + + CV_ASSERT( centers->cols == dims_selected ); + + for( i = 0; i < centers->rows; i++ ) + CV_CALL( icvConvertDataToSparse( centers->data.ptr + i*centers->step, 0, centers->type, + dst_centers->data.ptr + i*dst_centers->step, 0, dst_centers->type, + cvSize( 1, dims_selected ), comp_idx ? comp_idx->data.i : 0 )); + } + + if( dst_probs && (!probs || probs->data.ptr != dst_probs->data.ptr) ) + { + if( !probs ) + CV_ERROR( CV_StsNullPtr, "NULL probs" ); + + if( probs->cols != dst_probs->cols ) + CV_ERROR( CV_StsUnmatchedSizes, "Invalid number of columns in output probability matrix" ); + + if( dst_probs->rows != samples_all ) + CV_ERROR( CV_StsUnmatchedSizes, + "Number of rows in output probability matrix is " + "not equal to the total number of input samples" ); + + CV_ASSERT( probs->rows == samples_selected ); + + CV_CALL( icvConvertDataToSparse( probs->data.ptr, probs->step, probs->type, + dst_probs->data.ptr, dst_probs->step, dst_probs->type, + cvSize( probs->cols, samples_selected ), + sample_idx ? sample_idx->data.i : 0 )); + } + + __END__; +} + +#if 0 +CV_IMPL void +cvStatModelMultiPredict( const CvStatModel* stat_model, + const CvArr* predict_input, + int flags, CvMat* predict_output, + CvMat* probs, const CvMat* sample_idx ) +{ + CvMemStorage* storage = 0; + CvMat* sample_idx_buffer = 0; + CvSparseMat** sparse_rows = 0; + int samples_selected = 0; + + CV_FUNCNAME( "cvStatModelMultiPredict" ); + + __BEGIN__; + + int i; + int predict_output_step = 1, sample_idx_step = 1; + int type; + int d, sizes[CV_MAX_DIM]; + int tflag = flags == CV_COL_SAMPLE; + int samples_all, dims_all; + int is_sparse = CV_IS_SPARSE_MAT(predict_input); + CvMat predict_input_part; + CvArr* sample = &predict_input_part; + CvMat probs_part; + CvMat* probs1 = probs ? &probs_part : 0; + + if( !CV_IS_STAT_MODEL(stat_model) ) + CV_ERROR( !stat_model ? CV_StsNullPtr : CV_StsBadArg, "Invalid statistical model" ); + + if( !stat_model->predict ) + CV_ERROR( CV_StsNotImplemented, "There is no \"predict\" method" ); + + if( !predict_input || !predict_output ) + CV_ERROR( CV_StsNullPtr, "NULL input or output matrices" ); + + if( !is_sparse && !CV_IS_MAT(predict_input) ) + CV_ERROR( CV_StsBadArg, "predict_input should be a matrix or a sparse matrix" ); + + if( !CV_IS_MAT(predict_output) ) + CV_ERROR( CV_StsBadArg, "predict_output should be a matrix" ); + + type = cvGetElemType( predict_input ); + if( type != CV_32FC1 || + (CV_MAT_TYPE(predict_output->type) != CV_32FC1 && + CV_MAT_TYPE(predict_output->type) != CV_32SC1 )) + CV_ERROR( CV_StsUnsupportedFormat, "The input or output matrix has unsupported format" ); + + CV_CALL( d = cvGetDims( predict_input, sizes )); + if( d > 2 ) + CV_ERROR( CV_StsBadSize, "The input matrix should be 1- or 2-dimensional" ); + + if( !tflag ) + { + samples_all = samples_selected = sizes[0]; + dims_all = sizes[1]; + } + else + { + samples_all = samples_selected = sizes[1]; + dims_all = sizes[0]; + } + + if( sample_idx ) + { + if( !CV_IS_MAT(sample_idx) ) + CV_ERROR( CV_StsBadArg, "Invalid sample_idx matrix" ); + + if( sample_idx->cols != 1 && sample_idx->rows != 1 ) + CV_ERROR( CV_StsBadSize, "sample_idx must be 1-dimensional matrix" ); + + samples_selected = sample_idx->rows + sample_idx->cols - 1; + + if( CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ) + { + if( samples_selected > samples_all ) + CV_ERROR( CV_StsBadSize, "sample_idx is too large vector" ); + } + else if( samples_selected != samples_all ) + CV_ERROR( CV_StsUnmatchedSizes, "sample_idx has incorrect size" ); + + sample_idx_step = sample_idx->step ? + sample_idx->step / CV_ELEM_SIZE(sample_idx->type) : 1; + } + + if( predict_output->rows != 1 && predict_output->cols != 1 ) + CV_ERROR( CV_StsBadSize, "predict_output should be a 1-dimensional matrix" ); + + if( predict_output->rows + predict_output->cols - 1 != samples_all ) + CV_ERROR( CV_StsUnmatchedSizes, "predict_output and predict_input have uncoordinated sizes" ); + + predict_output_step = predict_output->step ? + predict_output->step / CV_ELEM_SIZE(predict_output->type) : 1; + + if( probs ) + { + if( !CV_IS_MAT(probs) ) + CV_ERROR( CV_StsBadArg, "Invalid matrix of probabilities" ); + + if( probs->rows != samples_all ) + CV_ERROR( CV_StsUnmatchedSizes, + "matrix of probabilities must have as many rows as the total number of samples" ); + + if( CV_MAT_TYPE(probs->type) != CV_32FC1 ) + CV_ERROR( CV_StsUnsupportedFormat, "matrix of probabilities must have 32fC1 type" ); + } + + if( is_sparse ) + { + CvSparseNode* node; + CvSparseMatIterator mat_iterator; + CvSparseMat* sparse = (CvSparseMat*)predict_input; + + if( sample_idx && CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ) + { + CV_CALL( sample_idx_buffer = cvCreateMat( 1, samples_all, CV_8UC1 )); + cvZero( sample_idx_buffer ); + for( i = 0; i < samples_selected; i++ ) + sample_idx_buffer->data.ptr[sample_idx->data.i[i*sample_idx_step]] = 1; + samples_selected = samples_all; + sample_idx = sample_idx_buffer; + sample_idx_step = 1; + } + + CV_CALL( sparse_rows = (CvSparseMat**)cvAlloc( samples_selected*sizeof(sparse_rows[0]))); + for( i = 0; i < samples_selected; i++ ) + { + if( sample_idx && sample_idx->data.ptr[i*sample_idx_step] == 0 ) + continue; + CV_CALL( sparse_rows[i] = cvCreateSparseMat( 1, &dims_all, type )); + if( !storage ) + storage = sparse_rows[i]->heap->storage; + else + { + // hack: to decrease memory footprint, make all the sparse matrices + // reside in the same storage + int elem_size = sparse_rows[i]->heap->elem_size; + cvReleaseMemStorage( &sparse_rows[i]->heap->storage ); + sparse_rows[i]->heap = cvCreateSet( 0, sizeof(CvSet), elem_size, storage ); + } + } + + // put each row (or column) of predict_input into separate sparse matrix. + node = cvInitSparseMatIterator( sparse, &mat_iterator ); + for( ; node != 0; node = cvGetNextSparseNode( &mat_iterator )) + { + int* idx = CV_NODE_IDX( sparse, node ); + int idx0 = idx[tflag ^ 1]; + int idx1 = idx[tflag]; + + if( sample_idx && sample_idx->data.ptr[idx0*sample_idx_step] == 0 ) + continue; + + assert( sparse_rows[idx0] != 0 ); + *(float*)cvPtrND( sparse, &idx1, 0, 1, 0 ) = *(float*)CV_NODE_VAL( sparse, node ); + } + } + + for( i = 0; i < samples_selected; i++ ) + { + int idx = i; + float response; + + if( sample_idx ) + { + if( CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ) + { + idx = sample_idx->data.i[i*sample_idx_step]; + if( (unsigned)idx >= (unsigned)samples_all ) + CV_ERROR( CV_StsOutOfRange, "Some of sample_idx elements are out of range" ); + } + else if( CV_MAT_TYPE(sample_idx->type) == CV_8UC1 && + sample_idx->data.ptr[i*sample_idx_step] == 0 ) + continue; + } + + if( !is_sparse ) + { + if( !tflag ) + cvGetRow( predict_input, &predict_input_part, idx ); + else + { + cvGetCol( predict_input, &predict_input_part, idx ); + } + } + else + sample = sparse_rows[idx]; + + if( probs ) + cvGetRow( probs, probs1, idx ); + + CV_CALL( response = stat_model->predict( stat_model, (CvMat*)sample, probs1 )); + + if( CV_MAT_TYPE(predict_output->type) == CV_32FC1 ) + predict_output->data.fl[idx*predict_output_step] = response; + else + { + CV_ASSERT( cvRound(response) == response ); + predict_output->data.i[idx*predict_output_step] = cvRound(response); + } + } + + __END__; + + if( sparse_rows ) + { + int i; + for( i = 0; i < samples_selected; i++ ) + if( sparse_rows[i] ) + { + sparse_rows[i]->heap->storage = 0; + cvReleaseSparseMat( &sparse_rows[i] ); + } + cvFree( &sparse_rows ); + } + + cvReleaseMat( &sample_idx_buffer ); + cvReleaseMemStorage( &storage ); +} +#endif + +// By P. Yarykin - begin - + +void cvCombineResponseMaps (CvMat* _responses, + const CvMat* old_response_map, + CvMat* new_response_map, + CvMat** out_response_map) +{ + int** old_data = NULL; + int** new_data = NULL; + + CV_FUNCNAME ("cvCombineResponseMaps"); + __BEGIN__ + + int i,j; + int old_n, new_n, out_n; + int samples, free_response; + int* first; + int* responses; + int* out_data; + + if( out_response_map ) + *out_response_map = 0; + +// Check input data. + if ((!ICV_IS_MAT_OF_TYPE (_responses, CV_32SC1)) || + (!ICV_IS_MAT_OF_TYPE (old_response_map, CV_32SC1)) || + (!ICV_IS_MAT_OF_TYPE (new_response_map, CV_32SC1))) + { + CV_ERROR (CV_StsBadArg, "Some of input arguments is not the CvMat") + } + +// Prepare sorted responses. + first = new_response_map->data.i; + new_n = new_response_map->cols; + CV_CALL (new_data = (int**)cvAlloc (new_n * sizeof (new_data[0]))); + for (i = 0; i < new_n; i++) + new_data[i] = first + i; + qsort (new_data, new_n, sizeof(int*), icvCmpIntegersPtr); + + first = old_response_map->data.i; + old_n = old_response_map->cols; + CV_CALL (old_data = (int**)cvAlloc (old_n * sizeof (old_data[0]))); + for (i = 0; i < old_n; i++) + old_data[i] = first + i; + qsort (old_data, old_n, sizeof(int*), icvCmpIntegersPtr); + +// Count the number of different responses. + for (i = 0, j = 0, out_n = 0; i < old_n && j < new_n; out_n++) + { + if (*old_data[i] == *new_data[j]) + { + i++; + j++; + } + else if (*old_data[i] < *new_data[j]) + i++; + else + j++; + } + out_n += old_n - i + new_n - j; + +// Create and fill the result response maps. + CV_CALL (*out_response_map = cvCreateMat (1, out_n, CV_32SC1)); + out_data = (*out_response_map)->data.i; + memcpy (out_data, first, old_n * sizeof (int)); + + free_response = old_n; + for (i = 0, j = 0; i < old_n && j < new_n; ) + { + if (*old_data[i] == *new_data[j]) + { + *new_data[j] = (int)(old_data[i] - first); + i++; + j++; + } + else if (*old_data[i] < *new_data[j]) + i++; + else + { + out_data[free_response] = *new_data[j]; + *new_data[j] = free_response++; + j++; + } + } + for (; j < new_n; j++) + { + out_data[free_response] = *new_data[j]; + *new_data[j] = free_response++; + } + CV_ASSERT (free_response == out_n); + +// Change according to out response map. + samples = _responses->cols + _responses->rows - 1; + responses = _responses->data.i; + first = new_response_map->data.i; + for (i = 0; i < samples; i++) + { + responses[i] = first[responses[i]]; + } + + __END__ + + cvFree(&old_data); + cvFree(&new_data); + +} + + +static int icvGetNumberOfCluster( double* prob_vector, int num_of_clusters, float r, + float outlier_thresh, int normalize_probs ) +{ + int max_prob_loc = 0; + + CV_FUNCNAME("icvGetNumberOfCluster"); + __BEGIN__; + + double prob, maxprob, sum; + int i; + + CV_ASSERT(prob_vector); + CV_ASSERT(num_of_clusters >= 0); + + maxprob = prob_vector[0]; + max_prob_loc = 0; + sum = maxprob; + for( i = 1; i < num_of_clusters; i++ ) + { + prob = prob_vector[i]; + sum += prob; + if( prob > maxprob ) + { + max_prob_loc = i; + maxprob = prob; + } + } + if( normalize_probs && fabs(sum - 1.) > FLT_EPSILON ) + { + for( i = 0; i < num_of_clusters; i++ ) + prob_vector[i] /= sum; + } + if( fabs(r - 1.) > FLT_EPSILON && fabs(sum - 1.) < outlier_thresh ) + max_prob_loc = -1; + + __END__; + + return max_prob_loc; + +} // End of icvGetNumberOfCluster + + +void icvFindClusterLabels( const CvMat* probs, float outlier_thresh, float r, + const CvMat* labels ) +{ + CvMat* counts = 0; + + CV_FUNCNAME("icvFindClusterLabels"); + __BEGIN__; + + int nclusters, nsamples; + int i, j; + double* probs_data; + + CV_ASSERT( ICV_IS_MAT_OF_TYPE(probs, CV_64FC1) ); + CV_ASSERT( ICV_IS_MAT_OF_TYPE(labels, CV_32SC1) ); + + nclusters = probs->cols; + nsamples = probs->rows; + CV_ASSERT( nsamples == labels->cols ); + + CV_CALL( counts = cvCreateMat( 1, nclusters + 1, CV_32SC1 ) ); + CV_CALL( cvSetZero( counts )); + for( i = 0; i < nsamples; i++ ) + { + labels->data.i[i] = icvGetNumberOfCluster( probs->data.db + i*probs->cols, + nclusters, r, outlier_thresh, 1 ); + counts->data.i[labels->data.i[i] + 1]++; + } + CV_ASSERT((int)cvSum(counts).val[0] == nsamples); + // Filling empty clusters with the vector, that has the maximal probability + for( j = 0; j < nclusters; j++ ) // outliers are ignored + { + int maxprob_loc = -1; + double maxprob = 0; + + if( counts->data.i[j+1] ) // j-th class is not empty + continue; + // look for the presentative, which is not lonely in it's cluster + // and that has a maximal probability among all these vectors + probs_data = probs->data.db; + for( i = 0; i < nsamples; i++, probs_data++ ) + { + int label = labels->data.i[i]; + double prob; + if( counts->data.i[label+1] == 0 || + (counts->data.i[label+1] <= 1 && label != -1) ) + continue; + prob = *probs_data; + if( prob >= maxprob ) + { + maxprob = prob; + maxprob_loc = i; + } + } + // maxprob_loc == 0 <=> number of vectors less then number of clusters + CV_ASSERT( maxprob_loc >= 0 ); + counts->data.i[labels->data.i[maxprob_loc] + 1]--; + labels->data.i[maxprob_loc] = j; + counts->data.i[j + 1]++; + } + + __END__; + + cvReleaseMat( &counts ); +} // End of icvFindClusterLabels + +/* End of file */ diff --git a/apps/traincascade/old_ml_precomp.hpp b/apps/traincascade/old_ml_precomp.hpp new file mode 100644 index 0000000000000000000000000000000000000000..32ae2698180b35647d29cc6119dd57c2699ce81b --- /dev/null +++ b/apps/traincascade/old_ml_precomp.hpp @@ -0,0 +1,376 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_PRECOMP_H__ +#define __OPENCV_PRECOMP_H__ + +#include "opencv2/core.hpp" +#include "old_ml.hpp" +#include "opencv2/core/core_c.h" +#include "opencv2/core/utility.hpp" + +#include "opencv2/core/private.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define ML_IMPL CV_IMPL +#define __BEGIN__ __CV_BEGIN__ +#define __END__ __CV_END__ +#define EXIT __CV_EXIT__ + +#define CV_MAT_ELEM_FLAG( mat, type, comp, vect, tflag ) \ + (( tflag == CV_ROW_SAMPLE ) \ + ? (CV_MAT_ELEM( mat, type, comp, vect )) \ + : (CV_MAT_ELEM( mat, type, vect, comp ))) + +/* Convert matrix to vector */ +#define ICV_MAT2VEC( mat, vdata, vstep, num ) \ + if( MIN( (mat).rows, (mat).cols ) != 1 ) \ + CV_ERROR( CV_StsBadArg, "" ); \ + (vdata) = ((mat).data.ptr); \ + if( (mat).rows == 1 ) \ + { \ + (vstep) = CV_ELEM_SIZE( (mat).type ); \ + (num) = (mat).cols; \ + } \ + else \ + { \ + (vstep) = (mat).step; \ + (num) = (mat).rows; \ + } + +/* get raw data */ +#define ICV_RAWDATA( mat, flags, rdata, sstep, cstep, m, n ) \ + (rdata) = (mat).data.ptr; \ + if( CV_IS_ROW_SAMPLE( flags ) ) \ + { \ + (sstep) = (mat).step; \ + (cstep) = CV_ELEM_SIZE( (mat).type ); \ + (m) = (mat).rows; \ + (n) = (mat).cols; \ + } \ + else \ + { \ + (cstep) = (mat).step; \ + (sstep) = CV_ELEM_SIZE( (mat).type ); \ + (n) = (mat).rows; \ + (m) = (mat).cols; \ + } + +#define ICV_IS_MAT_OF_TYPE( mat, mat_type) \ + (CV_IS_MAT( mat ) && CV_MAT_TYPE( mat->type ) == (mat_type) && \ + (mat)->cols > 0 && (mat)->rows > 0) + +/* + uchar* data; int sstep, cstep; - trainData->data + uchar* classes; int clstep; int ncl;- trainClasses + uchar* tmask; int tmstep; int ntm; - typeMask + uchar* missed;int msstep, mcstep; -missedMeasurements... + int mm, mn; == m,n == size,dim + uchar* sidx;int sistep; - sampleIdx + uchar* cidx;int cistep; - compIdx + int k, l; == n,m == dim,size (length of cidx, sidx) + int m, n; == size,dim +*/ +#define ICV_DECLARE_TRAIN_ARGS() \ + uchar* data; \ + int sstep, cstep; \ + uchar* classes; \ + int clstep; \ + int ncl; \ + uchar* tmask; \ + int tmstep; \ + int ntm; \ + uchar* missed; \ + int msstep, mcstep; \ + int mm, mn; \ + uchar* sidx; \ + int sistep; \ + uchar* cidx; \ + int cistep; \ + int k, l; \ + int m, n; \ + \ + data = classes = tmask = missed = sidx = cidx = NULL; \ + sstep = cstep = clstep = ncl = tmstep = ntm = msstep = mcstep = mm = mn = 0; \ + sistep = cistep = k = l = m = n = 0; + +#define ICV_TRAIN_DATA_REQUIRED( param, flags ) \ + if( !ICV_IS_MAT_OF_TYPE( (param), CV_32FC1 ) ) \ + { \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + else \ + { \ + ICV_RAWDATA( *(param), (flags), data, sstep, cstep, m, n ); \ + k = n; \ + l = m; \ + } + +#define ICV_TRAIN_CLASSES_REQUIRED( param ) \ + if( !ICV_IS_MAT_OF_TYPE( (param), CV_32FC1 ) ) \ + { \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + else \ + { \ + ICV_MAT2VEC( *(param), classes, clstep, ncl ); \ + if( m != ncl ) \ + { \ + CV_ERROR( CV_StsBadArg, "Unmatched sizes" ); \ + } \ + } + +#define ICV_ARG_NULL( param ) \ + if( (param) != NULL ) \ + { \ + CV_ERROR( CV_StsBadArg, #param " parameter must be NULL" ); \ + } + +#define ICV_MISSED_MEASUREMENTS_OPTIONAL( param, flags ) \ + if( param ) \ + { \ + if( !ICV_IS_MAT_OF_TYPE( param, CV_8UC1 ) ) \ + { \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + else \ + { \ + ICV_RAWDATA( *(param), (flags), missed, msstep, mcstep, mm, mn ); \ + if( mm != m || mn != n ) \ + { \ + CV_ERROR( CV_StsBadArg, "Unmatched sizes" ); \ + } \ + } \ + } + +#define ICV_COMP_IDX_OPTIONAL( param ) \ + if( param ) \ + { \ + if( !ICV_IS_MAT_OF_TYPE( param, CV_32SC1 ) ) \ + { \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + else \ + { \ + ICV_MAT2VEC( *(param), cidx, cistep, k ); \ + if( k > n ) \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + } + +#define ICV_SAMPLE_IDX_OPTIONAL( param ) \ + if( param ) \ + { \ + if( !ICV_IS_MAT_OF_TYPE( param, CV_32SC1 ) ) \ + { \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + else \ + { \ + ICV_MAT2VEC( *sampleIdx, sidx, sistep, l ); \ + if( l > m ) \ + CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ + } \ + } + +/****************************************************************************************/ +#define ICV_CONVERT_FLOAT_ARRAY_TO_MATRICE( array, matrice ) \ +{ \ + CvMat a, b; \ + int dims = (matrice)->cols; \ + int nsamples = (matrice)->rows; \ + int type = CV_MAT_TYPE((matrice)->type); \ + int i, offset = dims; \ + \ + CV_ASSERT( type == CV_32FC1 || type == CV_64FC1 ); \ + offset *= ((type == CV_32FC1) ? sizeof(float) : sizeof(double));\ + \ + b = cvMat( 1, dims, CV_32FC1 ); \ + cvGetRow( matrice, &a, 0 ); \ + for( i = 0; i < nsamples; i++, a.data.ptr += offset ) \ + { \ + b.data.fl = (float*)array[i]; \ + CV_CALL( cvConvert( &b, &a ) ); \ + } \ +} + +/****************************************************************************************\ +* Auxiliary functions declarations * +\****************************************************************************************/ + +/* Generates a set of classes centers in quantity that are generated as + uniform random vectors in parallelepiped, where is concentrated. Vectors in + should have horizontal orientation. If != NULL, the function doesn't + allocate any memory and stores generated centers in , returns . + If == NULL, the function allocates memory and creates the matrice. Centers + are supposed to be oriented horizontally. */ +CvMat* icvGenerateRandomClusterCenters( int seed, + const CvMat* data, + int num_of_clusters, + CvMat* centers CV_DEFAULT(0)); + +/* Fills the using by choosing the maximal probability. Outliers are + fixed by and have cluster label (-1). Function also controls that there + weren't "empty" clusters by filling empty clusters with the maximal probability vector. + If probs_sums != NULL, filles it with the sums of probabilities for each sample (it is + useful for normalizing probabilities' matrice of FCM) */ +void icvFindClusterLabels( const CvMat* probs, float outlier_thresh, float r, + const CvMat* labels ); + +typedef struct CvSparseVecElem32f +{ + int idx; + float val; +} +CvSparseVecElem32f; + +/* Prepare training data and related parameters */ +#define CV_TRAIN_STATMODEL_DEFRAGMENT_TRAIN_DATA 1 +#define CV_TRAIN_STATMODEL_SAMPLES_AS_ROWS 2 +#define CV_TRAIN_STATMODEL_SAMPLES_AS_COLUMNS 4 +#define CV_TRAIN_STATMODEL_CATEGORICAL_RESPONSE 8 +#define CV_TRAIN_STATMODEL_ORDERED_RESPONSE 16 +#define CV_TRAIN_STATMODEL_RESPONSES_ON_OUTPUT 32 +#define CV_TRAIN_STATMODEL_ALWAYS_COPY_TRAIN_DATA 64 +#define CV_TRAIN_STATMODEL_SPARSE_AS_SPARSE 128 + +int +cvPrepareTrainData( const char* /*funcname*/, + const CvMat* train_data, int tflag, + const CvMat* responses, int response_type, + const CvMat* var_idx, + const CvMat* sample_idx, + bool always_copy_data, + const float*** out_train_samples, + int* _sample_count, + int* _var_count, + int* _var_all, + CvMat** out_responses, + CvMat** out_response_map, + CvMat** out_var_idx, + CvMat** out_sample_idx=0 ); + +void +cvSortSamplesByClasses( const float** samples, const CvMat* classes, + int* class_ranges, const uchar** mask CV_DEFAULT(0) ); + +void +cvCombineResponseMaps (CvMat* _responses, + const CvMat* old_response_map, + CvMat* new_response_map, + CvMat** out_response_map); + +void +cvPreparePredictData( const CvArr* sample, int dims_all, const CvMat* comp_idx, + int class_count, const CvMat* prob, float** row_sample, + int as_sparse CV_DEFAULT(0) ); + +/* copies clustering [or batch "predict"] results + (labels and/or centers and/or probs) back to the output arrays */ +void +cvWritebackLabels( const CvMat* labels, CvMat* dst_labels, + const CvMat* centers, CvMat* dst_centers, + const CvMat* probs, CvMat* dst_probs, + const CvMat* sample_idx, int samples_all, + const CvMat* comp_idx, int dims_all ); +#define cvWritebackResponses cvWritebackLabels + +#define XML_FIELD_NAME "_name" +CvFileNode* icvFileNodeGetChild(CvFileNode* father, const char* name); +CvFileNode* icvFileNodeGetChildArrayElem(CvFileNode* father, const char* name,int index); +CvFileNode* icvFileNodeGetNext(CvFileNode* n, const char* name); + + +void cvCheckTrainData( const CvMat* train_data, int tflag, + const CvMat* missing_mask, + int* var_all, int* sample_all ); + +CvMat* cvPreprocessIndexArray( const CvMat* idx_arr, int data_arr_size, bool check_for_duplicates=false ); + +CvMat* cvPreprocessVarType( const CvMat* type_mask, const CvMat* var_idx, + int var_all, int* response_type ); + +CvMat* cvPreprocessOrderedResponses( const CvMat* responses, + const CvMat* sample_idx, int sample_all ); + +CvMat* cvPreprocessCategoricalResponses( const CvMat* responses, + const CvMat* sample_idx, int sample_all, + CvMat** out_response_map, CvMat** class_counts=0 ); + +const float** cvGetTrainSamples( const CvMat* train_data, int tflag, + const CvMat* var_idx, const CvMat* sample_idx, + int* _var_count, int* _sample_count, + bool always_copy_data=false ); + +namespace cv +{ + struct DTreeBestSplitFinder + { + DTreeBestSplitFinder(){ splitSize = 0, tree = 0; node = 0; } + DTreeBestSplitFinder( CvDTree* _tree, CvDTreeNode* _node); + DTreeBestSplitFinder( const DTreeBestSplitFinder& finder, Split ); + virtual ~DTreeBestSplitFinder() {} + virtual void operator()(const BlockedRange& range); + void join( DTreeBestSplitFinder& rhs ); + Ptr bestSplit; + Ptr split; + int splitSize; + CvDTree* tree; + CvDTreeNode* node; + }; + + struct ForestTreeBestSplitFinder : DTreeBestSplitFinder + { + ForestTreeBestSplitFinder() : DTreeBestSplitFinder() {} + ForestTreeBestSplitFinder( CvForestTree* _tree, CvDTreeNode* _node ); + ForestTreeBestSplitFinder( const ForestTreeBestSplitFinder& finder, Split ); + virtual void operator()(const BlockedRange& range); + }; +} + +#endif /* __ML_H__ */ diff --git a/apps/traincascade/old_ml_tree.cpp b/apps/traincascade/old_ml_tree.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7e346ccbc1a7311c12f77d7c00ae7c20d57112d --- /dev/null +++ b/apps/traincascade/old_ml_tree.cpp @@ -0,0 +1,4151 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// Intel License Agreement +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "old_ml_precomp.hpp" +#include + +using namespace cv; + +static const float ord_nan = FLT_MAX*0.5f; +static const int min_block_size = 1 << 16; +static const int block_size_delta = 1 << 10; + +CvDTreeTrainData::CvDTreeTrainData() +{ + var_idx = var_type = cat_count = cat_ofs = cat_map = + priors = priors_mult = counts = direction = split_buf = responses_copy = 0; + buf = 0; + tree_storage = temp_storage = 0; + + clear(); +} + + +CvDTreeTrainData::CvDTreeTrainData( const CvMat* _train_data, int _tflag, + const CvMat* _responses, const CvMat* _var_idx, + const CvMat* _sample_idx, const CvMat* _var_type, + const CvMat* _missing_mask, const CvDTreeParams& _params, + bool _shared, bool _add_labels ) +{ + var_idx = var_type = cat_count = cat_ofs = cat_map = + priors = priors_mult = counts = direction = split_buf = responses_copy = 0; + buf = 0; + + tree_storage = temp_storage = 0; + + set_data( _train_data, _tflag, _responses, _var_idx, _sample_idx, + _var_type, _missing_mask, _params, _shared, _add_labels ); +} + + +CvDTreeTrainData::~CvDTreeTrainData() +{ + clear(); +} + + +bool CvDTreeTrainData::set_params( const CvDTreeParams& _params ) +{ + bool ok = false; + + CV_FUNCNAME( "CvDTreeTrainData::set_params" ); + + __BEGIN__; + + // set parameters + params = _params; + + if( params.max_categories < 2 ) + CV_ERROR( CV_StsOutOfRange, "params.max_categories should be >= 2" ); + params.max_categories = MIN( params.max_categories, 15 ); + + if( params.max_depth < 0 ) + CV_ERROR( CV_StsOutOfRange, "params.max_depth should be >= 0" ); + params.max_depth = MIN( params.max_depth, 25 ); + + params.min_sample_count = MAX(params.min_sample_count,1); + + if( params.cv_folds < 0 ) + CV_ERROR( CV_StsOutOfRange, + "params.cv_folds should be =0 (the tree is not pruned) " + "or n>0 (tree is pruned using n-fold cross-validation)" ); + + if( params.cv_folds == 1 ) + params.cv_folds = 0; + + if( params.regression_accuracy < 0 ) + CV_ERROR( CV_StsOutOfRange, "params.regression_accuracy should be >= 0" ); + + ok = true; + + __END__; + + return ok; +} + +template +class LessThanPtr +{ +public: + bool operator()(T* a, T* b) const { return *a < *b; } +}; + +template +class LessThanIdx +{ +public: + LessThanIdx( const T* _arr ) : arr(_arr) {} + bool operator()(Idx a, Idx b) const { return arr[a] < arr[b]; } + const T* arr; +}; + +class LessThanPairs +{ +public: + bool operator()(const CvPair16u32s& a, const CvPair16u32s& b) const { return *a.i < *b.i; } +}; + +void CvDTreeTrainData::set_data( const CvMat* _train_data, int _tflag, + const CvMat* _responses, const CvMat* _var_idx, const CvMat* _sample_idx, + const CvMat* _var_type, const CvMat* _missing_mask, const CvDTreeParams& _params, + bool _shared, bool _add_labels, bool _update_data ) +{ + CvMat* sample_indices = 0; + CvMat* var_type0 = 0; + CvMat* tmp_map = 0; + int** int_ptr = 0; + CvPair16u32s* pair16u32s_ptr = 0; + CvDTreeTrainData* data = 0; + float *_fdst = 0; + int *_idst = 0; + unsigned short* udst = 0; + int* idst = 0; + + CV_FUNCNAME( "CvDTreeTrainData::set_data" ); + + __BEGIN__; + + int sample_all = 0, r_type, cv_n; + int total_c_count = 0; + int tree_block_size, temp_block_size, max_split_size, nv_size, cv_size = 0; + int ds_step, dv_step, ms_step = 0, mv_step = 0; // {data|mask}{sample|var}_step + int vi, i, size; + char err[100]; + const int *sidx = 0, *vidx = 0; + + uint64 effective_buf_size = 0; + int effective_buf_height = 0, effective_buf_width = 0; + + if( _update_data && data_root ) + { + data = new CvDTreeTrainData( _train_data, _tflag, _responses, _var_idx, + _sample_idx, _var_type, _missing_mask, _params, _shared, _add_labels ); + + // compare new and old train data + if( !(data->var_count == var_count && + cvNorm( data->var_type, var_type, CV_C ) < FLT_EPSILON && + cvNorm( data->cat_count, cat_count, CV_C ) < FLT_EPSILON && + cvNorm( data->cat_map, cat_map, CV_C ) < FLT_EPSILON) ) + CV_ERROR( CV_StsBadArg, + "The new training data must have the same types and the input and output variables " + "and the same categories for categorical variables" ); + + cvReleaseMat( &priors ); + cvReleaseMat( &priors_mult ); + cvReleaseMat( &buf ); + cvReleaseMat( &direction ); + cvReleaseMat( &split_buf ); + cvReleaseMemStorage( &temp_storage ); + + priors = data->priors; data->priors = 0; + priors_mult = data->priors_mult; data->priors_mult = 0; + buf = data->buf; data->buf = 0; + buf_count = data->buf_count; buf_size = data->buf_size; + sample_count = data->sample_count; + + direction = data->direction; data->direction = 0; + split_buf = data->split_buf; data->split_buf = 0; + temp_storage = data->temp_storage; data->temp_storage = 0; + nv_heap = data->nv_heap; cv_heap = data->cv_heap; + + data_root = new_node( 0, sample_count, 0, 0 ); + EXIT; + } + + clear(); + + var_all = 0; + rng = &cv::theRNG(); + + CV_CALL( set_params( _params )); + + // check parameter types and sizes + CV_CALL( cvCheckTrainData( _train_data, _tflag, _missing_mask, &var_all, &sample_all )); + + train_data = _train_data; + responses = _responses; + + if( _tflag == CV_ROW_SAMPLE ) + { + ds_step = _train_data->step/CV_ELEM_SIZE(_train_data->type); + dv_step = 1; + if( _missing_mask ) + ms_step = _missing_mask->step, mv_step = 1; + } + else + { + dv_step = _train_data->step/CV_ELEM_SIZE(_train_data->type); + ds_step = 1; + if( _missing_mask ) + mv_step = _missing_mask->step, ms_step = 1; + } + tflag = _tflag; + + sample_count = sample_all; + var_count = var_all; + + if( _sample_idx ) + { + CV_CALL( sample_indices = cvPreprocessIndexArray( _sample_idx, sample_all )); + sidx = sample_indices->data.i; + sample_count = sample_indices->rows + sample_indices->cols - 1; + } + + if( _var_idx ) + { + CV_CALL( var_idx = cvPreprocessIndexArray( _var_idx, var_all )); + vidx = var_idx->data.i; + var_count = var_idx->rows + var_idx->cols - 1; + } + + is_buf_16u = false; + if ( sample_count < 65536 ) + is_buf_16u = true; + + if( !CV_IS_MAT(_responses) || + (CV_MAT_TYPE(_responses->type) != CV_32SC1 && + CV_MAT_TYPE(_responses->type) != CV_32FC1) || + (_responses->rows != 1 && _responses->cols != 1) || + _responses->rows + _responses->cols - 1 != sample_all ) + CV_ERROR( CV_StsBadArg, "The array of _responses must be an integer or " + "floating-point vector containing as many elements as " + "the total number of samples in the training data matrix" ); + + r_type = CV_VAR_CATEGORICAL; + if( _var_type ) + CV_CALL( var_type0 = cvPreprocessVarType( _var_type, var_idx, var_count, &r_type )); + + CV_CALL( var_type = cvCreateMat( 1, var_count+2, CV_32SC1 )); + + cat_var_count = 0; + ord_var_count = -1; + + is_classifier = r_type == CV_VAR_CATEGORICAL; + + // step 0. calc the number of categorical vars + for( vi = 0; vi < var_count; vi++ ) + { + char vt = var_type0 ? var_type0->data.ptr[vi] : CV_VAR_ORDERED; + var_type->data.i[vi] = vt == CV_VAR_CATEGORICAL ? cat_var_count++ : ord_var_count--; + } + + ord_var_count = ~ord_var_count; + cv_n = params.cv_folds; + // set the two last elements of var_type array to be able + // to locate responses and cross-validation labels using + // the corresponding get_* functions. + var_type->data.i[var_count] = cat_var_count; + var_type->data.i[var_count+1] = cat_var_count+1; + + // in case of single ordered predictor we need dummy cv_labels + // for safe split_node_data() operation + have_labels = cv_n > 0 || (ord_var_count == 1 && cat_var_count == 0) || _add_labels; + + work_var_count = var_count + (is_classifier ? 1 : 0) // for responses class_labels + + (have_labels ? 1 : 0); // for cv_labels + + shared = _shared; + buf_count = shared ? 2 : 1; + + buf_size = -1; // the member buf_size is obsolete + + effective_buf_size = (uint64)(work_var_count + 1)*(uint64)sample_count * buf_count; // this is the total size of "CvMat buf" to be allocated + effective_buf_width = sample_count; + effective_buf_height = work_var_count+1; + + if (effective_buf_width >= effective_buf_height) + effective_buf_height *= buf_count; + else + effective_buf_width *= buf_count; + + if ((uint64)effective_buf_width * (uint64)effective_buf_height != effective_buf_size) + { + CV_Error(CV_StsBadArg, "The memory buffer cannot be allocated since its size exceeds integer fields limit"); + } + + + + if ( is_buf_16u ) + { + CV_CALL( buf = cvCreateMat( effective_buf_height, effective_buf_width, CV_16UC1 )); + CV_CALL( pair16u32s_ptr = (CvPair16u32s*)cvAlloc( sample_count*sizeof(pair16u32s_ptr[0]) )); + } + else + { + CV_CALL( buf = cvCreateMat( effective_buf_height, effective_buf_width, CV_32SC1 )); + CV_CALL( int_ptr = (int**)cvAlloc( sample_count*sizeof(int_ptr[0]) )); + } + + size = is_classifier ? (cat_var_count+1) : cat_var_count; + size = !size ? 1 : size; + CV_CALL( cat_count = cvCreateMat( 1, size, CV_32SC1 )); + CV_CALL( cat_ofs = cvCreateMat( 1, size, CV_32SC1 )); + + size = is_classifier ? (cat_var_count + 1)*params.max_categories : cat_var_count*params.max_categories; + size = !size ? 1 : size; + CV_CALL( cat_map = cvCreateMat( 1, size, CV_32SC1 )); + + // now calculate the maximum size of split, + // create memory storage that will keep nodes and splits of the decision tree + // allocate root node and the buffer for the whole training data + max_split_size = cvAlign(sizeof(CvDTreeSplit) + + (MAX(0,sample_count - 33)/32)*sizeof(int),sizeof(void*)); + tree_block_size = MAX((int)sizeof(CvDTreeNode)*8, max_split_size); + tree_block_size = MAX(tree_block_size + block_size_delta, min_block_size); + CV_CALL( tree_storage = cvCreateMemStorage( tree_block_size )); + CV_CALL( node_heap = cvCreateSet( 0, sizeof(*node_heap), sizeof(CvDTreeNode), tree_storage )); + + nv_size = var_count*sizeof(int); + nv_size = cvAlign(MAX( nv_size, (int)sizeof(CvSetElem) ), sizeof(void*)); + + temp_block_size = nv_size; + + if( cv_n ) + { + if( sample_count < cv_n*MAX(params.min_sample_count,10) ) + CV_ERROR( CV_StsOutOfRange, + "The many folds in cross-validation for such a small dataset" ); + + cv_size = cvAlign( cv_n*(sizeof(int) + sizeof(double)*2), sizeof(double) ); + temp_block_size = MAX(temp_block_size, cv_size); + } + + temp_block_size = MAX( temp_block_size + block_size_delta, min_block_size ); + CV_CALL( temp_storage = cvCreateMemStorage( temp_block_size )); + CV_CALL( nv_heap = cvCreateSet( 0, sizeof(*nv_heap), nv_size, temp_storage )); + if( cv_size ) + CV_CALL( cv_heap = cvCreateSet( 0, sizeof(*cv_heap), cv_size, temp_storage )); + + CV_CALL( data_root = new_node( 0, sample_count, 0, 0 )); + + max_c_count = 1; + + _fdst = 0; + _idst = 0; + if (ord_var_count) + _fdst = (float*)cvAlloc(sample_count*sizeof(_fdst[0])); + if (is_buf_16u && (cat_var_count || is_classifier)) + _idst = (int*)cvAlloc(sample_count*sizeof(_idst[0])); + + // transform the training data to convenient representation + for( vi = 0; vi <= var_count; vi++ ) + { + int ci; + const uchar* mask = 0; + int64 m_step = 0, step; + const int* idata = 0; + const float* fdata = 0; + int num_valid = 0; + + if( vi < var_count ) // analyze i-th input variable + { + int vi0 = vidx ? vidx[vi] : vi; + ci = get_var_type(vi); + step = ds_step; m_step = ms_step; + if( CV_MAT_TYPE(_train_data->type) == CV_32SC1 ) + idata = _train_data->data.i + vi0*dv_step; + else + fdata = _train_data->data.fl + vi0*dv_step; + if( _missing_mask ) + mask = _missing_mask->data.ptr + vi0*mv_step; + } + else // analyze _responses + { + ci = cat_var_count; + step = CV_IS_MAT_CONT(_responses->type) ? + 1 : _responses->step / CV_ELEM_SIZE(_responses->type); + if( CV_MAT_TYPE(_responses->type) == CV_32SC1 ) + idata = _responses->data.i; + else + fdata = _responses->data.fl; + } + + if( (vi < var_count && ci>=0) || + (vi == var_count && is_classifier) ) // process categorical variable or response + { + int c_count, prev_label; + int* c_map; + + if (is_buf_16u) + udst = (unsigned short*)(buf->data.s + vi*sample_count); + else + idst = buf->data.i + vi*sample_count; + + // copy data + for( i = 0; i < sample_count; i++ ) + { + int val = INT_MAX, si = sidx ? sidx[i] : i; + if( !mask || !mask[(size_t)si*m_step] ) + { + if( idata ) + val = idata[(size_t)si*step]; + else + { + float t = fdata[(size_t)si*step]; + val = cvRound(t); + if( fabs(t - val) > FLT_EPSILON ) + { + sprintf( err, "%d-th value of %d-th (categorical) " + "variable is not an integer", i, vi ); + CV_ERROR( CV_StsBadArg, err ); + } + } + + if( val == INT_MAX ) + { + sprintf( err, "%d-th value of %d-th (categorical) " + "variable is too large", i, vi ); + CV_ERROR( CV_StsBadArg, err ); + } + num_valid++; + } + if (is_buf_16u) + { + _idst[i] = val; + pair16u32s_ptr[i].u = udst + i; + pair16u32s_ptr[i].i = _idst + i; + } + else + { + idst[i] = val; + int_ptr[i] = idst + i; + } + } + + c_count = num_valid > 0; + if (is_buf_16u) + { + std::sort(pair16u32s_ptr, pair16u32s_ptr + sample_count, LessThanPairs()); + // count the categories + for( i = 1; i < num_valid; i++ ) + if (*pair16u32s_ptr[i].i != *pair16u32s_ptr[i-1].i) + c_count ++ ; + } + else + { + std::sort(int_ptr, int_ptr + sample_count, LessThanPtr()); + // count the categories + for( i = 1; i < num_valid; i++ ) + c_count += *int_ptr[i] != *int_ptr[i-1]; + } + + if( vi > 0 ) + max_c_count = MAX( max_c_count, c_count ); + cat_count->data.i[ci] = c_count; + cat_ofs->data.i[ci] = total_c_count; + + // resize cat_map, if need + if( cat_map->cols < total_c_count + c_count ) + { + tmp_map = cat_map; + CV_CALL( cat_map = cvCreateMat( 1, + MAX(cat_map->cols*3/2,total_c_count+c_count), CV_32SC1 )); + for( i = 0; i < total_c_count; i++ ) + cat_map->data.i[i] = tmp_map->data.i[i]; + cvReleaseMat( &tmp_map ); + } + + c_map = cat_map->data.i + total_c_count; + total_c_count += c_count; + + c_count = -1; + if (is_buf_16u) + { + // compact the class indices and build the map + prev_label = ~*pair16u32s_ptr[0].i; + for( i = 0; i < num_valid; i++ ) + { + int cur_label = *pair16u32s_ptr[i].i; + if( cur_label != prev_label ) + c_map[++c_count] = prev_label = cur_label; + *pair16u32s_ptr[i].u = (unsigned short)c_count; + } + // replace labels for missing values with -1 + for( ; i < sample_count; i++ ) + *pair16u32s_ptr[i].u = 65535; + } + else + { + // compact the class indices and build the map + prev_label = ~*int_ptr[0]; + for( i = 0; i < num_valid; i++ ) + { + int cur_label = *int_ptr[i]; + if( cur_label != prev_label ) + c_map[++c_count] = prev_label = cur_label; + *int_ptr[i] = c_count; + } + // replace labels for missing values with -1 + for( ; i < sample_count; i++ ) + *int_ptr[i] = -1; + } + } + else if( ci < 0 ) // process ordered variable + { + if (is_buf_16u) + udst = (unsigned short*)(buf->data.s + vi*sample_count); + else + idst = buf->data.i + vi*sample_count; + + for( i = 0; i < sample_count; i++ ) + { + float val = ord_nan; + int si = sidx ? sidx[i] : i; + if( !mask || !mask[(size_t)si*m_step] ) + { + if( idata ) + val = (float)idata[(size_t)si*step]; + else + val = fdata[(size_t)si*step]; + + if( fabs(val) >= ord_nan ) + { + sprintf( err, "%d-th value of %d-th (ordered) " + "variable (=%g) is too large", i, vi, val ); + CV_ERROR( CV_StsBadArg, err ); + } + num_valid++; + } + + if (is_buf_16u) + udst[i] = (unsigned short)i; // TODO: memory corruption may be here + else + idst[i] = i; + _fdst[i] = val; + + } + if (is_buf_16u) + std::sort(udst, udst + sample_count, LessThanIdx(_fdst)); + else + std::sort(idst, idst + sample_count, LessThanIdx(_fdst)); + } + + if( vi < var_count ) + data_root->set_num_valid(vi, num_valid); + } + + // set sample labels + if (is_buf_16u) + udst = (unsigned short*)(buf->data.s + work_var_count*sample_count); + else + idst = buf->data.i + work_var_count*sample_count; + + for (i = 0; i < sample_count; i++) + { + if (udst) + udst[i] = sidx ? (unsigned short)sidx[i] : (unsigned short)i; + else + idst[i] = sidx ? sidx[i] : i; + } + + if( cv_n ) + { + unsigned short* usdst = 0; + int* idst2 = 0; + + if (is_buf_16u) + { + usdst = (unsigned short*)(buf->data.s + (get_work_var_count()-1)*sample_count); + for( i = vi = 0; i < sample_count; i++ ) + { + usdst[i] = (unsigned short)vi++; + vi &= vi < cv_n ? -1 : 0; + } + + for( i = 0; i < sample_count; i++ ) + { + int a = (*rng)(sample_count); + int b = (*rng)(sample_count); + unsigned short unsh = (unsigned short)vi; + CV_SWAP( usdst[a], usdst[b], unsh ); + } + } + else + { + idst2 = buf->data.i + (get_work_var_count()-1)*sample_count; + for( i = vi = 0; i < sample_count; i++ ) + { + idst2[i] = vi++; + vi &= vi < cv_n ? -1 : 0; + } + + for( i = 0; i < sample_count; i++ ) + { + int a = (*rng)(sample_count); + int b = (*rng)(sample_count); + CV_SWAP( idst2[a], idst2[b], vi ); + } + } + } + + if ( cat_map ) + cat_map->cols = MAX( total_c_count, 1 ); + + max_split_size = cvAlign(sizeof(CvDTreeSplit) + + (MAX(0,max_c_count - 33)/32)*sizeof(int),sizeof(void*)); + CV_CALL( split_heap = cvCreateSet( 0, sizeof(*split_heap), max_split_size, tree_storage )); + + have_priors = is_classifier && params.priors; + if( is_classifier ) + { + int m = get_num_classes(); + double sum = 0; + CV_CALL( priors = cvCreateMat( 1, m, CV_64F )); + for( i = 0; i < m; i++ ) + { + double val = have_priors ? params.priors[i] : 1.; + if( val <= 0 ) + CV_ERROR( CV_StsOutOfRange, "Every class weight should be positive" ); + priors->data.db[i] = val; + sum += val; + } + + // normalize weights + if( have_priors ) + cvScale( priors, priors, 1./sum ); + + CV_CALL( priors_mult = cvCloneMat( priors )); + CV_CALL( counts = cvCreateMat( 1, m, CV_32SC1 )); + } + + + CV_CALL( direction = cvCreateMat( 1, sample_count, CV_8UC1 )); + CV_CALL( split_buf = cvCreateMat( 1, sample_count, CV_32SC1 )); + + __END__; + + if( data ) + delete data; + + if (_fdst) + cvFree( &_fdst ); + if (_idst) + cvFree( &_idst ); + cvFree( &int_ptr ); + cvFree( &pair16u32s_ptr); + cvReleaseMat( &var_type0 ); + cvReleaseMat( &sample_indices ); + cvReleaseMat( &tmp_map ); +} + +void CvDTreeTrainData::do_responses_copy() +{ + responses_copy = cvCreateMat( responses->rows, responses->cols, responses->type ); + cvCopy( responses, responses_copy); + responses = responses_copy; +} + +CvDTreeNode* CvDTreeTrainData::subsample_data( const CvMat* _subsample_idx ) +{ + CvDTreeNode* root = 0; + CvMat* isubsample_idx = 0; + CvMat* subsample_co = 0; + + bool isMakeRootCopy = true; + + CV_FUNCNAME( "CvDTreeTrainData::subsample_data" ); + + __BEGIN__; + + if( !data_root ) + CV_ERROR( CV_StsError, "No training data has been set" ); + + if( _subsample_idx ) + { + CV_CALL( isubsample_idx = cvPreprocessIndexArray( _subsample_idx, sample_count )); + + if( isubsample_idx->cols + isubsample_idx->rows - 1 == sample_count ) + { + const int* sidx = isubsample_idx->data.i; + for( int i = 0; i < sample_count; i++ ) + { + if( sidx[i] != i ) + { + isMakeRootCopy = false; + break; + } + } + } + else + isMakeRootCopy = false; + } + + if( isMakeRootCopy ) + { + // make a copy of the root node + CvDTreeNode temp; + int i; + root = new_node( 0, 1, 0, 0 ); + temp = *root; + *root = *data_root; + root->num_valid = temp.num_valid; + if( root->num_valid ) + { + for( i = 0; i < var_count; i++ ) + root->num_valid[i] = data_root->num_valid[i]; + } + root->cv_Tn = temp.cv_Tn; + root->cv_node_risk = temp.cv_node_risk; + root->cv_node_error = temp.cv_node_error; + } + else + { + int* sidx = isubsample_idx->data.i; + // co - array of count/offset pairs (to handle duplicated values in _subsample_idx) + int* co, cur_ofs = 0; + int vi, i; + int workVarCount = get_work_var_count(); + int count = isubsample_idx->rows + isubsample_idx->cols - 1; + + root = new_node( 0, count, 1, 0 ); + + CV_CALL( subsample_co = cvCreateMat( 1, sample_count*2, CV_32SC1 )); + cvZero( subsample_co ); + co = subsample_co->data.i; + for( i = 0; i < count; i++ ) + co[sidx[i]*2]++; + for( i = 0; i < sample_count; i++ ) + { + if( co[i*2] ) + { + co[i*2+1] = cur_ofs; + cur_ofs += co[i*2]; + } + else + co[i*2+1] = -1; + } + + cv::AutoBuffer inn_buf(sample_count*(2*sizeof(int) + sizeof(float))); + for( vi = 0; vi < workVarCount; vi++ ) + { + int ci = get_var_type(vi); + + if( ci >= 0 || vi >= var_count ) + { + int num_valid = 0; + const int* src = CvDTreeTrainData::get_cat_var_data( data_root, vi, (int*)(uchar*)inn_buf ); + + if (is_buf_16u) + { + unsigned short* udst = (unsigned short*)(buf->data.s + root->buf_idx*get_length_subbuf() + + vi*sample_count + root->offset); + for( i = 0; i < count; i++ ) + { + int val = src[sidx[i]]; + udst[i] = (unsigned short)val; + num_valid += val >= 0; + } + } + else + { + int* idst = buf->data.i + root->buf_idx*get_length_subbuf() + + vi*sample_count + root->offset; + for( i = 0; i < count; i++ ) + { + int val = src[sidx[i]]; + idst[i] = val; + num_valid += val >= 0; + } + } + + if( vi < var_count ) + root->set_num_valid(vi, num_valid); + } + else + { + int *src_idx_buf = (int*)(uchar*)inn_buf; + float *src_val_buf = (float*)(src_idx_buf + sample_count); + int* sample_indices_buf = (int*)(src_val_buf + sample_count); + const int* src_idx = 0; + const float* src_val = 0; + get_ord_var_data( data_root, vi, src_val_buf, src_idx_buf, &src_val, &src_idx, sample_indices_buf ); + int j = 0, idx, count_i; + int num_valid = data_root->get_num_valid(vi); + + if (is_buf_16u) + { + unsigned short* udst_idx = (unsigned short*)(buf->data.s + root->buf_idx*get_length_subbuf() + + vi*sample_count + data_root->offset); + for( i = 0; i < num_valid; i++ ) + { + idx = src_idx[i]; + count_i = co[idx*2]; + if( count_i ) + for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) + udst_idx[j] = (unsigned short)cur_ofs; + } + + root->set_num_valid(vi, j); + + for( ; i < sample_count; i++ ) + { + idx = src_idx[i]; + count_i = co[idx*2]; + if( count_i ) + for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) + udst_idx[j] = (unsigned short)cur_ofs; + } + } + else + { + int* idst_idx = buf->data.i + root->buf_idx*get_length_subbuf() + + vi*sample_count + root->offset; + for( i = 0; i < num_valid; i++ ) + { + idx = src_idx[i]; + count_i = co[idx*2]; + if( count_i ) + for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) + idst_idx[j] = cur_ofs; + } + + root->set_num_valid(vi, j); + + for( ; i < sample_count; i++ ) + { + idx = src_idx[i]; + count_i = co[idx*2]; + if( count_i ) + for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) + idst_idx[j] = cur_ofs; + } + } + } + } + // sample indices subsampling + const int* sample_idx_src = get_sample_indices(data_root, (int*)(uchar*)inn_buf); + if (is_buf_16u) + { + unsigned short* sample_idx_dst = (unsigned short*)(buf->data.s + root->buf_idx*get_length_subbuf() + + workVarCount*sample_count + root->offset); + for (i = 0; i < count; i++) + sample_idx_dst[i] = (unsigned short)sample_idx_src[sidx[i]]; + } + else + { + int* sample_idx_dst = buf->data.i + root->buf_idx*get_length_subbuf() + + workVarCount*sample_count + root->offset; + for (i = 0; i < count; i++) + sample_idx_dst[i] = sample_idx_src[sidx[i]]; + } + } + + __END__; + + cvReleaseMat( &isubsample_idx ); + cvReleaseMat( &subsample_co ); + + return root; +} + + +void CvDTreeTrainData::get_vectors( const CvMat* _subsample_idx, + float* values, uchar* missing, + float* _responses, bool get_class_idx ) +{ + CvMat* subsample_idx = 0; + CvMat* subsample_co = 0; + + CV_FUNCNAME( "CvDTreeTrainData::get_vectors" ); + + __BEGIN__; + + int i, vi, total = sample_count, count = total, cur_ofs = 0; + int* sidx = 0; + int* co = 0; + + cv::AutoBuffer inn_buf(sample_count*(2*sizeof(int) + sizeof(float))); + if( _subsample_idx ) + { + CV_CALL( subsample_idx = cvPreprocessIndexArray( _subsample_idx, sample_count )); + sidx = subsample_idx->data.i; + CV_CALL( subsample_co = cvCreateMat( 1, sample_count*2, CV_32SC1 )); + co = subsample_co->data.i; + cvZero( subsample_co ); + count = subsample_idx->cols + subsample_idx->rows - 1; + for( i = 0; i < count; i++ ) + co[sidx[i]*2]++; + for( i = 0; i < total; i++ ) + { + int count_i = co[i*2]; + if( count_i ) + { + co[i*2+1] = cur_ofs*var_count; + cur_ofs += count_i; + } + } + } + + if( missing ) + memset( missing, 1, count*var_count ); + + for( vi = 0; vi < var_count; vi++ ) + { + int ci = get_var_type(vi); + if( ci >= 0 ) // categorical + { + float* dst = values + vi; + uchar* m = missing ? missing + vi : 0; + const int* src = get_cat_var_data(data_root, vi, (int*)(uchar*)inn_buf); + + for( i = 0; i < count; i++, dst += var_count ) + { + int idx = sidx ? sidx[i] : i; + int val = src[idx]; + *dst = (float)val; + if( m ) + { + *m = (!is_buf_16u && val < 0) || (is_buf_16u && (val == 65535)); + m += var_count; + } + } + } + else // ordered + { + float* dst = values + vi; + uchar* m = missing ? missing + vi : 0; + int count1 = data_root->get_num_valid(vi); + float *src_val_buf = (float*)(uchar*)inn_buf; + int* src_idx_buf = (int*)(src_val_buf + sample_count); + int* sample_indices_buf = src_idx_buf + sample_count; + const float *src_val = 0; + const int* src_idx = 0; + get_ord_var_data(data_root, vi, src_val_buf, src_idx_buf, &src_val, &src_idx, sample_indices_buf); + + for( i = 0; i < count1; i++ ) + { + int idx = src_idx[i]; + int count_i = 1; + if( co ) + { + count_i = co[idx*2]; + cur_ofs = co[idx*2+1]; + } + else + cur_ofs = idx*var_count; + if( count_i ) + { + float val = src_val[i]; + for( ; count_i > 0; count_i--, cur_ofs += var_count ) + { + dst[cur_ofs] = val; + if( m ) + m[cur_ofs] = 0; + } + } + } + } + } + + // copy responses + if( _responses ) + { + if( is_classifier ) + { + const int* src = get_class_labels(data_root, (int*)(uchar*)inn_buf); + for( i = 0; i < count; i++ ) + { + int idx = sidx ? sidx[i] : i; + int val = get_class_idx ? src[idx] : + cat_map->data.i[cat_ofs->data.i[cat_var_count]+src[idx]]; + _responses[i] = (float)val; + } + } + else + { + float* val_buf = (float*)(uchar*)inn_buf; + int* sample_idx_buf = (int*)(val_buf + sample_count); + const float* _values = get_ord_responses(data_root, val_buf, sample_idx_buf); + for( i = 0; i < count; i++ ) + { + int idx = sidx ? sidx[i] : i; + _responses[i] = _values[idx]; + } + } + } + + __END__; + + cvReleaseMat( &subsample_idx ); + cvReleaseMat( &subsample_co ); +} + + +CvDTreeNode* CvDTreeTrainData::new_node( CvDTreeNode* parent, int count, + int storage_idx, int offset ) +{ + CvDTreeNode* node = (CvDTreeNode*)cvSetNew( node_heap ); + + node->sample_count = count; + node->depth = parent ? parent->depth + 1 : 0; + node->parent = parent; + node->left = node->right = 0; + node->split = 0; + node->value = 0; + node->class_idx = 0; + node->maxlr = 0.; + + node->buf_idx = storage_idx; + node->offset = offset; + if( nv_heap ) + node->num_valid = (int*)cvSetNew( nv_heap ); + else + node->num_valid = 0; + node->alpha = node->node_risk = node->tree_risk = node->tree_error = 0.; + node->complexity = 0; + + if( params.cv_folds > 0 && cv_heap ) + { + int cv_n = params.cv_folds; + node->Tn = INT_MAX; + node->cv_Tn = (int*)cvSetNew( cv_heap ); + node->cv_node_risk = (double*)cvAlignPtr(node->cv_Tn + cv_n, sizeof(double)); + node->cv_node_error = node->cv_node_risk + cv_n; + } + else + { + node->Tn = 0; + node->cv_Tn = 0; + node->cv_node_risk = 0; + node->cv_node_error = 0; + } + + return node; +} + + +CvDTreeSplit* CvDTreeTrainData::new_split_ord( int vi, float cmp_val, + int split_point, int inversed, float quality ) +{ + CvDTreeSplit* split = (CvDTreeSplit*)cvSetNew( split_heap ); + split->var_idx = vi; + split->condensed_idx = INT_MIN; + split->ord.c = cmp_val; + split->ord.split_point = split_point; + split->inversed = inversed; + split->quality = quality; + split->next = 0; + + return split; +} + + +CvDTreeSplit* CvDTreeTrainData::new_split_cat( int vi, float quality ) +{ + CvDTreeSplit* split = (CvDTreeSplit*)cvSetNew( split_heap ); + int i, n = (max_c_count + 31)/32; + + split->var_idx = vi; + split->condensed_idx = INT_MIN; + split->inversed = 0; + split->quality = quality; + for( i = 0; i < n; i++ ) + split->subset[i] = 0; + split->next = 0; + + return split; +} + + +void CvDTreeTrainData::free_node( CvDTreeNode* node ) +{ + CvDTreeSplit* split = node->split; + free_node_data( node ); + while( split ) + { + CvDTreeSplit* next = split->next; + cvSetRemoveByPtr( split_heap, split ); + split = next; + } + node->split = 0; + cvSetRemoveByPtr( node_heap, node ); +} + + +void CvDTreeTrainData::free_node_data( CvDTreeNode* node ) +{ + if( node->num_valid ) + { + cvSetRemoveByPtr( nv_heap, node->num_valid ); + node->num_valid = 0; + } + // do not free cv_* fields, as all the cross-validation related data is released at once. +} + + +void CvDTreeTrainData::free_train_data() +{ + cvReleaseMat( &counts ); + cvReleaseMat( &buf ); + cvReleaseMat( &direction ); + cvReleaseMat( &split_buf ); + cvReleaseMemStorage( &temp_storage ); + cvReleaseMat( &responses_copy ); + cv_heap = nv_heap = 0; +} + + +void CvDTreeTrainData::clear() +{ + free_train_data(); + + cvReleaseMemStorage( &tree_storage ); + + cvReleaseMat( &var_idx ); + cvReleaseMat( &var_type ); + cvReleaseMat( &cat_count ); + cvReleaseMat( &cat_ofs ); + cvReleaseMat( &cat_map ); + cvReleaseMat( &priors ); + cvReleaseMat( &priors_mult ); + + node_heap = split_heap = 0; + + sample_count = var_all = var_count = max_c_count = ord_var_count = cat_var_count = 0; + have_labels = have_priors = is_classifier = false; + + buf_count = buf_size = 0; + shared = false; + + data_root = 0; + + rng = &cv::theRNG(); +} + + +int CvDTreeTrainData::get_num_classes() const +{ + return is_classifier ? cat_count->data.i[cat_var_count] : 0; +} + + +int CvDTreeTrainData::get_var_type(int vi) const +{ + return var_type->data.i[vi]; +} + +void CvDTreeTrainData::get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* sorted_indices_buf, + const float** ord_values, const int** sorted_indices, int* sample_indices_buf ) +{ + int vidx = var_idx ? var_idx->data.i[vi] : vi; + int node_sample_count = n->sample_count; + int td_step = train_data->step/CV_ELEM_SIZE(train_data->type); + + const int* sample_indices = get_sample_indices(n, sample_indices_buf); + + if( !is_buf_16u ) + *sorted_indices = buf->data.i + n->buf_idx*get_length_subbuf() + + vi*sample_count + n->offset; + else { + const unsigned short* short_indices = (const unsigned short*)(buf->data.s + n->buf_idx*get_length_subbuf() + + vi*sample_count + n->offset ); + for( int i = 0; i < node_sample_count; i++ ) + sorted_indices_buf[i] = short_indices[i]; + *sorted_indices = sorted_indices_buf; + } + + if( tflag == CV_ROW_SAMPLE ) + { + for( int i = 0; i < node_sample_count && + ((((*sorted_indices)[i] >= 0) && !is_buf_16u) || (((*sorted_indices)[i] != 65535) && is_buf_16u)); i++ ) + { + int idx = (*sorted_indices)[i]; + idx = sample_indices[idx]; + ord_values_buf[i] = *(train_data->data.fl + idx * td_step + vidx); + } + } + else + for( int i = 0; i < node_sample_count && + ((((*sorted_indices)[i] >= 0) && !is_buf_16u) || (((*sorted_indices)[i] != 65535) && is_buf_16u)); i++ ) + { + int idx = (*sorted_indices)[i]; + idx = sample_indices[idx]; + ord_values_buf[i] = *(train_data->data.fl + vidx* td_step + idx); + } + + *ord_values = ord_values_buf; +} + + +const int* CvDTreeTrainData::get_class_labels( CvDTreeNode* n, int* labels_buf ) +{ + if (is_classifier) + return get_cat_var_data( n, var_count, labels_buf); + return 0; +} + +const int* CvDTreeTrainData::get_sample_indices( CvDTreeNode* n, int* indices_buf ) +{ + return get_cat_var_data( n, get_work_var_count(), indices_buf ); +} + +const float* CvDTreeTrainData::get_ord_responses( CvDTreeNode* n, float* values_buf, int*sample_indices_buf ) +{ + int _sample_count = n->sample_count; + int r_step = CV_IS_MAT_CONT(responses->type) ? 1 : responses->step/CV_ELEM_SIZE(responses->type); + const int* indices = get_sample_indices(n, sample_indices_buf); + + for( int i = 0; i < _sample_count && + (((indices[i] >= 0) && !is_buf_16u) || ((indices[i] != 65535) && is_buf_16u)); i++ ) + { + int idx = indices[i]; + values_buf[i] = *(responses->data.fl + idx * r_step); + } + + return values_buf; +} + + +const int* CvDTreeTrainData::get_cv_labels( CvDTreeNode* n, int* labels_buf ) +{ + if (have_labels) + return get_cat_var_data( n, get_work_var_count()- 1, labels_buf); + return 0; +} + + +const int* CvDTreeTrainData::get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf) +{ + const int* cat_values = 0; + if( !is_buf_16u ) + cat_values = buf->data.i + n->buf_idx*get_length_subbuf() + + vi*sample_count + n->offset; + else { + const unsigned short* short_values = (const unsigned short*)(buf->data.s + n->buf_idx*get_length_subbuf() + + vi*sample_count + n->offset); + for( int i = 0; i < n->sample_count; i++ ) + cat_values_buf[i] = short_values[i]; + cat_values = cat_values_buf; + } + return cat_values; +} + + +int CvDTreeTrainData::get_child_buf_idx( CvDTreeNode* n ) +{ + int idx = n->buf_idx + 1; + if( idx >= buf_count ) + idx = shared ? 1 : 0; + return idx; +} + + +void CvDTreeTrainData::write_params( CvFileStorage* fs ) const +{ + CV_FUNCNAME( "CvDTreeTrainData::write_params" ); + + __BEGIN__; + + int vi, vcount = var_count; + + cvWriteInt( fs, "is_classifier", is_classifier ? 1 : 0 ); + cvWriteInt( fs, "var_all", var_all ); + cvWriteInt( fs, "var_count", var_count ); + cvWriteInt( fs, "ord_var_count", ord_var_count ); + cvWriteInt( fs, "cat_var_count", cat_var_count ); + + cvStartWriteStruct( fs, "training_params", CV_NODE_MAP ); + cvWriteInt( fs, "use_surrogates", params.use_surrogates ? 1 : 0 ); + + if( is_classifier ) + { + cvWriteInt( fs, "max_categories", params.max_categories ); + } + else + { + cvWriteReal( fs, "regression_accuracy", params.regression_accuracy ); + } + + cvWriteInt( fs, "max_depth", params.max_depth ); + cvWriteInt( fs, "min_sample_count", params.min_sample_count ); + cvWriteInt( fs, "cross_validation_folds", params.cv_folds ); + + if( params.cv_folds > 1 ) + { + cvWriteInt( fs, "use_1se_rule", params.use_1se_rule ? 1 : 0 ); + cvWriteInt( fs, "truncate_pruned_tree", params.truncate_pruned_tree ? 1 : 0 ); + } + + if( priors ) + cvWrite( fs, "priors", priors ); + + cvEndWriteStruct( fs ); + + if( var_idx ) + cvWrite( fs, "var_idx", var_idx ); + + cvStartWriteStruct( fs, "var_type", CV_NODE_SEQ+CV_NODE_FLOW ); + + for( vi = 0; vi < vcount; vi++ ) + cvWriteInt( fs, 0, var_type->data.i[vi] >= 0 ); + + cvEndWriteStruct( fs ); + + if( cat_count && (cat_var_count > 0 || is_classifier) ) + { + CV_ASSERT( cat_count != 0 ); + cvWrite( fs, "cat_count", cat_count ); + cvWrite( fs, "cat_map", cat_map ); + } + + __END__; +} + + +void CvDTreeTrainData::read_params( CvFileStorage* fs, CvFileNode* node ) +{ + CV_FUNCNAME( "CvDTreeTrainData::read_params" ); + + __BEGIN__; + + CvFileNode *tparams_node, *vartype_node; + CvSeqReader reader; + int vi, max_split_size, tree_block_size; + + is_classifier = (cvReadIntByName( fs, node, "is_classifier" ) != 0); + var_all = cvReadIntByName( fs, node, "var_all" ); + var_count = cvReadIntByName( fs, node, "var_count", var_all ); + cat_var_count = cvReadIntByName( fs, node, "cat_var_count" ); + ord_var_count = cvReadIntByName( fs, node, "ord_var_count" ); + + tparams_node = cvGetFileNodeByName( fs, node, "training_params" ); + + if( tparams_node ) // training parameters are not necessary + { + params.use_surrogates = cvReadIntByName( fs, tparams_node, "use_surrogates", 1 ) != 0; + + if( is_classifier ) + { + params.max_categories = cvReadIntByName( fs, tparams_node, "max_categories" ); + } + else + { + params.regression_accuracy = + (float)cvReadRealByName( fs, tparams_node, "regression_accuracy" ); + } + + params.max_depth = cvReadIntByName( fs, tparams_node, "max_depth" ); + params.min_sample_count = cvReadIntByName( fs, tparams_node, "min_sample_count" ); + params.cv_folds = cvReadIntByName( fs, tparams_node, "cross_validation_folds" ); + + if( params.cv_folds > 1 ) + { + params.use_1se_rule = cvReadIntByName( fs, tparams_node, "use_1se_rule" ) != 0; + params.truncate_pruned_tree = + cvReadIntByName( fs, tparams_node, "truncate_pruned_tree" ) != 0; + } + + priors = (CvMat*)cvReadByName( fs, tparams_node, "priors" ); + if( priors ) + { + if( !CV_IS_MAT(priors) ) + CV_ERROR( CV_StsParseError, "priors must stored as a matrix" ); + priors_mult = cvCloneMat( priors ); + } + } + + CV_CALL( var_idx = (CvMat*)cvReadByName( fs, node, "var_idx" )); + if( var_idx ) + { + if( !CV_IS_MAT(var_idx) || + (var_idx->cols != 1 && var_idx->rows != 1) || + var_idx->cols + var_idx->rows - 1 != var_count || + CV_MAT_TYPE(var_idx->type) != CV_32SC1 ) + CV_ERROR( CV_StsParseError, + "var_idx (if exist) must be valid 1d integer vector containing elements" ); + + for( vi = 0; vi < var_count; vi++ ) + if( (unsigned)var_idx->data.i[vi] >= (unsigned)var_all ) + CV_ERROR( CV_StsOutOfRange, "some of var_idx elements are out of range" ); + } + + ////// read var type + CV_CALL( var_type = cvCreateMat( 1, var_count + 2, CV_32SC1 )); + + cat_var_count = 0; + ord_var_count = -1; + vartype_node = cvGetFileNodeByName( fs, node, "var_type" ); + + if( vartype_node && CV_NODE_TYPE(vartype_node->tag) == CV_NODE_INT && var_count == 1 ) + var_type->data.i[0] = vartype_node->data.i ? cat_var_count++ : ord_var_count--; + else + { + if( !vartype_node || CV_NODE_TYPE(vartype_node->tag) != CV_NODE_SEQ || + vartype_node->data.seq->total != var_count ) + CV_ERROR( CV_StsParseError, "var_type must exist and be a sequence of 0's and 1's" ); + + cvStartReadSeq( vartype_node->data.seq, &reader ); + + for( vi = 0; vi < var_count; vi++ ) + { + CvFileNode* n = (CvFileNode*)reader.ptr; + if( CV_NODE_TYPE(n->tag) != CV_NODE_INT || (n->data.i & ~1) ) + CV_ERROR( CV_StsParseError, "var_type must exist and be a sequence of 0's and 1's" ); + var_type->data.i[vi] = n->data.i ? cat_var_count++ : ord_var_count--; + CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + } + } + var_type->data.i[var_count] = cat_var_count; + + ord_var_count = ~ord_var_count; + ////// + + if( cat_var_count > 0 || is_classifier ) + { + int ccount, total_c_count = 0; + CV_CALL( cat_count = (CvMat*)cvReadByName( fs, node, "cat_count" )); + CV_CALL( cat_map = (CvMat*)cvReadByName( fs, node, "cat_map" )); + + if( !CV_IS_MAT(cat_count) || !CV_IS_MAT(cat_map) || + (cat_count->cols != 1 && cat_count->rows != 1) || + CV_MAT_TYPE(cat_count->type) != CV_32SC1 || + cat_count->cols + cat_count->rows - 1 != cat_var_count + is_classifier || + (cat_map->cols != 1 && cat_map->rows != 1) || + CV_MAT_TYPE(cat_map->type) != CV_32SC1 ) + CV_ERROR( CV_StsParseError, + "Both cat_count and cat_map must exist and be valid 1d integer vectors of an appropriate size" ); + + ccount = cat_var_count + is_classifier; + + CV_CALL( cat_ofs = cvCreateMat( 1, ccount + 1, CV_32SC1 )); + cat_ofs->data.i[0] = 0; + max_c_count = 1; + + for( vi = 0; vi < ccount; vi++ ) + { + int val = cat_count->data.i[vi]; + if( val <= 0 ) + CV_ERROR( CV_StsOutOfRange, "some of cat_count elements are out of range" ); + max_c_count = MAX( max_c_count, val ); + cat_ofs->data.i[vi+1] = total_c_count += val; + } + + if( cat_map->cols + cat_map->rows - 1 != total_c_count ) + CV_ERROR( CV_StsBadSize, + "cat_map vector length is not equal to the total number of categories in all categorical vars" ); + } + + max_split_size = cvAlign(sizeof(CvDTreeSplit) + + (MAX(0,max_c_count - 33)/32)*sizeof(int),sizeof(void*)); + + tree_block_size = MAX((int)sizeof(CvDTreeNode)*8, max_split_size); + tree_block_size = MAX(tree_block_size + block_size_delta, min_block_size); + CV_CALL( tree_storage = cvCreateMemStorage( tree_block_size )); + CV_CALL( node_heap = cvCreateSet( 0, sizeof(node_heap[0]), + sizeof(CvDTreeNode), tree_storage )); + CV_CALL( split_heap = cvCreateSet( 0, sizeof(split_heap[0]), + max_split_size, tree_storage )); + + __END__; +} + +/////////////////////// Decision Tree ///////////////////////// +CvDTreeParams::CvDTreeParams() : max_categories(10), max_depth(INT_MAX), min_sample_count(10), + cv_folds(10), use_surrogates(true), use_1se_rule(true), + truncate_pruned_tree(true), regression_accuracy(0.01f), priors(0) +{} + +CvDTreeParams::CvDTreeParams( int _max_depth, int _min_sample_count, + float _regression_accuracy, bool _use_surrogates, + int _max_categories, int _cv_folds, + bool _use_1se_rule, bool _truncate_pruned_tree, + const float* _priors ) : + max_categories(_max_categories), max_depth(_max_depth), + min_sample_count(_min_sample_count), cv_folds (_cv_folds), + use_surrogates(_use_surrogates), use_1se_rule(_use_1se_rule), + truncate_pruned_tree(_truncate_pruned_tree), + regression_accuracy(_regression_accuracy), + priors(_priors) +{} + +CvDTree::CvDTree() +{ + data = 0; + var_importance = 0; + default_model_name = "my_tree"; + + clear(); +} + + +void CvDTree::clear() +{ + cvReleaseMat( &var_importance ); + if( data ) + { + if( !data->shared ) + delete data; + else + free_tree(); + data = 0; + } + root = 0; + pruned_tree_idx = -1; +} + + +CvDTree::~CvDTree() +{ + clear(); +} + + +const CvDTreeNode* CvDTree::get_root() const +{ + return root; +} + + +int CvDTree::get_pruned_tree_idx() const +{ + return pruned_tree_idx; +} + + +CvDTreeTrainData* CvDTree::get_data() +{ + return data; +} + + +bool CvDTree::train( const CvMat* _train_data, int _tflag, + const CvMat* _responses, const CvMat* _var_idx, + const CvMat* _sample_idx, const CvMat* _var_type, + const CvMat* _missing_mask, CvDTreeParams _params ) +{ + bool result = false; + + CV_FUNCNAME( "CvDTree::train" ); + + __BEGIN__; + + clear(); + data = new CvDTreeTrainData( _train_data, _tflag, _responses, + _var_idx, _sample_idx, _var_type, + _missing_mask, _params, false ); + CV_CALL( result = do_train(0) ); + + __END__; + + return result; +} + +bool CvDTree::train( const Mat& _train_data, int _tflag, + const Mat& _responses, const Mat& _var_idx, + const Mat& _sample_idx, const Mat& _var_type, + const Mat& _missing_mask, CvDTreeParams _params ) +{ + train_data_hdr = _train_data; + train_data_mat = _train_data; + responses_hdr = _responses; + responses_mat = _responses; + + CvMat vidx=_var_idx, sidx=_sample_idx, vtype=_var_type, mmask=_missing_mask; + + return train(&train_data_hdr, _tflag, &responses_hdr, vidx.data.ptr ? &vidx : 0, sidx.data.ptr ? &sidx : 0, + vtype.data.ptr ? &vtype : 0, mmask.data.ptr ? &mmask : 0, _params); +} + + +bool CvDTree::train( CvMLData* _data, CvDTreeParams _params ) +{ + bool result = false; + + CV_FUNCNAME( "CvDTree::train" ); + + __BEGIN__; + + const CvMat* values = _data->get_values(); + const CvMat* response = _data->get_responses(); + const CvMat* missing = _data->get_missing(); + const CvMat* var_types = _data->get_var_types(); + const CvMat* train_sidx = _data->get_train_sample_idx(); + const CvMat* var_idx = _data->get_var_idx(); + + CV_CALL( result = train( values, CV_ROW_SAMPLE, response, var_idx, + train_sidx, var_types, missing, _params ) ); + + __END__; + + return result; +} + +bool CvDTree::train( CvDTreeTrainData* _data, const CvMat* _subsample_idx ) +{ + bool result = false; + + CV_FUNCNAME( "CvDTree::train" ); + + __BEGIN__; + + clear(); + data = _data; + data->shared = true; + CV_CALL( result = do_train(_subsample_idx)); + + __END__; + + return result; +} + + +bool CvDTree::do_train( const CvMat* _subsample_idx ) +{ + bool result = false; + + CV_FUNCNAME( "CvDTree::do_train" ); + + __BEGIN__; + + root = data->subsample_data( _subsample_idx ); + + CV_CALL( try_split_node(root)); + + if( root->split ) + { + CV_Assert( root->left ); + CV_Assert( root->right ); + + if( data->params.cv_folds > 0 ) + CV_CALL( prune_cv() ); + + if( !data->shared ) + data->free_train_data(); + + result = true; + } + + __END__; + + return result; +} + + +void CvDTree::try_split_node( CvDTreeNode* node ) +{ + CvDTreeSplit* best_split = 0; + int i, n = node->sample_count, vi; + bool can_split = true; + double quality_scale; + + calc_node_value( node ); + + if( node->sample_count <= data->params.min_sample_count || + node->depth >= data->params.max_depth ) + can_split = false; + + if( can_split && data->is_classifier ) + { + // check if we have a "pure" node, + // we assume that cls_count is filled by calc_node_value() + int* cls_count = data->counts->data.i; + int nz = 0, m = data->get_num_classes(); + for( i = 0; i < m; i++ ) + nz += cls_count[i] != 0; + if( nz == 1 ) // there is only one class + can_split = false; + } + else if( can_split ) + { + if( sqrt(node->node_risk)/n < data->params.regression_accuracy ) + can_split = false; + } + + if( can_split ) + { + best_split = find_best_split(node); + // TODO: check the split quality ... + node->split = best_split; + } + if( !can_split || !best_split ) + { + data->free_node_data(node); + return; + } + + quality_scale = calc_node_dir( node ); + if( data->params.use_surrogates ) + { + // find all the surrogate splits + // and sort them by their similarity to the primary one + for( vi = 0; vi < data->var_count; vi++ ) + { + CvDTreeSplit* split; + int ci = data->get_var_type(vi); + + if( vi == best_split->var_idx ) + continue; + + if( ci >= 0 ) + split = find_surrogate_split_cat( node, vi ); + else + split = find_surrogate_split_ord( node, vi ); + + if( split ) + { + // insert the split + CvDTreeSplit* prev_split = node->split; + split->quality = (float)(split->quality*quality_scale); + + while( prev_split->next && + prev_split->next->quality > split->quality ) + prev_split = prev_split->next; + split->next = prev_split->next; + prev_split->next = split; + } + } + } + split_node_data( node ); + try_split_node( node->left ); + try_split_node( node->right ); +} + + +// calculate direction (left(-1),right(1),missing(0)) +// for each sample using the best split +// the function returns scale coefficients for surrogate split quality factors. +// the scale is applied to normalize surrogate split quality relatively to the +// best (primary) split quality. That is, if a surrogate split is absolutely +// identical to the primary split, its quality will be set to the maximum value = +// quality of the primary split; otherwise, it will be lower. +// besides, the function compute node->maxlr, +// minimum possible quality (w/o considering the above mentioned scale) +// for a surrogate split. Surrogate splits with quality less than node->maxlr +// are not discarded. +double CvDTree::calc_node_dir( CvDTreeNode* node ) +{ + char* dir = (char*)data->direction->data.ptr; + int i, n = node->sample_count, vi = node->split->var_idx; + double L, R; + + assert( !node->split->inversed ); + + if( data->get_var_type(vi) >= 0 ) // split on categorical var + { + cv::AutoBuffer inn_buf(n*(!data->have_priors ? 1 : 2)); + int* labels_buf = (int*)inn_buf; + const int* labels = data->get_cat_var_data( node, vi, labels_buf ); + const int* subset = node->split->subset; + if( !data->have_priors ) + { + int sum = 0, sum_abs = 0; + + for( i = 0; i < n; i++ ) + { + int idx = labels[i]; + int d = ( ((idx >= 0)&&(!data->is_buf_16u)) || ((idx != 65535)&&(data->is_buf_16u)) ) ? + CV_DTREE_CAT_DIR(idx,subset) : 0; + sum += d; sum_abs += d & 1; + dir[i] = (char)d; + } + + R = (sum_abs + sum) >> 1; + L = (sum_abs - sum) >> 1; + } + else + { + const double* priors = data->priors_mult->data.db; + double sum = 0, sum_abs = 0; + int* responses_buf = labels_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + + for( i = 0; i < n; i++ ) + { + int idx = labels[i]; + double w = priors[responses[i]]; + int d = idx >= 0 ? CV_DTREE_CAT_DIR(idx,subset) : 0; + sum += d*w; sum_abs += (d & 1)*w; + dir[i] = (char)d; + } + + R = (sum_abs + sum) * 0.5; + L = (sum_abs - sum) * 0.5; + } + } + else // split on ordered var + { + int split_point = node->split->ord.split_point; + int n1 = node->get_num_valid(vi); + cv::AutoBuffer inn_buf(n*(sizeof(int)*(data->have_priors ? 3 : 2) + sizeof(float))); + float* val_buf = (float*)(uchar*)inn_buf; + int* sorted_buf = (int*)(val_buf + n); + int* sample_idx_buf = sorted_buf + n; + const float* val = 0; + const int* sorted = 0; + data->get_ord_var_data( node, vi, val_buf, sorted_buf, &val, &sorted, sample_idx_buf); + + assert( 0 <= split_point && split_point < n1-1 ); + + if( !data->have_priors ) + { + for( i = 0; i <= split_point; i++ ) + dir[sorted[i]] = (char)-1; + for( ; i < n1; i++ ) + dir[sorted[i]] = (char)1; + for( ; i < n; i++ ) + dir[sorted[i]] = (char)0; + + L = split_point-1; + R = n1 - split_point + 1; + } + else + { + const double* priors = data->priors_mult->data.db; + int* responses_buf = sample_idx_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + L = R = 0; + + for( i = 0; i <= split_point; i++ ) + { + int idx = sorted[i]; + double w = priors[responses[idx]]; + dir[idx] = (char)-1; + L += w; + } + + for( ; i < n1; i++ ) + { + int idx = sorted[i]; + double w = priors[responses[idx]]; + dir[idx] = (char)1; + R += w; + } + + for( ; i < n; i++ ) + dir[sorted[i]] = (char)0; + } + } + node->maxlr = MAX( L, R ); + return node->split->quality/(L + R); +} + + +namespace cv +{ + +template<> CV_EXPORTS void DefaultDeleter::operator ()(CvDTreeSplit* obj) const +{ + fastFree(obj); +} + +DTreeBestSplitFinder::DTreeBestSplitFinder( CvDTree* _tree, CvDTreeNode* _node) +{ + tree = _tree; + node = _node; + splitSize = tree->get_data()->split_heap->elem_size; + + bestSplit.reset((CvDTreeSplit*)fastMalloc(splitSize)); + memset(bestSplit.get(), 0, splitSize); + bestSplit->quality = -1; + bestSplit->condensed_idx = INT_MIN; + split.reset((CvDTreeSplit*)fastMalloc(splitSize)); + memset(split.get(), 0, splitSize); + //haveSplit = false; +} + +DTreeBestSplitFinder::DTreeBestSplitFinder( const DTreeBestSplitFinder& finder, Split ) +{ + tree = finder.tree; + node = finder.node; + splitSize = tree->get_data()->split_heap->elem_size; + + bestSplit.reset((CvDTreeSplit*)fastMalloc(splitSize)); + memcpy(bestSplit.get(), finder.bestSplit.get(), splitSize); + split.reset((CvDTreeSplit*)fastMalloc(splitSize)); + memset(split.get(), 0, splitSize); +} + +void DTreeBestSplitFinder::operator()(const BlockedRange& range) +{ + int vi, vi1 = range.begin(), vi2 = range.end(); + int n = node->sample_count; + CvDTreeTrainData* data = tree->get_data(); + AutoBuffer inn_buf(2*n*(sizeof(int) + sizeof(float))); + + for( vi = vi1; vi < vi2; vi++ ) + { + CvDTreeSplit *res; + int ci = data->get_var_type(vi); + if( node->get_num_valid(vi) <= 1 ) + continue; + + if( data->is_classifier ) + { + if( ci >= 0 ) + res = tree->find_split_cat_class( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); + else + res = tree->find_split_ord_class( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); + } + else + { + if( ci >= 0 ) + res = tree->find_split_cat_reg( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); + else + res = tree->find_split_ord_reg( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); + } + + if( res && bestSplit->quality < split->quality ) + memcpy( bestSplit.get(), split.get(), splitSize ); + } +} + +void DTreeBestSplitFinder::join( DTreeBestSplitFinder& rhs ) +{ + if( bestSplit->quality < rhs.bestSplit->quality ) + memcpy( bestSplit.get(), rhs.bestSplit.get(), splitSize ); +} +} + + +CvDTreeSplit* CvDTree::find_best_split( CvDTreeNode* node ) +{ + DTreeBestSplitFinder finder( this, node ); + + cv::parallel_reduce(cv::BlockedRange(0, data->var_count), finder); + + CvDTreeSplit *bestSplit = 0; + if( finder.bestSplit->quality > 0 ) + { + bestSplit = data->new_split_cat( 0, -1.0f ); + memcpy( bestSplit, finder.bestSplit, finder.splitSize ); + } + + return bestSplit; +} + +CvDTreeSplit* CvDTree::find_split_ord_class( CvDTreeNode* node, int vi, + float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) +{ + const float epsilon = FLT_EPSILON*2; + int n = node->sample_count; + int n1 = node->get_num_valid(vi); + int m = data->get_num_classes(); + + int base_size = 2*m*sizeof(int); + cv::AutoBuffer inn_buf(base_size); + if( !_ext_buf ) + inn_buf.allocate(base_size + n*(3*sizeof(int)+sizeof(float))); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; + float* values_buf = (float*)ext_buf; + int* sorted_indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = sorted_indices_buf + n; + const float* values = 0; + const int* sorted_indices = 0; + data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, + &sorted_indices, sample_indices_buf ); + int* responses_buf = sample_indices_buf + n; + const int* responses = data->get_class_labels( node, responses_buf ); + + const int* rc0 = data->counts->data.i; + int* lc = (int*)base_buf; + int* rc = lc + m; + int i, best_i = -1; + double lsum2 = 0, rsum2 = 0, best_val = init_quality; + const double* priors = data->have_priors ? data->priors_mult->data.db : 0; + + // init arrays of class instance counters on both sides of the split + for( i = 0; i < m; i++ ) + { + lc[i] = 0; + rc[i] = rc0[i]; + } + + // compensate for missing values + for( i = n1; i < n; i++ ) + { + rc[responses[sorted_indices[i]]]--; + } + + if( !priors ) + { + int L = 0, R = n1; + + for( i = 0; i < m; i++ ) + rsum2 += (double)rc[i]*rc[i]; + + for( i = 0; i < n1 - 1; i++ ) + { + int idx = responses[sorted_indices[i]]; + int lv, rv; + L++; R--; + lv = lc[idx]; rv = rc[idx]; + lsum2 += lv*2 + 1; + rsum2 -= rv*2 - 1; + lc[idx] = lv + 1; rc[idx] = rv - 1; + + if( values[i] + epsilon < values[i+1] ) + { + double val = (lsum2*R + rsum2*L)/((double)L*R); + if( best_val < val ) + { + best_val = val; + best_i = i; + } + } + } + } + else + { + double L = 0, R = 0; + for( i = 0; i < m; i++ ) + { + double wv = rc[i]*priors[i]; + R += wv; + rsum2 += wv*wv; + } + + for( i = 0; i < n1 - 1; i++ ) + { + int idx = responses[sorted_indices[i]]; + int lv, rv; + double p = priors[idx], p2 = p*p; + L += p; R -= p; + lv = lc[idx]; rv = rc[idx]; + lsum2 += p2*(lv*2 + 1); + rsum2 -= p2*(rv*2 - 1); + lc[idx] = lv + 1; rc[idx] = rv - 1; + + if( values[i] + epsilon < values[i+1] ) + { + double val = (lsum2*R + rsum2*L)/((double)L*R); + if( best_val < val ) + { + best_val = val; + best_i = i; + } + } + } + } + + CvDTreeSplit* split = 0; + if( best_i >= 0 ) + { + split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); + split->var_idx = vi; + split->ord.c = (values[best_i] + values[best_i+1])*0.5f; + split->ord.split_point = best_i; + split->inversed = 0; + split->quality = (float)best_val; + } + return split; +} + + +void CvDTree::cluster_categories( const int* vectors, int n, int m, + int* csums, int k, int* labels ) +{ + // TODO: consider adding priors (class weights) and sample weights to the clustering algorithm + int iters = 0, max_iters = 100; + int i, j, idx; + cv::AutoBuffer buf(n + k); + double *v_weights = buf, *c_weights = buf + n; + bool modified = true; + RNG* r = data->rng; + + // assign labels randomly + for( i = 0; i < n; i++ ) + { + int sum = 0; + const int* v = vectors + i*m; + labels[i] = i < k ? i : r->uniform(0, k); + + // compute weight of each vector + for( j = 0; j < m; j++ ) + sum += v[j]; + v_weights[i] = sum ? 1./sum : 0.; + } + + for( i = 0; i < n; i++ ) + { + int i1 = (*r)(n); + int i2 = (*r)(n); + CV_SWAP( labels[i1], labels[i2], j ); + } + + for( iters = 0; iters <= max_iters; iters++ ) + { + // calculate csums + for( i = 0; i < k; i++ ) + { + for( j = 0; j < m; j++ ) + csums[i*m + j] = 0; + } + + for( i = 0; i < n; i++ ) + { + const int* v = vectors + i*m; + int* s = csums + labels[i]*m; + for( j = 0; j < m; j++ ) + s[j] += v[j]; + } + + // exit the loop here, when we have up-to-date csums + if( iters == max_iters || !modified ) + break; + + modified = false; + + // calculate weight of each cluster + for( i = 0; i < k; i++ ) + { + const int* s = csums + i*m; + int sum = 0; + for( j = 0; j < m; j++ ) + sum += s[j]; + c_weights[i] = sum ? 1./sum : 0; + } + + // now for each vector determine the closest cluster + for( i = 0; i < n; i++ ) + { + const int* v = vectors + i*m; + double alpha = v_weights[i]; + double min_dist2 = DBL_MAX; + int min_idx = -1; + + for( idx = 0; idx < k; idx++ ) + { + const int* s = csums + idx*m; + double dist2 = 0., beta = c_weights[idx]; + for( j = 0; j < m; j++ ) + { + double t = v[j]*alpha - s[j]*beta; + dist2 += t*t; + } + if( min_dist2 > dist2 ) + { + min_dist2 = dist2; + min_idx = idx; + } + } + + if( min_idx != labels[i] ) + modified = true; + labels[i] = min_idx; + } + } +} + + +CvDTreeSplit* CvDTree::find_split_cat_class( CvDTreeNode* node, int vi, float init_quality, + CvDTreeSplit* _split, uchar* _ext_buf ) +{ + int ci = data->get_var_type(vi); + int n = node->sample_count; + int m = data->get_num_classes(); + int _mi = data->cat_count->data.i[ci], mi = _mi; + + int base_size = m*(3 + mi)*sizeof(int) + (mi+1)*sizeof(double); + if( m > 2 && mi > data->params.max_categories ) + base_size += (m*std::min(data->params.max_categories, n) + mi)*sizeof(int); + else + base_size += mi*sizeof(int*); + cv::AutoBuffer inn_buf(base_size); + if( !_ext_buf ) + inn_buf.allocate(base_size + 2*n*sizeof(int)); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; + + int* lc = (int*)base_buf; + int* rc = lc + m; + int* _cjk = rc + m*2, *cjk = _cjk; + double* c_weights = (double*)alignPtr(cjk + m*mi, sizeof(double)); + + int* labels_buf = (int*)ext_buf; + const int* labels = data->get_cat_var_data(node, vi, labels_buf); + int* responses_buf = labels_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + + int* cluster_labels = 0; + int** int_ptr = 0; + int i, j, k, idx; + double L = 0, R = 0; + double best_val = init_quality; + int prevcode = 0, best_subset = -1, subset_i, subset_n, subtract = 0; + const double* priors = data->priors_mult->data.db; + + // init array of counters: + // c_{jk} - number of samples that have vi-th input variable = j and response = k. + for( j = -1; j < mi; j++ ) + for( k = 0; k < m; k++ ) + cjk[j*m + k] = 0; + + for( i = 0; i < n; i++ ) + { + j = ( labels[i] == 65535 && data->is_buf_16u) ? -1 : labels[i]; + k = responses[i]; + cjk[j*m + k]++; + } + + if( m > 2 ) + { + if( mi > data->params.max_categories ) + { + mi = MIN(data->params.max_categories, n); + cjk = (int*)(c_weights + _mi); + cluster_labels = cjk + m*mi; + cluster_categories( _cjk, _mi, m, cjk, mi, cluster_labels ); + } + subset_i = 1; + subset_n = 1 << mi; + } + else + { + assert( m == 2 ); + int_ptr = (int**)(c_weights + _mi); + for( j = 0; j < mi; j++ ) + int_ptr[j] = cjk + j*2 + 1; + std::sort(int_ptr, int_ptr + mi, LessThanPtr()); + subset_i = 0; + subset_n = mi; + } + + for( k = 0; k < m; k++ ) + { + int sum = 0; + for( j = 0; j < mi; j++ ) + sum += cjk[j*m + k]; + rc[k] = sum; + lc[k] = 0; + } + + for( j = 0; j < mi; j++ ) + { + double sum = 0; + for( k = 0; k < m; k++ ) + sum += cjk[j*m + k]*priors[k]; + c_weights[j] = sum; + R += c_weights[j]; + } + + for( ; subset_i < subset_n; subset_i++ ) + { + double weight; + int* crow; + double lsum2 = 0, rsum2 = 0; + + if( m == 2 ) + idx = (int)(int_ptr[subset_i] - cjk)/2; + else + { + int graycode = (subset_i>>1)^subset_i; + int diff = graycode ^ prevcode; + + // determine index of the changed bit. + Cv32suf u; + idx = diff >= (1 << 16) ? 16 : 0; + u.f = (float)(((diff >> 16) | diff) & 65535); + idx += (u.i >> 23) - 127; + subtract = graycode < prevcode; + prevcode = graycode; + } + + crow = cjk + idx*m; + weight = c_weights[idx]; + if( weight < FLT_EPSILON ) + continue; + + if( !subtract ) + { + for( k = 0; k < m; k++ ) + { + int t = crow[k]; + int lval = lc[k] + t; + int rval = rc[k] - t; + double p = priors[k], p2 = p*p; + lsum2 += p2*lval*lval; + rsum2 += p2*rval*rval; + lc[k] = lval; rc[k] = rval; + } + L += weight; + R -= weight; + } + else + { + for( k = 0; k < m; k++ ) + { + int t = crow[k]; + int lval = lc[k] - t; + int rval = rc[k] + t; + double p = priors[k], p2 = p*p; + lsum2 += p2*lval*lval; + rsum2 += p2*rval*rval; + lc[k] = lval; rc[k] = rval; + } + L -= weight; + R += weight; + } + + if( L > FLT_EPSILON && R > FLT_EPSILON ) + { + double val = (lsum2*R + rsum2*L)/((double)L*R); + if( best_val < val ) + { + best_val = val; + best_subset = subset_i; + } + } + } + + CvDTreeSplit* split = 0; + if( best_subset >= 0 ) + { + split = _split ? _split : data->new_split_cat( 0, -1.0f ); + split->var_idx = vi; + split->quality = (float)best_val; + memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); + if( m == 2 ) + { + for( i = 0; i <= best_subset; i++ ) + { + idx = (int)(int_ptr[i] - cjk) >> 1; + split->subset[idx >> 5] |= 1 << (idx & 31); + } + } + else + { + for( i = 0; i < _mi; i++ ) + { + idx = cluster_labels ? cluster_labels[i] : i; + if( best_subset & (1 << idx) ) + split->subset[i >> 5] |= 1 << (i & 31); + } + } + } + return split; +} + + +CvDTreeSplit* CvDTree::find_split_ord_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) +{ + const float epsilon = FLT_EPSILON*2; + int n = node->sample_count; + int n1 = node->get_num_valid(vi); + + cv::AutoBuffer inn_buf; + if( !_ext_buf ) + inn_buf.allocate(2*n*(sizeof(int) + sizeof(float))); + uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; + float* values_buf = (float*)ext_buf; + int* sorted_indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = sorted_indices_buf + n; + const float* values = 0; + const int* sorted_indices = 0; + data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); + float* responses_buf = (float*)(sample_indices_buf + n); + const float* responses = data->get_ord_responses( node, responses_buf, sample_indices_buf ); + + int i, best_i = -1; + double best_val = init_quality, lsum = 0, rsum = node->value*n; + int L = 0, R = n1; + + // compensate for missing values + for( i = n1; i < n; i++ ) + rsum -= responses[sorted_indices[i]]; + + // find the optimal split + for( i = 0; i < n1 - 1; i++ ) + { + float t = responses[sorted_indices[i]]; + L++; R--; + lsum += t; + rsum -= t; + + if( values[i] + epsilon < values[i+1] ) + { + double val = (lsum*lsum*R + rsum*rsum*L)/((double)L*R); + if( best_val < val ) + { + best_val = val; + best_i = i; + } + } + } + + CvDTreeSplit* split = 0; + if( best_i >= 0 ) + { + split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); + split->var_idx = vi; + split->ord.c = (values[best_i] + values[best_i+1])*0.5f; + split->ord.split_point = best_i; + split->inversed = 0; + split->quality = (float)best_val; + } + return split; +} + +CvDTreeSplit* CvDTree::find_split_cat_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) +{ + int ci = data->get_var_type(vi); + int n = node->sample_count; + int mi = data->cat_count->data.i[ci]; + + int base_size = (mi+2)*sizeof(double) + (mi+1)*(sizeof(int) + sizeof(double*)); + cv::AutoBuffer inn_buf(base_size); + if( !_ext_buf ) + inn_buf.allocate(base_size + n*(2*sizeof(int) + sizeof(float))); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; + int* labels_buf = (int*)ext_buf; + const int* labels = data->get_cat_var_data(node, vi, labels_buf); + float* responses_buf = (float*)(labels_buf + n); + int* sample_indices_buf = (int*)(responses_buf + n); + const float* responses = data->get_ord_responses(node, responses_buf, sample_indices_buf); + + double* sum = (double*)cv::alignPtr(base_buf,sizeof(double)) + 1; + int* counts = (int*)(sum + mi) + 1; + double** sum_ptr = (double**)(counts + mi); + int i, L = 0, R = 0; + double best_val = init_quality, lsum = 0, rsum = 0; + int best_subset = -1, subset_i; + + for( i = -1; i < mi; i++ ) + sum[i] = counts[i] = 0; + + // calculate sum response and weight of each category of the input var + for( i = 0; i < n; i++ ) + { + int idx = ( (labels[i] == 65535) && data->is_buf_16u ) ? -1 : labels[i]; + double s = sum[idx] + responses[i]; + int nc = counts[idx] + 1; + sum[idx] = s; + counts[idx] = nc; + } + + // calculate average response in each category + for( i = 0; i < mi; i++ ) + { + R += counts[i]; + rsum += sum[i]; + sum[i] /= MAX(counts[i],1); + sum_ptr[i] = sum + i; + } + + std::sort(sum_ptr, sum_ptr + mi, LessThanPtr()); + + // revert back to unnormalized sums + // (there should be a very little loss of accuracy) + for( i = 0; i < mi; i++ ) + sum[i] *= counts[i]; + + for( subset_i = 0; subset_i < mi-1; subset_i++ ) + { + int idx = (int)(sum_ptr[subset_i] - sum); + int ni = counts[idx]; + + if( ni ) + { + double s = sum[idx]; + lsum += s; L += ni; + rsum -= s; R -= ni; + + if( L && R ) + { + double val = (lsum*lsum*R + rsum*rsum*L)/((double)L*R); + if( best_val < val ) + { + best_val = val; + best_subset = subset_i; + } + } + } + } + + CvDTreeSplit* split = 0; + if( best_subset >= 0 ) + { + split = _split ? _split : data->new_split_cat( 0, -1.0f); + split->var_idx = vi; + split->quality = (float)best_val; + memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); + for( i = 0; i <= best_subset; i++ ) + { + int idx = (int)(sum_ptr[i] - sum); + split->subset[idx >> 5] |= 1 << (idx & 31); + } + } + return split; +} + +CvDTreeSplit* CvDTree::find_surrogate_split_ord( CvDTreeNode* node, int vi, uchar* _ext_buf ) +{ + const float epsilon = FLT_EPSILON*2; + const char* dir = (char*)data->direction->data.ptr; + int n = node->sample_count, n1 = node->get_num_valid(vi); + cv::AutoBuffer inn_buf; + if( !_ext_buf ) + inn_buf.allocate( n*(sizeof(int)*(data->have_priors ? 3 : 2) + sizeof(float)) ); + uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; + float* values_buf = (float*)ext_buf; + int* sorted_indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = sorted_indices_buf + n; + const float* values = 0; + const int* sorted_indices = 0; + data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); + // LL - number of samples that both the primary and the surrogate splits send to the left + // LR - ... primary split sends to the left and the surrogate split sends to the right + // RL - ... primary split sends to the right and the surrogate split sends to the left + // RR - ... both send to the right + int i, best_i = -1, best_inversed = 0; + double best_val; + + if( !data->have_priors ) + { + int LL = 0, RL = 0, LR, RR; + int worst_val = cvFloor(node->maxlr), _best_val = worst_val; + int sum = 0, sum_abs = 0; + + for( i = 0; i < n1; i++ ) + { + int d = dir[sorted_indices[i]]; + sum += d; sum_abs += d & 1; + } + + // sum_abs = R + L; sum = R - L + RR = (sum_abs + sum) >> 1; + LR = (sum_abs - sum) >> 1; + + // initially all the samples are sent to the right by the surrogate split, + // LR of them are sent to the left by primary split, and RR - to the right. + // now iteratively compute LL, LR, RL and RR for every possible surrogate split value. + for( i = 0; i < n1 - 1; i++ ) + { + int d = dir[sorted_indices[i]]; + + if( d < 0 ) + { + LL++; LR--; + if( LL + RR > _best_val && values[i] + epsilon < values[i+1] ) + { + best_val = LL + RR; + best_i = i; best_inversed = 0; + } + } + else if( d > 0 ) + { + RL++; RR--; + if( RL + LR > _best_val && values[i] + epsilon < values[i+1] ) + { + best_val = RL + LR; + best_i = i; best_inversed = 1; + } + } + } + best_val = _best_val; + } + else + { + double LL = 0, RL = 0, LR, RR; + double worst_val = node->maxlr; + double sum = 0, sum_abs = 0; + const double* priors = data->priors_mult->data.db; + int* responses_buf = sample_indices_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + best_val = worst_val; + + for( i = 0; i < n1; i++ ) + { + int idx = sorted_indices[i]; + double w = priors[responses[idx]]; + int d = dir[idx]; + sum += d*w; sum_abs += (d & 1)*w; + } + + // sum_abs = R + L; sum = R - L + RR = (sum_abs + sum)*0.5; + LR = (sum_abs - sum)*0.5; + + // initially all the samples are sent to the right by the surrogate split, + // LR of them are sent to the left by primary split, and RR - to the right. + // now iteratively compute LL, LR, RL and RR for every possible surrogate split value. + for( i = 0; i < n1 - 1; i++ ) + { + int idx = sorted_indices[i]; + double w = priors[responses[idx]]; + int d = dir[idx]; + + if( d < 0 ) + { + LL += w; LR -= w; + if( LL + RR > best_val && values[i] + epsilon < values[i+1] ) + { + best_val = LL + RR; + best_i = i; best_inversed = 0; + } + } + else if( d > 0 ) + { + RL += w; RR -= w; + if( RL + LR > best_val && values[i] + epsilon < values[i+1] ) + { + best_val = RL + LR; + best_i = i; best_inversed = 1; + } + } + } + } + return best_i >= 0 && best_val > node->maxlr ? data->new_split_ord( vi, + (values[best_i] + values[best_i+1])*0.5f, best_i, best_inversed, (float)best_val ) : 0; +} + + +CvDTreeSplit* CvDTree::find_surrogate_split_cat( CvDTreeNode* node, int vi, uchar* _ext_buf ) +{ + const char* dir = (char*)data->direction->data.ptr; + int n = node->sample_count; + int i, mi = data->cat_count->data.i[data->get_var_type(vi)], l_win = 0; + + int base_size = (2*(mi+1)+1)*sizeof(double) + (!data->have_priors ? 2*(mi+1)*sizeof(int) : 0); + cv::AutoBuffer inn_buf(base_size); + if( !_ext_buf ) + inn_buf.allocate(base_size + n*(sizeof(int) + (data->have_priors ? sizeof(int) : 0))); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; + + int* labels_buf = (int*)ext_buf; + const int* labels = data->get_cat_var_data(node, vi, labels_buf); + // LL - number of samples that both the primary and the surrogate splits send to the left + // LR - ... primary split sends to the left and the surrogate split sends to the right + // RL - ... primary split sends to the right and the surrogate split sends to the left + // RR - ... both send to the right + CvDTreeSplit* split = data->new_split_cat( vi, 0 ); + double best_val = 0; + double* lc = (double*)cv::alignPtr(base_buf,sizeof(double)) + 1; + double* rc = lc + mi + 1; + + for( i = -1; i < mi; i++ ) + lc[i] = rc[i] = 0; + + // for each category calculate the weight of samples + // sent to the left (lc) and to the right (rc) by the primary split + if( !data->have_priors ) + { + int* _lc = (int*)rc + 1; + int* _rc = _lc + mi + 1; + + for( i = -1; i < mi; i++ ) + _lc[i] = _rc[i] = 0; + + for( i = 0; i < n; i++ ) + { + int idx = ( (labels[i] == 65535) && (data->is_buf_16u) ) ? -1 : labels[i]; + int d = dir[i]; + int sum = _lc[idx] + d; + int sum_abs = _rc[idx] + (d & 1); + _lc[idx] = sum; _rc[idx] = sum_abs; + } + + for( i = 0; i < mi; i++ ) + { + int sum = _lc[i]; + int sum_abs = _rc[i]; + lc[i] = (sum_abs - sum) >> 1; + rc[i] = (sum_abs + sum) >> 1; + } + } + else + { + const double* priors = data->priors_mult->data.db; + int* responses_buf = labels_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + + for( i = 0; i < n; i++ ) + { + int idx = ( (labels[i] == 65535) && (data->is_buf_16u) ) ? -1 : labels[i]; + double w = priors[responses[i]]; + int d = dir[i]; + double sum = lc[idx] + d*w; + double sum_abs = rc[idx] + (d & 1)*w; + lc[idx] = sum; rc[idx] = sum_abs; + } + + for( i = 0; i < mi; i++ ) + { + double sum = lc[i]; + double sum_abs = rc[i]; + lc[i] = (sum_abs - sum) * 0.5; + rc[i] = (sum_abs + sum) * 0.5; + } + } + + // 2. now form the split. + // in each category send all the samples to the same direction as majority + for( i = 0; i < mi; i++ ) + { + double lval = lc[i], rval = rc[i]; + if( lval > rval ) + { + split->subset[i >> 5] |= 1 << (i & 31); + best_val += lval; + l_win++; + } + else + best_val += rval; + } + + split->quality = (float)best_val; + if( split->quality <= node->maxlr || l_win == 0 || l_win == mi ) + cvSetRemoveByPtr( data->split_heap, split ), split = 0; + + return split; +} + + +void CvDTree::calc_node_value( CvDTreeNode* node ) +{ + int i, j, k, n = node->sample_count, cv_n = data->params.cv_folds; + int m = data->get_num_classes(); + + int base_size = data->is_classifier ? m*cv_n*sizeof(int) : 2*cv_n*sizeof(double)+cv_n*sizeof(int); + int ext_size = n*(sizeof(int) + (data->is_classifier ? sizeof(int) : sizeof(int)+sizeof(float))); + cv::AutoBuffer inn_buf(base_size + ext_size); + uchar* base_buf = (uchar*)inn_buf; + uchar* ext_buf = base_buf + base_size; + + int* cv_labels_buf = (int*)ext_buf; + const int* cv_labels = data->get_cv_labels(node, cv_labels_buf); + + if( data->is_classifier ) + { + // in case of classification tree: + // * node value is the label of the class that has the largest weight in the node. + // * node risk is the weighted number of misclassified samples, + // * j-th cross-validation fold value and risk are calculated as above, + // but using the samples with cv_labels(*)!=j. + // * j-th cross-validation fold error is calculated as the weighted number of + // misclassified samples with cv_labels(*)==j. + + // compute the number of instances of each class + int* cls_count = data->counts->data.i; + int* responses_buf = cv_labels_buf + n; + const int* responses = data->get_class_labels(node, responses_buf); + int* cv_cls_count = (int*)base_buf; + double max_val = -1, total_weight = 0; + int max_k = -1; + double* priors = data->priors_mult->data.db; + + for( k = 0; k < m; k++ ) + cls_count[k] = 0; + + if( cv_n == 0 ) + { + for( i = 0; i < n; i++ ) + cls_count[responses[i]]++; + } + else + { + for( j = 0; j < cv_n; j++ ) + for( k = 0; k < m; k++ ) + cv_cls_count[j*m + k] = 0; + + for( i = 0; i < n; i++ ) + { + j = cv_labels[i]; k = responses[i]; + cv_cls_count[j*m + k]++; + } + + for( j = 0; j < cv_n; j++ ) + for( k = 0; k < m; k++ ) + cls_count[k] += cv_cls_count[j*m + k]; + } + + if( data->have_priors && node->parent == 0 ) + { + // compute priors_mult from priors, take the sample ratio into account. + double sum = 0; + for( k = 0; k < m; k++ ) + { + int n_k = cls_count[k]; + priors[k] = data->priors->data.db[k]*(n_k ? 1./n_k : 0.); + sum += priors[k]; + } + sum = 1./sum; + for( k = 0; k < m; k++ ) + priors[k] *= sum; + } + + for( k = 0; k < m; k++ ) + { + double val = cls_count[k]*priors[k]; + total_weight += val; + if( max_val < val ) + { + max_val = val; + max_k = k; + } + } + + node->class_idx = max_k; + node->value = data->cat_map->data.i[ + data->cat_ofs->data.i[data->cat_var_count] + max_k]; + node->node_risk = total_weight - max_val; + + for( j = 0; j < cv_n; j++ ) + { + double sum_k = 0, sum = 0, max_val_k = 0; + max_val = -1; max_k = -1; + + for( k = 0; k < m; k++ ) + { + double w = priors[k]; + double val_k = cv_cls_count[j*m + k]*w; + double val = cls_count[k]*w - val_k; + sum_k += val_k; + sum += val; + if( max_val < val ) + { + max_val = val; + max_val_k = val_k; + max_k = k; + } + } + + node->cv_Tn[j] = INT_MAX; + node->cv_node_risk[j] = sum - max_val; + node->cv_node_error[j] = sum_k - max_val_k; + } + } + else + { + // in case of regression tree: + // * node value is 1/n*sum_i(Y_i), where Y_i is i-th response, + // n is the number of samples in the node. + // * node risk is the sum of squared errors: sum_i((Y_i - )^2) + // * j-th cross-validation fold value and risk are calculated as above, + // but using the samples with cv_labels(*)!=j. + // * j-th cross-validation fold error is calculated + // using samples with cv_labels(*)==j as the test subset: + // error_j = sum_(i,cv_labels(i)==j)((Y_i - )^2), + // where node_value_j is the node value calculated + // as described in the previous bullet, and summation is done + // over the samples with cv_labels(*)==j. + + double sum = 0, sum2 = 0; + float* values_buf = (float*)(cv_labels_buf + n); + int* sample_indices_buf = (int*)(values_buf + n); + const float* values = data->get_ord_responses(node, values_buf, sample_indices_buf); + double *cv_sum = 0, *cv_sum2 = 0; + int* cv_count = 0; + + if( cv_n == 0 ) + { + for( i = 0; i < n; i++ ) + { + double t = values[i]; + sum += t; + sum2 += t*t; + } + } + else + { + cv_sum = (double*)base_buf; + cv_sum2 = cv_sum + cv_n; + cv_count = (int*)(cv_sum2 + cv_n); + + for( j = 0; j < cv_n; j++ ) + { + cv_sum[j] = cv_sum2[j] = 0.; + cv_count[j] = 0; + } + + for( i = 0; i < n; i++ ) + { + j = cv_labels[i]; + double t = values[i]; + double s = cv_sum[j] + t; + double s2 = cv_sum2[j] + t*t; + int nc = cv_count[j] + 1; + cv_sum[j] = s; + cv_sum2[j] = s2; + cv_count[j] = nc; + } + + for( j = 0; j < cv_n; j++ ) + { + sum += cv_sum[j]; + sum2 += cv_sum2[j]; + } + } + + node->node_risk = sum2 - (sum/n)*sum; + node->value = sum/n; + + for( j = 0; j < cv_n; j++ ) + { + double s = cv_sum[j], si = sum - s; + double s2 = cv_sum2[j], s2i = sum2 - s2; + int c = cv_count[j], ci = n - c; + double r = si/MAX(ci,1); + node->cv_node_risk[j] = s2i - r*r*ci; + node->cv_node_error[j] = s2 - 2*r*s + c*r*r; + node->cv_Tn[j] = INT_MAX; + } + } +} + + +void CvDTree::complete_node_dir( CvDTreeNode* node ) +{ + int vi, i, n = node->sample_count, nl, nr, d0 = 0, d1 = -1; + int nz = n - node->get_num_valid(node->split->var_idx); + char* dir = (char*)data->direction->data.ptr; + + // try to complete direction using surrogate splits + if( nz && data->params.use_surrogates ) + { + cv::AutoBuffer inn_buf(n*(2*sizeof(int)+sizeof(float))); + CvDTreeSplit* split = node->split->next; + for( ; split != 0 && nz; split = split->next ) + { + int inversed_mask = split->inversed ? -1 : 0; + vi = split->var_idx; + + if( data->get_var_type(vi) >= 0 ) // split on categorical var + { + int* labels_buf = (int*)(uchar*)inn_buf; + const int* labels = data->get_cat_var_data(node, vi, labels_buf); + const int* subset = split->subset; + + for( i = 0; i < n; i++ ) + { + int idx = labels[i]; + if( !dir[i] && ( ((idx >= 0)&&(!data->is_buf_16u)) || ((idx != 65535)&&(data->is_buf_16u)) )) + + { + int d = CV_DTREE_CAT_DIR(idx,subset); + dir[i] = (char)((d ^ inversed_mask) - inversed_mask); + if( --nz ) + break; + } + } + } + else // split on ordered var + { + float* values_buf = (float*)(uchar*)inn_buf; + int* sorted_indices_buf = (int*)(values_buf + n); + int* sample_indices_buf = sorted_indices_buf + n; + const float* values = 0; + const int* sorted_indices = 0; + data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); + int split_point = split->ord.split_point; + int n1 = node->get_num_valid(vi); + + assert( 0 <= split_point && split_point < n-1 ); + + for( i = 0; i < n1; i++ ) + { + int idx = sorted_indices[i]; + if( !dir[idx] ) + { + int d = i <= split_point ? -1 : 1; + dir[idx] = (char)((d ^ inversed_mask) - inversed_mask); + if( --nz ) + break; + } + } + } + } + } + + // find the default direction for the rest + if( nz ) + { + for( i = nr = 0; i < n; i++ ) + nr += dir[i] > 0; + nl = n - nr - nz; + d0 = nl > nr ? -1 : nr > nl; + } + + // make sure that every sample is directed either to the left or to the right + for( i = 0; i < n; i++ ) + { + int d = dir[i]; + if( !d ) + { + d = d0; + if( !d ) + d = d1, d1 = -d1; + } + d = d > 0; + dir[i] = (char)d; // remap (-1,1) to (0,1) + } +} + + +void CvDTree::split_node_data( CvDTreeNode* node ) +{ + int vi, i, n = node->sample_count, nl, nr, scount = data->sample_count; + char* dir = (char*)data->direction->data.ptr; + CvDTreeNode *left = 0, *right = 0; + int* new_idx = data->split_buf->data.i; + int new_buf_idx = data->get_child_buf_idx( node ); + int work_var_count = data->get_work_var_count(); + CvMat* buf = data->buf; + size_t length_buf_row = data->get_length_subbuf(); + cv::AutoBuffer inn_buf(n*(3*sizeof(int) + sizeof(float))); + int* temp_buf = (int*)(uchar*)inn_buf; + + complete_node_dir(node); + + for( i = nl = nr = 0; i < n; i++ ) + { + int d = dir[i]; + // initialize new indices for splitting ordered variables + new_idx[i] = (nl & (d-1)) | (nr & -d); // d ? ri : li + nr += d; + nl += d^1; + } + + bool split_input_data; + node->left = left = data->new_node( node, nl, new_buf_idx, node->offset ); + node->right = right = data->new_node( node, nr, new_buf_idx, node->offset + nl ); + + split_input_data = node->depth + 1 < data->params.max_depth && + (node->left->sample_count > data->params.min_sample_count || + node->right->sample_count > data->params.min_sample_count); + + // split ordered variables, keep both halves sorted. + for( vi = 0; vi < data->var_count; vi++ ) + { + int ci = data->get_var_type(vi); + + if( ci >= 0 || !split_input_data ) + continue; + + int n1 = node->get_num_valid(vi); + float* src_val_buf = (float*)(uchar*)(temp_buf + n); + int* src_sorted_idx_buf = (int*)(src_val_buf + n); + int* src_sample_idx_buf = src_sorted_idx_buf + n; + const float* src_val = 0; + const int* src_sorted_idx = 0; + data->get_ord_var_data(node, vi, src_val_buf, src_sorted_idx_buf, &src_val, &src_sorted_idx, src_sample_idx_buf); + + for(i = 0; i < n; i++) + temp_buf[i] = src_sorted_idx[i]; + + if (data->is_buf_16u) + { + unsigned short *ldst, *rdst, *ldst0, *rdst0; + //unsigned short tl, tr; + ldst0 = ldst = (unsigned short*)(buf->data.s + left->buf_idx*length_buf_row + + vi*scount + left->offset); + rdst0 = rdst = (unsigned short*)(ldst + nl); + + // split sorted + for( i = 0; i < n1; i++ ) + { + int idx = temp_buf[i]; + int d = dir[idx]; + idx = new_idx[idx]; + if (d) + { + *rdst = (unsigned short)idx; + rdst++; + } + else + { + *ldst = (unsigned short)idx; + ldst++; + } + } + + left->set_num_valid(vi, (int)(ldst - ldst0)); + right->set_num_valid(vi, (int)(rdst - rdst0)); + + // split missing + for( ; i < n; i++ ) + { + int idx = temp_buf[i]; + int d = dir[idx]; + idx = new_idx[idx]; + if (d) + { + *rdst = (unsigned short)idx; + rdst++; + } + else + { + *ldst = (unsigned short)idx; + ldst++; + } + } + } + else + { + int *ldst0, *ldst, *rdst0, *rdst; + ldst0 = ldst = buf->data.i + left->buf_idx*length_buf_row + + vi*scount + left->offset; + rdst0 = rdst = buf->data.i + right->buf_idx*length_buf_row + + vi*scount + right->offset; + + // split sorted + for( i = 0; i < n1; i++ ) + { + int idx = temp_buf[i]; + int d = dir[idx]; + idx = new_idx[idx]; + if (d) + { + *rdst = idx; + rdst++; + } + else + { + *ldst = idx; + ldst++; + } + } + + left->set_num_valid(vi, (int)(ldst - ldst0)); + right->set_num_valid(vi, (int)(rdst - rdst0)); + + // split missing + for( ; i < n; i++ ) + { + int idx = temp_buf[i]; + int d = dir[idx]; + idx = new_idx[idx]; + if (d) + { + *rdst = idx; + rdst++; + } + else + { + *ldst = idx; + ldst++; + } + } + } + } + + // split categorical vars, responses and cv_labels using new_idx relocation table + for( vi = 0; vi < work_var_count; vi++ ) + { + int ci = data->get_var_type(vi); + int n1 = node->get_num_valid(vi), nr1 = 0; + + if( ci < 0 || (vi < data->var_count && !split_input_data) ) + continue; + + int *src_lbls_buf = temp_buf + n; + const int* src_lbls = data->get_cat_var_data(node, vi, src_lbls_buf); + + for(i = 0; i < n; i++) + temp_buf[i] = src_lbls[i]; + + if (data->is_buf_16u) + { + unsigned short *ldst = (unsigned short *)(buf->data.s + left->buf_idx*length_buf_row + + vi*scount + left->offset); + unsigned short *rdst = (unsigned short *)(buf->data.s + right->buf_idx*length_buf_row + + vi*scount + right->offset); + + for( i = 0; i < n; i++ ) + { + int d = dir[i]; + int idx = temp_buf[i]; + if (d) + { + *rdst = (unsigned short)idx; + rdst++; + nr1 += (idx != 65535 )&d; + } + else + { + *ldst = (unsigned short)idx; + ldst++; + } + } + + if( vi < data->var_count ) + { + left->set_num_valid(vi, n1 - nr1); + right->set_num_valid(vi, nr1); + } + } + else + { + int *ldst = buf->data.i + left->buf_idx*length_buf_row + + vi*scount + left->offset; + int *rdst = buf->data.i + right->buf_idx*length_buf_row + + vi*scount + right->offset; + + for( i = 0; i < n; i++ ) + { + int d = dir[i]; + int idx = temp_buf[i]; + if (d) + { + *rdst = idx; + rdst++; + nr1 += (idx >= 0)&d; + } + else + { + *ldst = idx; + ldst++; + } + + } + + if( vi < data->var_count ) + { + left->set_num_valid(vi, n1 - nr1); + right->set_num_valid(vi, nr1); + } + } + } + + + // split sample indices + int *sample_idx_src_buf = temp_buf + n; + const int* sample_idx_src = data->get_sample_indices(node, sample_idx_src_buf); + + for(i = 0; i < n; i++) + temp_buf[i] = sample_idx_src[i]; + + int pos = data->get_work_var_count(); + if (data->is_buf_16u) + { + unsigned short* ldst = (unsigned short*)(buf->data.s + left->buf_idx*length_buf_row + + pos*scount + left->offset); + unsigned short* rdst = (unsigned short*)(buf->data.s + right->buf_idx*length_buf_row + + pos*scount + right->offset); + for (i = 0; i < n; i++) + { + int d = dir[i]; + unsigned short idx = (unsigned short)temp_buf[i]; + if (d) + { + *rdst = idx; + rdst++; + } + else + { + *ldst = idx; + ldst++; + } + } + } + else + { + int* ldst = buf->data.i + left->buf_idx*length_buf_row + + pos*scount + left->offset; + int* rdst = buf->data.i + right->buf_idx*length_buf_row + + pos*scount + right->offset; + for (i = 0; i < n; i++) + { + int d = dir[i]; + int idx = temp_buf[i]; + if (d) + { + *rdst = idx; + rdst++; + } + else + { + *ldst = idx; + ldst++; + } + } + } + + // deallocate the parent node data that is not needed anymore + data->free_node_data(node); +} + +float CvDTree::calc_error( CvMLData* _data, int type, std::vector *resp ) +{ + float err = 0; + const CvMat* values = _data->get_values(); + const CvMat* response = _data->get_responses(); + const CvMat* missing = _data->get_missing(); + const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); + const CvMat* var_types = _data->get_var_types(); + int* sidx = sample_idx ? sample_idx->data.i : 0; + int r_step = CV_IS_MAT_CONT(response->type) ? + 1 : response->step / CV_ELEM_SIZE(response->type); + bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; + int sample_count = sample_idx ? sample_idx->cols : 0; + sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; + float* pred_resp = 0; + if( resp && (sample_count > 0) ) + { + resp->resize( sample_count ); + pred_resp = &((*resp)[0]); + } + + if ( is_classifier ) + { + for( int i = 0; i < sample_count; i++ ) + { + CvMat sample, miss; + int si = sidx ? sidx[i] : i; + cvGetRow( values, &sample, si ); + if( missing ) + cvGetRow( missing, &miss, si ); + float r = (float)predict( &sample, missing ? &miss : 0 )->value; + if( pred_resp ) + pred_resp[i] = r; + int d = fabs((double)r - response->data.fl[(size_t)si*r_step]) <= FLT_EPSILON ? 0 : 1; + err += d; + } + err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; + } + else + { + for( int i = 0; i < sample_count; i++ ) + { + CvMat sample, miss; + int si = sidx ? sidx[i] : i; + cvGetRow( values, &sample, si ); + if( missing ) + cvGetRow( missing, &miss, si ); + float r = (float)predict( &sample, missing ? &miss : 0 )->value; + if( pred_resp ) + pred_resp[i] = r; + float d = r - response->data.fl[(size_t)si*r_step]; + err += d*d; + } + err = sample_count ? err / (float)sample_count : -FLT_MAX; + } + return err; +} + +void CvDTree::prune_cv() +{ + CvMat* ab = 0; + CvMat* temp = 0; + CvMat* err_jk = 0; + + // 1. build tree sequence for each cv fold, calculate error_{Tj,beta_k}. + // 2. choose the best tree index (if need, apply 1SE rule). + // 3. store the best index and cut the branches. + + CV_FUNCNAME( "CvDTree::prune_cv" ); + + __BEGIN__; + + int ti, j, tree_count = 0, cv_n = data->params.cv_folds, n = root->sample_count; + // currently, 1SE for regression is not implemented + bool use_1se = data->params.use_1se_rule != 0 && data->is_classifier; + double* err; + double min_err = 0, min_err_se = 0; + int min_idx = -1; + + CV_CALL( ab = cvCreateMat( 1, 256, CV_64F )); + + // build the main tree sequence, calculate alpha's + for(;;tree_count++) + { + double min_alpha = update_tree_rnc(tree_count, -1); + if( cut_tree(tree_count, -1, min_alpha) ) + break; + + if( ab->cols <= tree_count ) + { + CV_CALL( temp = cvCreateMat( 1, ab->cols*3/2, CV_64F )); + for( ti = 0; ti < ab->cols; ti++ ) + temp->data.db[ti] = ab->data.db[ti]; + cvReleaseMat( &ab ); + ab = temp; + temp = 0; + } + + ab->data.db[tree_count] = min_alpha; + } + + ab->data.db[0] = 0.; + + if( tree_count > 0 ) + { + for( ti = 1; ti < tree_count-1; ti++ ) + ab->data.db[ti] = sqrt(ab->data.db[ti]*ab->data.db[ti+1]); + ab->data.db[tree_count-1] = DBL_MAX*0.5; + + CV_CALL( err_jk = cvCreateMat( cv_n, tree_count, CV_64F )); + err = err_jk->data.db; + + for( j = 0; j < cv_n; j++ ) + { + int tj = 0, tk = 0; + for( ; tk < tree_count; tj++ ) + { + double min_alpha = update_tree_rnc(tj, j); + if( cut_tree(tj, j, min_alpha) ) + min_alpha = DBL_MAX; + + for( ; tk < tree_count; tk++ ) + { + if( ab->data.db[tk] > min_alpha ) + break; + err[j*tree_count + tk] = root->tree_error; + } + } + } + + for( ti = 0; ti < tree_count; ti++ ) + { + double sum_err = 0; + for( j = 0; j < cv_n; j++ ) + sum_err += err[j*tree_count + ti]; + if( ti == 0 || sum_err < min_err ) + { + min_err = sum_err; + min_idx = ti; + if( use_1se ) + min_err_se = sqrt( sum_err*(n - sum_err) ); + } + else if( sum_err < min_err + min_err_se ) + min_idx = ti; + } + } + + pruned_tree_idx = min_idx; + free_prune_data(data->params.truncate_pruned_tree != 0); + + __END__; + + cvReleaseMat( &err_jk ); + cvReleaseMat( &ab ); + cvReleaseMat( &temp ); +} + + +double CvDTree::update_tree_rnc( int T, int fold ) +{ + CvDTreeNode* node = root; + double min_alpha = DBL_MAX; + + for(;;) + { + CvDTreeNode* parent; + for(;;) + { + int t = fold >= 0 ? node->cv_Tn[fold] : node->Tn; + if( t <= T || !node->left ) + { + node->complexity = 1; + node->tree_risk = node->node_risk; + node->tree_error = 0.; + if( fold >= 0 ) + { + node->tree_risk = node->cv_node_risk[fold]; + node->tree_error = node->cv_node_error[fold]; + } + break; + } + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + { + parent->complexity += node->complexity; + parent->tree_risk += node->tree_risk; + parent->tree_error += node->tree_error; + + parent->alpha = ((fold >= 0 ? parent->cv_node_risk[fold] : parent->node_risk) + - parent->tree_risk)/(parent->complexity - 1); + min_alpha = MIN( min_alpha, parent->alpha ); + } + + if( !parent ) + break; + + parent->complexity = node->complexity; + parent->tree_risk = node->tree_risk; + parent->tree_error = node->tree_error; + node = parent->right; + } + + return min_alpha; +} + + +int CvDTree::cut_tree( int T, int fold, double min_alpha ) +{ + CvDTreeNode* node = root; + if( !node->left ) + return 1; + + for(;;) + { + CvDTreeNode* parent; + for(;;) + { + int t = fold >= 0 ? node->cv_Tn[fold] : node->Tn; + if( t <= T || !node->left ) + break; + if( node->alpha <= min_alpha + FLT_EPSILON ) + { + if( fold >= 0 ) + node->cv_Tn[fold] = T; + else + node->Tn = T; + if( node == root ) + return 1; + break; + } + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + ; + + if( !parent ) + break; + + node = parent->right; + } + + return 0; +} + + +void CvDTree::free_prune_data(bool _cut_tree) +{ + CvDTreeNode* node = root; + + for(;;) + { + CvDTreeNode* parent; + for(;;) + { + // do not call cvSetRemoveByPtr( cv_heap, node->cv_Tn ) + // as we will clear the whole cross-validation heap at the end + node->cv_Tn = 0; + node->cv_node_error = node->cv_node_risk = 0; + if( !node->left ) + break; + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + { + if( _cut_tree && parent->Tn <= pruned_tree_idx ) + { + data->free_node( parent->left ); + data->free_node( parent->right ); + parent->left = parent->right = 0; + } + } + + if( !parent ) + break; + + node = parent->right; + } + + if( data->cv_heap ) + cvClearSet( data->cv_heap ); +} + + +void CvDTree::free_tree() +{ + if( root && data && data->shared ) + { + pruned_tree_idx = INT_MIN; + free_prune_data(true); + data->free_node(root); + root = 0; + } +} + +CvDTreeNode* CvDTree::predict( const CvMat* _sample, + const CvMat* _missing, bool preprocessed_input ) const +{ + cv::AutoBuffer catbuf; + + int i, mstep = 0; + const uchar* m = 0; + CvDTreeNode* node = root; + + if( !node ) + CV_Error( CV_StsError, "The tree has not been trained yet" ); + + if( !CV_IS_MAT(_sample) || CV_MAT_TYPE(_sample->type) != CV_32FC1 || + (_sample->cols != 1 && _sample->rows != 1) || + (_sample->cols + _sample->rows - 1 != data->var_all && !preprocessed_input) || + (_sample->cols + _sample->rows - 1 != data->var_count && preprocessed_input) ) + CV_Error( CV_StsBadArg, + "the input sample must be 1d floating-point vector with the same " + "number of elements as the total number of variables used for training" ); + + const float* sample = _sample->data.fl; + int step = CV_IS_MAT_CONT(_sample->type) ? 1 : _sample->step/sizeof(sample[0]); + + if( data->cat_count && !preprocessed_input ) // cache for categorical variables + { + int n = data->cat_count->cols; + catbuf.allocate(n); + for( i = 0; i < n; i++ ) + catbuf[i] = -1; + } + + if( _missing ) + { + if( !CV_IS_MAT(_missing) || !CV_IS_MASK_ARR(_missing) || + !CV_ARE_SIZES_EQ(_missing, _sample) ) + CV_Error( CV_StsBadArg, + "the missing data mask must be 8-bit vector of the same size as input sample" ); + m = _missing->data.ptr; + mstep = CV_IS_MAT_CONT(_missing->type) ? 1 : _missing->step/sizeof(m[0]); + } + + const int* vtype = data->var_type->data.i; + const int* vidx = data->var_idx && !preprocessed_input ? data->var_idx->data.i : 0; + const int* cmap = data->cat_map ? data->cat_map->data.i : 0; + const int* cofs = data->cat_ofs ? data->cat_ofs->data.i : 0; + + while( node->Tn > pruned_tree_idx && node->left ) + { + CvDTreeSplit* split = node->split; + int dir = 0; + for( ; !dir && split != 0; split = split->next ) + { + int vi = split->var_idx; + int ci = vtype[vi]; + i = vidx ? vidx[vi] : vi; + float val = sample[(size_t)i*step]; + if( m && m[(size_t)i*mstep] ) + continue; + if( ci < 0 ) // ordered + dir = val <= split->ord.c ? -1 : 1; + else // categorical + { + int c; + if( preprocessed_input ) + c = cvRound(val); + else + { + c = catbuf[ci]; + if( c < 0 ) + { + int a = c = cofs[ci]; + int b = (ci+1 >= data->cat_ofs->cols) ? data->cat_map->cols : cofs[ci+1]; + + int ival = cvRound(val); + if( ival != val ) + CV_Error( CV_StsBadArg, + "one of input categorical variable is not an integer" ); + + int sh = 0; + while( a < b ) + { + sh++; + c = (a + b) >> 1; + if( ival < cmap[c] ) + b = c; + else if( ival > cmap[c] ) + a = c+1; + else + break; + } + + if( c < 0 || ival != cmap[c] ) + continue; + + catbuf[ci] = c -= cofs[ci]; + } + } + c = ( (c == 65535) && data->is_buf_16u ) ? -1 : c; + dir = CV_DTREE_CAT_DIR(c, split->subset); + } + + if( split->inversed ) + dir = -dir; + } + + if( !dir ) + { + double diff = node->right->sample_count - node->left->sample_count; + dir = diff < 0 ? -1 : 1; + } + node = dir < 0 ? node->left : node->right; + } + + return node; +} + + +CvDTreeNode* CvDTree::predict( const Mat& _sample, const Mat& _missing, bool preprocessed_input ) const +{ + CvMat sample = _sample, mmask = _missing; + return predict(&sample, mmask.data.ptr ? &mmask : 0, preprocessed_input); +} + + +const CvMat* CvDTree::get_var_importance() +{ + if( !var_importance ) + { + CvDTreeNode* node = root; + double* importance; + if( !node ) + return 0; + var_importance = cvCreateMat( 1, data->var_count, CV_64F ); + cvZero( var_importance ); + importance = var_importance->data.db; + + for(;;) + { + CvDTreeNode* parent; + for( ;; node = node->left ) + { + CvDTreeSplit* split = node->split; + + if( !node->left || node->Tn <= pruned_tree_idx ) + break; + + for( ; split != 0; split = split->next ) + importance[split->var_idx] += split->quality; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + ; + + if( !parent ) + break; + + node = parent->right; + } + + cvNormalize( var_importance, var_importance, 1., 0, CV_L1 ); + } + + return var_importance; +} + + +void CvDTree::write_split( CvFileStorage* fs, CvDTreeSplit* split ) const +{ + int ci; + + cvStartWriteStruct( fs, 0, CV_NODE_MAP + CV_NODE_FLOW ); + cvWriteInt( fs, "var", split->var_idx ); + cvWriteReal( fs, "quality", split->quality ); + + ci = data->get_var_type(split->var_idx); + if( ci >= 0 ) // split on a categorical var + { + int i, n = data->cat_count->data.i[ci], to_right = 0, default_dir; + for( i = 0; i < n; i++ ) + to_right += CV_DTREE_CAT_DIR(i,split->subset) > 0; + + // ad-hoc rule when to use inverse categorical split notation + // to achieve more compact and clear representation + default_dir = to_right <= 1 || to_right <= MIN(3, n/2) || to_right <= n/3 ? -1 : 1; + + cvStartWriteStruct( fs, default_dir*(split->inversed ? -1 : 1) > 0 ? + "in" : "not_in", CV_NODE_SEQ+CV_NODE_FLOW ); + + for( i = 0; i < n; i++ ) + { + int dir = CV_DTREE_CAT_DIR(i,split->subset); + if( dir*default_dir < 0 ) + cvWriteInt( fs, 0, i ); + } + cvEndWriteStruct( fs ); + } + else + cvWriteReal( fs, !split->inversed ? "le" : "gt", split->ord.c ); + + cvEndWriteStruct( fs ); +} + + +void CvDTree::write_node( CvFileStorage* fs, CvDTreeNode* node ) const +{ + CvDTreeSplit* split; + + cvStartWriteStruct( fs, 0, CV_NODE_MAP ); + + cvWriteInt( fs, "depth", node->depth ); + cvWriteInt( fs, "sample_count", node->sample_count ); + cvWriteReal( fs, "value", node->value ); + + if( data->is_classifier ) + cvWriteInt( fs, "norm_class_idx", node->class_idx ); + + cvWriteInt( fs, "Tn", node->Tn ); + cvWriteInt( fs, "complexity", node->complexity ); + cvWriteReal( fs, "alpha", node->alpha ); + cvWriteReal( fs, "node_risk", node->node_risk ); + cvWriteReal( fs, "tree_risk", node->tree_risk ); + cvWriteReal( fs, "tree_error", node->tree_error ); + + if( node->left ) + { + cvStartWriteStruct( fs, "splits", CV_NODE_SEQ ); + + for( split = node->split; split != 0; split = split->next ) + write_split( fs, split ); + + cvEndWriteStruct( fs ); + } + + cvEndWriteStruct( fs ); +} + + +void CvDTree::write_tree_nodes( CvFileStorage* fs ) const +{ + //CV_FUNCNAME( "CvDTree::write_tree_nodes" ); + + __BEGIN__; + + CvDTreeNode* node = root; + + // traverse the tree and save all the nodes in depth-first order + for(;;) + { + CvDTreeNode* parent; + for(;;) + { + write_node( fs, node ); + if( !node->left ) + break; + node = node->left; + } + + for( parent = node->parent; parent && parent->right == node; + node = parent, parent = parent->parent ) + ; + + if( !parent ) + break; + + node = parent->right; + } + + __END__; +} + + +void CvDTree::write( CvFileStorage* fs, const char* name ) const +{ + //CV_FUNCNAME( "CvDTree::write" ); + + __BEGIN__; + + cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_TREE ); + + //get_var_importance(); + data->write_params( fs ); + //if( var_importance ) + //cvWrite( fs, "var_importance", var_importance ); + write( fs ); + + cvEndWriteStruct( fs ); + + __END__; +} + + +void CvDTree::write( CvFileStorage* fs ) const +{ + //CV_FUNCNAME( "CvDTree::write" ); + + __BEGIN__; + + cvWriteInt( fs, "best_tree_idx", pruned_tree_idx ); + + cvStartWriteStruct( fs, "nodes", CV_NODE_SEQ ); + write_tree_nodes( fs ); + cvEndWriteStruct( fs ); + + __END__; +} + + +CvDTreeSplit* CvDTree::read_split( CvFileStorage* fs, CvFileNode* fnode ) +{ + CvDTreeSplit* split = 0; + + CV_FUNCNAME( "CvDTree::read_split" ); + + __BEGIN__; + + int vi, ci; + + if( !fnode || CV_NODE_TYPE(fnode->tag) != CV_NODE_MAP ) + CV_ERROR( CV_StsParseError, "some of the splits are not stored properly" ); + + vi = cvReadIntByName( fs, fnode, "var", -1 ); + if( (unsigned)vi >= (unsigned)data->var_count ) + CV_ERROR( CV_StsOutOfRange, "Split variable index is out of range" ); + + ci = data->get_var_type(vi); + if( ci >= 0 ) // split on categorical var + { + int i, n = data->cat_count->data.i[ci], inversed = 0, val; + CvSeqReader reader; + CvFileNode* inseq; + split = data->new_split_cat( vi, 0 ); + inseq = cvGetFileNodeByName( fs, fnode, "in" ); + if( !inseq ) + { + inseq = cvGetFileNodeByName( fs, fnode, "not_in" ); + inversed = 1; + } + if( !inseq || + (CV_NODE_TYPE(inseq->tag) != CV_NODE_SEQ && CV_NODE_TYPE(inseq->tag) != CV_NODE_INT)) + CV_ERROR( CV_StsParseError, + "Either 'in' or 'not_in' tags should be inside a categorical split data" ); + + if( CV_NODE_TYPE(inseq->tag) == CV_NODE_INT ) + { + val = inseq->data.i; + if( (unsigned)val >= (unsigned)n ) + CV_ERROR( CV_StsOutOfRange, "some of in/not_in elements are out of range" ); + + split->subset[val >> 5] |= 1 << (val & 31); + } + else + { + cvStartReadSeq( inseq->data.seq, &reader ); + + for( i = 0; i < reader.seq->total; i++ ) + { + CvFileNode* inode = (CvFileNode*)reader.ptr; + val = inode->data.i; + if( CV_NODE_TYPE(inode->tag) != CV_NODE_INT || (unsigned)val >= (unsigned)n ) + CV_ERROR( CV_StsOutOfRange, "some of in/not_in elements are out of range" ); + + split->subset[val >> 5] |= 1 << (val & 31); + CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + } + } + + // for categorical splits we do not use inversed splits, + // instead we inverse the variable set in the split + if( inversed ) + for( i = 0; i < (n + 31) >> 5; i++ ) + split->subset[i] ^= -1; + } + else + { + CvFileNode* cmp_node; + split = data->new_split_ord( vi, 0, 0, 0, 0 ); + + cmp_node = cvGetFileNodeByName( fs, fnode, "le" ); + if( !cmp_node ) + { + cmp_node = cvGetFileNodeByName( fs, fnode, "gt" ); + split->inversed = 1; + } + + split->ord.c = (float)cvReadReal( cmp_node ); + } + + split->quality = (float)cvReadRealByName( fs, fnode, "quality" ); + + __END__; + + return split; +} + + +CvDTreeNode* CvDTree::read_node( CvFileStorage* fs, CvFileNode* fnode, CvDTreeNode* parent ) +{ + CvDTreeNode* node = 0; + + CV_FUNCNAME( "CvDTree::read_node" ); + + __BEGIN__; + + CvFileNode* splits; + int i, depth; + + if( !fnode || CV_NODE_TYPE(fnode->tag) != CV_NODE_MAP ) + CV_ERROR( CV_StsParseError, "some of the tree elements are not stored properly" ); + + CV_CALL( node = data->new_node( parent, 0, 0, 0 )); + depth = cvReadIntByName( fs, fnode, "depth", -1 ); + if( depth != node->depth ) + CV_ERROR( CV_StsParseError, "incorrect node depth" ); + + node->sample_count = cvReadIntByName( fs, fnode, "sample_count" ); + node->value = cvReadRealByName( fs, fnode, "value" ); + if( data->is_classifier ) + node->class_idx = cvReadIntByName( fs, fnode, "norm_class_idx" ); + + node->Tn = cvReadIntByName( fs, fnode, "Tn" ); + node->complexity = cvReadIntByName( fs, fnode, "complexity" ); + node->alpha = cvReadRealByName( fs, fnode, "alpha" ); + node->node_risk = cvReadRealByName( fs, fnode, "node_risk" ); + node->tree_risk = cvReadRealByName( fs, fnode, "tree_risk" ); + node->tree_error = cvReadRealByName( fs, fnode, "tree_error" ); + + splits = cvGetFileNodeByName( fs, fnode, "splits" ); + if( splits ) + { + CvSeqReader reader; + CvDTreeSplit* last_split = 0; + + if( CV_NODE_TYPE(splits->tag) != CV_NODE_SEQ ) + CV_ERROR( CV_StsParseError, "splits tag must stored as a sequence" ); + + cvStartReadSeq( splits->data.seq, &reader ); + for( i = 0; i < reader.seq->total; i++ ) + { + CvDTreeSplit* split; + CV_CALL( split = read_split( fs, (CvFileNode*)reader.ptr )); + if( !last_split ) + node->split = last_split = split; + else + last_split = last_split->next = split; + + CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + } + } + + __END__; + + return node; +} + + +void CvDTree::read_tree_nodes( CvFileStorage* fs, CvFileNode* fnode ) +{ + CV_FUNCNAME( "CvDTree::read_tree_nodes" ); + + __BEGIN__; + + CvSeqReader reader; + CvDTreeNode _root; + CvDTreeNode* parent = &_root; + int i; + parent->left = parent->right = parent->parent = 0; + + cvStartReadSeq( fnode->data.seq, &reader ); + + for( i = 0; i < reader.seq->total; i++ ) + { + CvDTreeNode* node; + + CV_CALL( node = read_node( fs, (CvFileNode*)reader.ptr, parent != &_root ? parent : 0 )); + if( !parent->left ) + parent->left = node; + else + parent->right = node; + if( node->split ) + parent = node; + else + { + while( parent && parent->right ) + parent = parent->parent; + } + + CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + } + + root = _root.left; + + __END__; +} + + +void CvDTree::read( CvFileStorage* fs, CvFileNode* fnode ) +{ + CvDTreeTrainData* _data = new CvDTreeTrainData(); + _data->read_params( fs, fnode ); + + read( fs, fnode, _data ); + get_var_importance(); +} + + +// a special entry point for reading weak decision trees from the tree ensembles +void CvDTree::read( CvFileStorage* fs, CvFileNode* node, CvDTreeTrainData* _data ) +{ + CV_FUNCNAME( "CvDTree::read" ); + + __BEGIN__; + + CvFileNode* tree_nodes; + + clear(); + data = _data; + + tree_nodes = cvGetFileNodeByName( fs, node, "nodes" ); + if( !tree_nodes || CV_NODE_TYPE(tree_nodes->tag) != CV_NODE_SEQ ) + CV_ERROR( CV_StsParseError, "nodes tag is missing" ); + + pruned_tree_idx = cvReadIntByName( fs, node, "best_tree_idx", -1 ); + read_tree_nodes( fs, tree_nodes ); + + __END__; +} + +Mat CvDTree::getVarImportance() +{ + return cvarrToMat(get_var_importance()); +} + +/* End of file. */ diff --git a/apps/traincascade/traincascade.cpp b/apps/traincascade/traincascade.cpp index 52bacc8083082e5df3dd392a10bdc7c5957c3dcb..d1c3e4e87a891e13786d2b2ebaea1885ca82207b 100644 --- a/apps/traincascade/traincascade.cpp +++ b/apps/traincascade/traincascade.cpp @@ -1,6 +1,4 @@ #include "opencv2/core.hpp" - -#include "cv.h" #include "cascadeclassifier.h" using namespace std; diff --git a/apps/traincascade/traincascade_features.h b/apps/traincascade/traincascade_features.h index dfba7a3d430e004cb5bd5ae1ac001853f26207a4..c8f024b507c74d3db3b391b4984e90dec444800d 100644 --- a/apps/traincascade/traincascade_features.h +++ b/apps/traincascade/traincascade_features.h @@ -2,9 +2,6 @@ #define _OPENCV_FEATURES_H_ #include "imagestorage.h" -#include "cxcore.h" -#include "cv.h" -#include "ml.h" #include #define FEATURES "features" diff --git a/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.rst b/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.rst index b484e8d03c4f80039d29e4916c4455fc0a108bbc..34859084cc1b26b6d7efb262fa7feacf9c439374 100644 --- a/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.rst +++ b/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.rst @@ -760,6 +760,27 @@ They are :math:`[R_2, -t]`. By decomposing ``E``, you can only get the direction of the translation, so the function returns unit ``t``. +decomposeHomographyMat +-------------------------- +Decompose a homography matrix to rotation(s), translation(s) and plane normal(s). + +.. ocv:function:: int decomposeHomographyMat( InputArray H, InputArray K, OutputArrayOfArrays rotations, OutputArrayOfArrays translations, OutputArrayOfArrays normals) + + :param H: The input homography matrix between two images. + + :param K: The input intrinsic camera calibration matrix. + + :param rotations: Array of rotation matrices. + + :param translations: Array of translation matrices. + + :param normals: Array of plane normal matrices. + +This function extracts relative camera motion between two views observing a planar object from the homography ``H`` induced by the plane. +The intrinsic camera matrix ``K`` must also be provided. The function may return up to four mathematical solution sets. At least two of the +solutions may further be invalidated if point correspondences are available by applying positive depth constraint (all points must be in front of the camera). +The decomposition method is described in detail in [Malis]_. + recoverPose --------------- @@ -1518,3 +1539,5 @@ The function reconstructs 3-dimensional points (in homogeneous coordinates) by u .. [Slabaugh] Slabaugh, G.G. Computing Euler angles from a rotation matrix. http://www.soi.city.ac.uk/~sbbh653/publications/euler.pdf (verified: 2013-04-15) .. [Zhang2000] Z. Zhang. A Flexible New Technique for Camera Calibration. IEEE Transactions on Pattern Analysis and Machine Intelligence, 22(11):1330-1334, 2000. + +.. [Malis] Malis, E. and Vargas, M. Deeper understanding of the homography decomposition for vision-based control, Research Report 6303, INRIA (2007) diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index 8b9b69c3aa45e9bdd0cf6ef93781ea4381a89ac1..b18c00f8d29eedbfb51f51fcae0fc382f9200823 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -314,6 +314,11 @@ CV_EXPORTS_W int estimateAffine3D(InputArray src, InputArray dst, double ransacThreshold = 3, double confidence = 0.99); +CV_EXPORTS_W int decomposeHomographyMat(InputArray H, + InputArray K, + OutputArrayOfArrays rotations, + OutputArrayOfArrays translations, + OutputArrayOfArrays normals); class CV_EXPORTS_W StereoMatcher : public Algorithm { diff --git a/modules/calib3d/src/homography_decomp.cpp b/modules/calib3d/src/homography_decomp.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1a642612f06daa4971fae38fa2bf0ba0d95dddd0 --- /dev/null +++ b/modules/calib3d/src/homography_decomp.cpp @@ -0,0 +1,482 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // This is a homography decomposition implementation contributed to OpenCV + // by Samson Yilma. It implements the homography decomposition algorithm + // descriped in the research report: + // Malis, E and Vargas, M, "Deeper understanding of the homography decomposition + // for vision-based control", Research Report 6303, INRIA (2007) + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2014, Samson Yilma¸ (samson_yilma@yahoo.com), all rights reserved. + // + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "precomp.hpp" +#include + +namespace cv +{ + +namespace HomographyDecomposition +{ + +//struct to hold solutions of homography decomposition +typedef struct _CameraMotion { + cv::Matx33d R; //!< rotation matrix + cv::Vec3d n; //!< normal of the plane the camera is looking at + cv::Vec3d t; //!< translation vector +} CameraMotion; + +inline int signd(const double x) +{ + return ( x >= 0 ? 1 : -1 ); +} + +class HomographyDecomp { + +public: + HomographyDecomp() {} + virtual ~HomographyDecomp() {} + virtual void decomposeHomography(const cv::Matx33d& H, const cv::Matx33d& K, + std::vector& camMotions); + bool isRotationValid(const cv::Matx33d& R, const double epsilon=0.01); + +protected: + bool passesSameSideOfPlaneConstraint(CameraMotion& motion); + virtual void decompose(std::vector& camMotions) = 0; + const cv::Matx33d& getHnorm() const { + return _Hnorm; + } + +private: + cv::Matx33d normalize(const cv::Matx33d& H, const cv::Matx33d& K); + void removeScale(); + cv::Matx33d _Hnorm; +}; + +class HomographyDecompZhang : public HomographyDecomp { + +public: + HomographyDecompZhang():HomographyDecomp() {} + virtual ~HomographyDecompZhang() {} + +private: + virtual void decompose(std::vector& camMotions); + bool findMotionFrom_tstar_n(const cv::Vec3d& tstar, const cv::Vec3d& n, CameraMotion& motion); +}; + +class HomographyDecompInria : public HomographyDecomp { + +public: + HomographyDecompInria():HomographyDecomp() {} + virtual ~HomographyDecompInria() {} + +private: + virtual void decompose(std::vector& camMotions); + double oppositeOfMinor(const cv::Matx33d& M, const int row, const int col); + void findRmatFrom_tstar_n(const cv::Vec3d& tstar, const cv::Vec3d& n, const double v, cv::Matx33d& R); +}; + +// normalizes homography with intrinsic camera parameters +Matx33d HomographyDecomp::normalize(const Matx33d& H, const Matx33d& K) +{ + return K.inv() * H * K; +} + +void HomographyDecomp::removeScale() +{ + Mat W; + SVD::compute(_Hnorm, W); + _Hnorm = _Hnorm * (1.0/W.at(1)); +} + +/*! This checks that the input is a pure rotation matrix 'm'. + * The conditions for this are: R' * R = I and det(R) = 1 (proper rotation matrix) + */ +bool HomographyDecomp::isRotationValid(const Matx33d& R, const double epsilon) +{ + Matx33d RtR = R.t() * R; + Matx33d I(1,0,0, 0,1,0, 0,0,1); + if (norm(RtR, I, NORM_INF) > epsilon) + return false; + return (fabs(determinant(R) - 1.0) < epsilon); +} + +bool HomographyDecomp::passesSameSideOfPlaneConstraint(CameraMotion& motion) +{ + typedef Matx Matx11d; + Matx31d t = Matx31d(motion.t); + Matx31d n = Matx31d(motion.n); + Matx11d proj = n.t() * motion.R.t() * t; + if ( (1 + proj(0, 0) ) <= 0 ) + return false; + return true; +} + +//!main routine to decompose homography +void HomographyDecomp::decomposeHomography(const Matx33d& H, const cv::Matx33d& K, + std::vector& camMotions) +{ + //normalize homography matrix with intrinsic camera matrix + _Hnorm = normalize(H, K); + //remove scale of the normalized homography + removeScale(); + //apply decomposition + decompose(camMotions); +} + +/* function computes R&t from tstar, and plane normal(n) using + R = H * inv(I + tstar*transpose(n) ); + t = R * tstar; + returns true if computed R&t is a valid solution + */ +bool HomographyDecompZhang::findMotionFrom_tstar_n(const cv::Vec3d& tstar, const cv::Vec3d& n, CameraMotion& motion) +{ + Matx31d tstar_m = Mat(tstar); + Matx31d n_m = Mat(n); + Matx33d temp = tstar_m * n_m.t(); + temp(0, 0) += 1.0; + temp(1, 1) += 1.0; + temp(2, 2) += 1.0; + motion.R = getHnorm() * temp.inv(); + motion.t = motion.R * tstar; + motion.n = n; + return passesSameSideOfPlaneConstraint(motion); +} + +void HomographyDecompZhang::decompose(std::vector& camMotions) +{ + Mat W, U, Vt; + SVD::compute(getHnorm(), W, U, Vt); + double lambda1=W.at(0); + double lambda3=W.at(2); + double lambda1m3 = (lambda1-lambda3); + double lambda1m3_2 = lambda1m3*lambda1m3; + double lambda1t3 = lambda1*lambda3; + + double t1 = 1.0/(2.0*lambda1t3); + double t2 = sqrt(1.0+4.0*lambda1t3/lambda1m3_2); + double t12 = t1*t2; + + double e1 = -t1 + t12; //t1*(-1.0f + t2 ); + double e3 = -t1 - t12; //t1*(-1.0f - t2); + double e1_2 = e1*e1; + double e3_2 = e3*e3; + + double nv1p = sqrt(e1_2*lambda1m3_2 + 2*e1*(lambda1t3-1) + 1.0); + double nv3p = sqrt(e3_2*lambda1m3_2 + 2*e3*(lambda1t3-1) + 1.0); + double v1p[3], v3p[3]; + + v1p[0]=Vt.at(0)*nv1p, v1p[1]=Vt.at(1)*nv1p, v1p[2]=Vt.at(2)*nv1p; + v3p[0]=Vt.at(6)*nv3p, v3p[1]=Vt.at(7)*nv3p, v3p[2]=Vt.at(8)*nv3p; + + /*The eight solutions are + (A): tstar = +- (v1p - v3p)/(e1 -e3), n = +- (e1*v3p - e3*v1p)/(e1-e3) + (B): tstar = +- (v1p + v3p)/(e1 -e3), n = +- (e1*v3p + e3*v1p)/(e1-e3) + */ + double v1pmv3p[3], v1ppv3p[3]; + double e1v3me3v1[3], e1v3pe3v1[3]; + double inv_e1me3 = 1.0/(e1-e3); + + for(int kk=0;kk<3;++kk){ + v1pmv3p[kk] = v1p[kk]-v3p[kk]; + v1ppv3p[kk] = v1p[kk]+v3p[kk]; + } + + for(int kk=0; kk<3; ++kk){ + double e1v3 = e1*v3p[kk]; + double e3v1=e3*v1p[kk]; + e1v3me3v1[kk] = e1v3-e3v1; + e1v3pe3v1[kk] = e1v3+e3v1; + } + + Vec3d tstar_p, tstar_n; + Vec3d n_p, n_n; + + ///Solution group A + for(int kk=0; kk<3; ++kk) { + tstar_p[kk] = v1pmv3p[kk]*inv_e1me3; + tstar_n[kk] = -tstar_p[kk]; + n_p[kk] = e1v3me3v1[kk]*inv_e1me3; + n_n[kk] = -n_p[kk]; + } + + CameraMotion cmotion; + //(A) Four different combinations for solution A + // (i) (+, +) + if (findMotionFrom_tstar_n(tstar_p, n_p, cmotion)) + camMotions.push_back(cmotion); + + // (ii) (+, -) + if (findMotionFrom_tstar_n(tstar_p, n_n, cmotion)) + camMotions.push_back(cmotion); + + // (iii) (-, +) + if (findMotionFrom_tstar_n(tstar_n, n_p, cmotion)) + camMotions.push_back(cmotion); + + // (iv) (-, -) + if (findMotionFrom_tstar_n(tstar_n, n_n, cmotion)) + camMotions.push_back(cmotion); + ////////////////////////////////////////////////////////////////// + ///Solution group B + for(int kk=0;kk<3;++kk){ + tstar_p[kk] = v1ppv3p[kk]*inv_e1me3; + tstar_n[kk] = -tstar_p[kk]; + n_p[kk] = e1v3pe3v1[kk]*inv_e1me3; + n_n[kk] = -n_p[kk]; + } + + //(B) Four different combinations for solution B + // (i) (+, +) + if (findMotionFrom_tstar_n(tstar_p, n_p, cmotion)) + camMotions.push_back(cmotion); + + // (ii) (+, -) + if (findMotionFrom_tstar_n(tstar_p, n_n, cmotion)) + camMotions.push_back(cmotion); + + // (iii) (-, +) + if (findMotionFrom_tstar_n(tstar_n, n_p, cmotion)) + camMotions.push_back(cmotion); + + // (iv) (-, -) + if (findMotionFrom_tstar_n(tstar_n, n_n, cmotion)) + camMotions.push_back(cmotion); +} + +double HomographyDecompInria::oppositeOfMinor(const Matx33d& M, const int row, const int col) +{ + int x1 = col == 0 ? 1 : 0; + int x2 = col == 2 ? 1 : 2; + int y1 = row == 0 ? 1 : 0; + int y2 = row == 2 ? 1 : 2; + + return (M(y1, x2) * M(y2, x1) - M(y1, x1) * M(y2, x2)); +} + +//computes R = H( I - (2/v)*te_star*ne_t ) +void HomographyDecompInria::findRmatFrom_tstar_n(const cv::Vec3d& tstar, const cv::Vec3d& n, const double v, cv::Matx33d& R) +{ + Matx31d tstar_m = Matx31d(tstar); + Matx31d n_m = Matx31d(n); + Matx33d I(1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0); + + R = getHnorm() * (I - (2/v) * tstar_m * n_m.t() ); +} + +void HomographyDecompInria::decompose(std::vector& camMotions) +{ + const double epsilon = 0.001; + Matx33d S; + + //S = H'H - I + S = getHnorm().t() * getHnorm(); + S(0, 0) -= 1.0; + S(1, 1) -= 1.0; + S(2, 2) -= 1.0; + + //check if H is rotation matrix + if( norm(S, NORM_INF) < epsilon) { + CameraMotion motion; + motion.R = Matx33d(getHnorm()); + motion.t = Vec3d(0, 0, 0); + motion.n = Vec3d(0, 0, 0); + camMotions.push_back(motion); + return; + } + + //! Compute nvectors + Vec3d npa, npb; + + double M00 = oppositeOfMinor(S, 0, 0); + double M11 = oppositeOfMinor(S, 1, 1); + double M22 = oppositeOfMinor(S, 2, 2); + + double rtM00 = sqrt(M00); + double rtM11 = sqrt(M11); + double rtM22 = sqrt(M22); + + double M01 = oppositeOfMinor(S, 0, 1); + double M12 = oppositeOfMinor(S, 1, 2); + double M02 = oppositeOfMinor(S, 0, 2); + + int e12 = signd(M12); + int e02 = signd(M02); + int e01 = signd(M01); + + double nS00 = abs(S(0, 0)); + double nS11 = abs(S(1, 1)); + double nS22 = abs(S(2, 2)); + + //find max( |Sii| ), i=0, 1, 2 + int indx = 0; + if(nS00 < nS11){ + indx = 1; + if( nS11 < nS22 ) + indx = 2; + } + else { + if(nS00 < nS22 ) + indx = 2; + } + + switch (indx) { + case 0: + npa[0] = S(0, 0), npb[0] = S(0, 0); + npa[1] = S(0, 1) + rtM22, npb[1] = S(0, 1) - rtM22; + npa[2] = S(0, 2) + e12 * rtM11, npb[2] = S(0, 2) - e12 * rtM11; + break; + case 1: + npa[0] = S(0, 1) + rtM22, npb[0] = S(0, 1) - rtM22; + npa[1] = S(1, 1), npb[1] = S(1, 1); + npa[2] = S(1, 2) - e02 * rtM00, npb[2] = S(1, 2) + e02 * rtM00; + break; + case 2: + npa[0] = S(0, 2) + e01 * rtM11, npb[0] = S(0, 2) - e01 * rtM11; + npa[1] = S(1, 2) + rtM00, npb[1] = S(1, 2) - rtM00; + npa[2] = S(2, 2), npb[2] = S(2, 2); + break; + default: + break; + } + + double traceS = S(0, 0) + S(1, 1) + S(2, 2); + double v = 2.0 * sqrt(1 + traceS - M00 - M11 - M22); + + double ESii = signd(S(indx, indx)) ; + double r_2 = 2 + traceS + v; + double nt_2 = 2 + traceS - v; + + double r = sqrt(r_2); + double n_t = sqrt(nt_2); + + Vec3d na = npa / norm(npa); + Vec3d nb = npb / norm(npb); + + double half_nt = 0.5 * n_t; + double esii_t_r = ESii * r; + + Vec3d ta_star = half_nt * (esii_t_r * nb - n_t * na); + Vec3d tb_star = half_nt * (esii_t_r * na - n_t * nb); + + camMotions.resize(4); + + Matx33d Ra, Rb; + Vec3d ta, tb; + + //Ra, ta, na + findRmatFrom_tstar_n(ta_star, na, v, Ra); + ta = Ra * ta_star; + + camMotions[0].R = Ra; + camMotions[0].t = ta; + camMotions[0].n = na; + + //Ra, -ta, -na + camMotions[1].R = Ra; + camMotions[1].t = -ta; + camMotions[1].n = -na; + + //Rb, tb, nb + findRmatFrom_tstar_n(tb_star, nb, v, Rb); + tb = Rb * tb_star; + + camMotions[2].R = Rb; + camMotions[2].t = tb; + camMotions[2].n = nb; + + //Rb, -tb, -nb + camMotions[3].R = Rb; + camMotions[3].t = -tb; + camMotions[3].n = -nb; +} + +} //namespace HomographyDecomposition + +// function decomposes image-to-image homography to rotation and translation matrices +int decomposeHomographyMat(InputArray _H, + InputArray _K, + OutputArrayOfArrays _rotations, + OutputArrayOfArrays _translations, + OutputArrayOfArrays _normals) +{ + using namespace std; + using namespace HomographyDecomposition; + + Mat H = _H.getMat().reshape(1, 3); + CV_Assert(H.cols == 3 && H.rows == 3); + + Mat K = _K.getMat().reshape(1, 3); + CV_Assert(K.cols == 3 && K.rows == 3); + + auto_ptr hdecomp(new HomographyDecompInria); + + vector motions; + hdecomp->decomposeHomography(H, K, motions); + + int nsols = static_cast(motions.size()); + int depth = CV_64F; //double precision matrices used in CameraMotion struct + + if (_rotations.needed()) { + _rotations.create(nsols, 1, depth); + for (int k = 0; k < nsols; ++k ) { + _rotations.getMatRef(k) = Mat(motions[k].R); + } + } + + if (_translations.needed()) { + _translations.create(nsols, 1, depth); + for (int k = 0; k < nsols; ++k ) { + _translations.getMatRef(k) = Mat(motions[k].t); + } + } + + if (_normals.needed()) { + _normals.create(nsols, 1, depth); + for (int k = 0; k < nsols; ++k ) { + _normals.getMatRef(k) = Mat(motions[k].n); + } + } + + return nsols; +} + +} //namespace cv diff --git a/modules/calib3d/test/test_homography_decomp.cpp b/modules/calib3d/test/test_homography_decomp.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7e1c8ea503159c852f0585ba82e3fdcf4ea4acb2 --- /dev/null +++ b/modules/calib3d/test/test_homography_decomp.cpp @@ -0,0 +1,138 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// + // + // This is a test file for the function decomposeHomography contributed to OpenCV + // by Samson Yilma. + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // + // Copyright (C) 2014, Samson Yilma¸ (samson_yilma@yahoo.com), all rights reserved. + // + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + +#include "test_precomp.hpp" +#include "opencv2/calib3d.hpp" +#include + +using namespace cv; +using namespace std; + +class CV_HomographyDecompTest: public cvtest::BaseTest { + +public: + CV_HomographyDecompTest() + { + buildTestDataSet(); + } + +protected: + void run(int) + { + vector rotations; + vector translations; + vector normals; + + decomposeHomographyMat(_H, _K, rotations, translations, normals); + + //there should be at least 1 solution + ASSERT_GT(static_cast(rotations.size()), 0); + ASSERT_GT(static_cast(translations.size()), 0); + ASSERT_GT(static_cast(normals.size()), 0); + + ASSERT_EQ(rotations.size(), normals.size()); + ASSERT_EQ(translations.size(), normals.size()); + + ASSERT_TRUE(containsValidMotion(rotations, translations, normals)); + + decomposeHomographyMat(_H, _K, rotations, noArray(), noArray()); + ASSERT_GT(static_cast(rotations.size()), 0); + } + +private: + + void buildTestDataSet() + { + _K = Matx33d(640, 0.0, 320, + 0, 640, 240, + 0, 0, 1); + + _H = Matx33d(2.649157564634028, 4.583875997496426, 70.694447785121326, + -1.072756858861583, 3.533262150437228, 1513.656999614321649, + 0.001303887589576, 0.003042206876298, 1.000000000000000 + ); + + //expected solution for the given homography and intrinsic matrices + _R = Matx33d(0.43307983549125, 0.545749113549648, -0.717356090899523, + -0.85630229674426, 0.497582023798831, -0.138414255706431, + 0.281404038139784, 0.67421809131173, 0.682818960388909); + + _t = Vec3d(1.826751712278038, 1.264718492450820, 0.195080809998819); + _n = Vec3d(0.244875830334816, 0.480857890778889, 0.841909446789566); + } + + bool containsValidMotion(std::vector& rotations, + std::vector& translations, + std::vector& normals + ) + { + double max_error = 1.0e-3; + + vector::iterator riter = rotations.begin(); + vector::iterator titer = translations.begin(); + vector::iterator niter = normals.begin(); + + for (; + riter != rotations.end() && titer != translations.end() && niter != normals.end(); + ++riter, ++titer, ++niter) { + + double rdist = norm(*riter, _R, NORM_INF); + double tdist = norm(*titer, _t, NORM_INF); + double ndist = norm(*niter, _n, NORM_INF); + + if ( rdist < max_error + && tdist < max_error + && ndist < max_error ) + return true; + } + + return false; + } + + Matx33d _R, _K, _H; + Vec3d _t, _n; +}; + +TEST(Calib3d_DecomposeHomography, regression) { CV_HomographyDecompTest test; test.safe_run(); } diff --git a/modules/core/doc/basic_structures.rst b/modules/core/doc/basic_structures.rst index a94fa1731ea0012992651ba55c1e8b2d3a38f629..4f238df492a107eaaeb70f6ba16584af7b7703c5 100644 --- a/modules/core/doc/basic_structures.rst +++ b/modules/core/doc/basic_structures.rst @@ -845,7 +845,6 @@ For convenience, the following types from the OpenCV C API already have such a s that calls the appropriate release function: * ``CvCapture`` -* :ocv:struct:`CvDTreeSplit` * :ocv:struct:`CvFileStorage` * ``CvHaarClassifierCascade`` * :ocv:struct:`CvMat` diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index 99fa83a98d0b6e1e76996871400a03ad814dbbed..5ab0d49b9236123c9e77d038e6e52a27fce26e82 100644 --- a/modules/core/include/opencv2/core/ocl.hpp +++ b/modules/core/include/opencv2/core/ocl.hpp @@ -636,6 +636,9 @@ protected: CV_EXPORTS MatAllocator* getOpenCLAllocator(); +CV_EXPORTS_W bool isPerformanceCheckBypassed(); +#define OCL_PERFORMANCE_CHECK(condition) (cv::ocl::isPerformanceCheckBypassed() || (condition)) + }} #endif diff --git a/modules/core/src/arithm.cpp b/modules/core/src/arithm.cpp index 29501a0715fa6d9816912bd93571a4c9a42d4fb1..2211fcd367e4af58178f7f92c611de9cbdc4cfc0 100644 --- a/modules/core/src/arithm.cpp +++ b/modules/core/src/arithm.cpp @@ -1491,6 +1491,9 @@ static bool ocl_arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst, if (!doubleSupport && (depth2 == CV_64F || depth1 == CV_64F)) return false; + if( (oclop == OCL_OP_MUL_SCALE || oclop == OCL_OP_DIV_SCALE) && (depth1 >= CV_32F || depth2 >= CV_32F || ddepth >= CV_32F) ) + return false; + int kercn = haveMask || haveScalar ? cn : ocl::predictOptimalVectorWidth(_src1, _src2, _dst); int scalarcn = kercn == 3 ? 4 : kercn, rowsPerWI = d.isIntel() ? 4 : 1; @@ -1604,7 +1607,7 @@ static void arithm_op(InputArray _src1, InputArray _src2, OutputArray _dst, Size sz1 = dims1 <= 2 ? psrc1->size() : Size(); Size sz2 = dims2 <= 2 ? psrc2->size() : Size(); #ifdef HAVE_OPENCL - bool use_opencl = _dst.isUMat() && dims1 <= 2 && dims2 <= 2; + bool use_opencl = OCL_PERFORMANCE_CHECK(_dst.isUMat()) && dims1 <= 2 && dims2 <= 2; #endif bool src1Scalar = checkScalar(*psrc1, type2, kind1, kind2); bool src2Scalar = checkScalar(*psrc2, type1, kind2, kind1); @@ -2979,7 +2982,7 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op) haveScalar = true; } - CV_OCL_RUN(_src1.dims() <= 2 && _src2.dims() <= 2 && _dst.isUMat(), + CV_OCL_RUN(_src1.dims() <= 2 && _src2.dims() <= 2 && OCL_PERFORMANCE_CHECK(_dst.isUMat()), ocl_compare(_src1, _src2, _dst, op, haveScalar)) int kind1 = _src1.kind(), kind2 = _src2.kind(); @@ -3497,7 +3500,7 @@ void cv::inRange(InputArray _src, InputArray _lowerb, InputArray _upperb, OutputArray _dst) { CV_OCL_RUN(_src.dims() <= 2 && _lowerb.dims() <= 2 && - _upperb.dims() <= 2 && _dst.isUMat(), + _upperb.dims() <= 2 && OCL_PERFORMANCE_CHECK(_dst.isUMat()), ocl_inRange(_src, _lowerb, _upperb, _dst)) int skind = _src.kind(), lkind = _lowerb.kind(), ukind = _upperb.kind(); diff --git a/modules/core/src/convert.cpp b/modules/core/src/convert.cpp index 21d5bdaca759c0179d3922e9b4b8aa245f55cb08..d6abaa4adbf80dfd5be07984c787ece61721a54a 100644 --- a/modules/core/src/convert.cpp +++ b/modules/core/src/convert.cpp @@ -1541,7 +1541,7 @@ static bool ocl_convertScaleAbs( InputArray _src, OutputArray _dst, double alpha kercn = ocl::predictOptimalVectorWidth(_src, _dst), rowsPerWI = d.isIntel() ? 4 : 1; bool doubleSupport = d.doubleFPConfig() > 0; - if (!doubleSupport && depth == CV_64F) + if (depth == CV_32F || depth == CV_64F) return false; char cvt[2][50]; diff --git a/modules/core/src/copy.cpp b/modules/core/src/copy.cpp index 6900b518037563b31fe9c51a6d467f7538065f65..8bd2f457d91d2c849e627fd880ea0ffb54df5e6b 100644 --- a/modules/core/src/copy.cpp +++ b/modules/core/src/copy.cpp @@ -432,7 +432,7 @@ Mat& Mat::setTo(InputArray _value, InputArray _mask) IppStatus status = (IppStatus)-1; IppiSize roisize = { cols, rows }; - int mstep = (int)mask.step, dstep = (int)step; + int mstep = (int)mask.step[0], dstep = (int)step[0]; if (isContinuous() && mask.isContinuous()) { @@ -616,7 +616,7 @@ static bool ocl_flip(InputArray _src, OutputArray _dst, int flipCode ) { CV_Assert(flipCode >= -1 && flipCode <= 1); int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type), - flipType, kercn = std::min(ocl::predictOptimalVectorWidth(_src, _dst), 4);; + flipType, kercn = std::min(ocl::predictOptimalVectorWidth(_src, _dst), 4); if (cn > 4) return false; @@ -631,7 +631,7 @@ static bool ocl_flip(InputArray _src, OutputArray _dst, int flipCode ) ocl::Device dev = ocl::Device::getDefault(); int pxPerWIy = (dev.isIntel() && (dev.type() & ocl::Device::TYPE_GPU)) ? 4 : 1; - kercn = std::max(kercn, cn); + kercn = (cn!=3 || flipType == FLIP_ROWS) ? std::max(kercn, cn) : cn; ocl::Kernel k(kernelName, ocl::core::flip_oclsrc, format( "-D T=%s -D T1=%s -D cn=%d -D PIX_PER_WI_Y=%d -D kercn=%d", @@ -762,7 +762,7 @@ void flip( InputArray _src, OutputArray _dst, int flip_mode ) flipHoriz( dst.data, dst.step, dst.data, dst.step, dst.size(), esz ); } -#ifdef HAVE_OPENCL +/*#ifdef HAVE_OPENCL static bool ocl_repeat(InputArray _src, int ny, int nx, OutputArray _dst) { @@ -790,7 +790,7 @@ static bool ocl_repeat(InputArray _src, int ny, int nx, OutputArray _dst) return k.run(2, globalsize, NULL, false); } -#endif +#endif*/ void repeat(InputArray _src, int ny, int nx, OutputArray _dst) { @@ -800,8 +800,8 @@ void repeat(InputArray _src, int ny, int nx, OutputArray _dst) Size ssize = _src.size(); _dst.create(ssize.height*ny, ssize.width*nx, _src.type()); - CV_OCL_RUN(_dst.isUMat(), - ocl_repeat(_src, ny, nx, _dst)) + /*CV_OCL_RUN(_dst.isUMat(), + ocl_repeat(_src, ny, nx, _dst))*/ Mat src = _src.getMat(), dst = _dst.getMat(); Size dsize = dst.size(); diff --git a/modules/core/src/lapack.cpp b/modules/core/src/lapack.cpp index f6bc7c88c96cd6a6413a11f09378e4554c1fe966..8895a5659310c2727234c80798376e6e9e8b3ecb 100644 --- a/modules/core/src/lapack.cpp +++ b/modules/core/src/lapack.cpp @@ -1557,13 +1557,17 @@ static void _SVDcompute( InputArray _aarr, OutputArray _w, { if( !at ) { - transpose(temp_u, _u); - temp_v.copyTo(_vt); + if( _u.needed() ) + transpose(temp_u, _u); + if( _vt.needed() ) + temp_v.copyTo(_vt); } else { - transpose(temp_v, _u); - temp_u.copyTo(_vt); + if( _u.needed() ) + transpose(temp_v, _u); + if( _vt.needed() ) + temp_u.copyTo(_vt); } } } diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index ba6df7261ad31de7c1c50117fd7de4fffb581f55..398abcaaa6e17e92bdb89e5f832e19ccafba4b6e 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -3336,7 +3336,7 @@ static inline void reduceSumC_8u16u16s32f_64f(const cv::Mat& srcmat, cv::Mat& ds stype == CV_32FC3 ? (ippiSumHint)ippiSum_32f_C3R : stype == CV_32FC4 ? (ippiSumHint)ippiSum_32f_C4R : 0; func = - sdepth == CV_8U ? (cv::ReduceFunc)cv::reduceC_ > : + sdepth == CV_8U ? (cv::ReduceFunc)cv::reduceC_ > : sdepth == CV_16U ? (cv::ReduceFunc)cv::reduceC_ > : sdepth == CV_16S ? (cv::ReduceFunc)cv::reduceC_ > : sdepth == CV_32F ? (cv::ReduceFunc)cv::reduceC_ > : 0; @@ -3459,6 +3459,9 @@ static bool ocl_reduce(InputArray _src, OutputArray _dst, if (!doubleSupport && (sdepth == CV_64F || ddepth == CV_64F)) return false; + if ((op == CV_REDUCE_SUM && sdepth == CV_32F) || op == CV_REDUCE_MIN || op == CV_REDUCE_MAX) + return false; + if (op == CV_REDUCE_AVG) { if (sdepth < CV_32S && ddepth < CV_32S) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 32db8c91b4678ec138dc5a134c9780abf7559897..433249a2ea406dc7d060707dc089fe7ae70a45a5 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -57,6 +57,28 @@ # endif #endif + +// TODO Move to some common place +static bool getBoolParameter(const char* name, bool defaultValue) +{ + const char* envValue = getenv(name); + if (envValue == NULL) + { + return defaultValue; + } + cv::String value = envValue; + if (value == "1" || value == "True" || value == "true" || value == "TRUE") + { + return true; + } + if (value == "0" || value == "False" || value == "false" || value == "FALSE") + { + return false; + } + CV_ErrorNoReturn(cv::Error::StsBadArg, cv::format("Invalid value for %s parameter: %s", name, value.c_str())); +} + + // TODO Move to some common place static size_t getConfigurationParameterForSize(const char* name, size_t defaultValue) { @@ -1302,10 +1324,22 @@ OCL_FUNC(cl_int, clReleaseEvent, (cl_event event), (event)) #endif +static bool isRaiseError() +{ + static bool initialized = false; + static bool value = false; + if (!initialized) + { + value = getBoolParameter("OPENCV_OPENCL_RAISE_ERROR", false); + initialized = true; + } + return value; +} + #ifdef _DEBUG #define CV_OclDbgAssert CV_DbgAssert #else -#define CV_OclDbgAssert(expr) (void)(expr) +#define CV_OclDbgAssert(expr) do { if (isRaiseError()) { CV_Assert(expr); } else { (void)(expr); } } while ((void)0, 0) #endif namespace cv { namespace ocl { @@ -4711,4 +4745,16 @@ void* Image2D::ptr() const return p ? p->handle : 0; } +bool isPerformanceCheckBypassed() +{ + static bool initialized = false; + static bool value = false; + if (!initialized) + { + value = getBoolParameter("OPENCV_OPENCL_PERF_CHECK_BYPASS", false); + initialized = true; + } + return value; +} + }} diff --git a/modules/core/src/opencl/meanstddev.cl b/modules/core/src/opencl/meanstddev.cl index 39e917e96dbaefc84ee3b5daf24cd45d6b4f59a6..ed68c645384dbe470c0a8e6fdb4d995ee9330f38 100644 --- a/modules/core/src/opencl/meanstddev.cl +++ b/modules/core/src/opencl/meanstddev.cl @@ -59,7 +59,7 @@ __kernel void meanStdDev(__global const uchar * srcptr, int src_step, int src_of for (int grain = groups * WGS; id < total; id += grain) { #ifdef HAVE_MASK -#ifdef HAVE_SRC_CONT +#ifdef HAVE_MASK_CONT int mask_index = id; #else int mask_index = mad24(id / cols, mask_step, id % cols); diff --git a/modules/core/src/opencl/minmaxloc.cl b/modules/core/src/opencl/minmaxloc.cl index 664673e5a2250722c8ae2896414751507c700223..1d84567ef987f98e0a1bce02b816d511965fae32 100644 --- a/modules/core/src/opencl/minmaxloc.cl +++ b/modules/core/src/opencl/minmaxloc.cl @@ -209,7 +209,7 @@ __kernel void minmaxloc(__global const uchar * srcptr, int src_step, int src_off #if kercn == 1 #ifdef NEED_MINVAL -#if NEED_MINLOC +#ifdef NEED_MINLOC if (minval > temp) { minval = temp; @@ -326,7 +326,7 @@ __kernel void minmaxloc(__global const uchar * srcptr, int src_step, int src_off int lid2 = lsize + lid; #ifdef NEED_MINVAL -#ifdef NEED_MAXLOC +#ifdef NEED_MINLOC if (localmem_min[lid] >= localmem_min[lid2]) { if (localmem_min[lid] == localmem_min[lid2]) diff --git a/modules/core/src/stat.cpp b/modules/core/src/stat.cpp index 888fd7cacc377ff23bdaf06f9ceeefcdd8084e3f..26bae7a44dbfd8c976ff2e27a2344d07a9ec4c81 100644 --- a/modules/core/src/stat.cpp +++ b/modules/core/src/stat.cpp @@ -568,7 +568,7 @@ cv::Scalar cv::sum( InputArray _src ) { #ifdef HAVE_OPENCL Scalar _res; - CV_OCL_RUN_(_src.isUMat() && _src.dims() <= 2, + CV_OCL_RUN_(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2, ocl_sum(_src, _res, OCL_OP_SUM), _res) #endif @@ -719,7 +719,7 @@ int cv::countNonZero( InputArray _src ) #ifdef HAVE_OPENCL int res = -1; - CV_OCL_RUN_(_src.isUMat() && _src.dims() <= 2, + CV_OCL_RUN_(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2, ocl_countNonZero(_src, res), res) #endif @@ -918,7 +918,8 @@ static bool ocl_meanStdDev( InputArray _src, OutputArray _mean, OutputArray _sdv { int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type); bool doubleSupport = ocl::Device::getDefault().doubleFPConfig() > 0, - isContinuous = _src.isContinuous(); + isContinuous = _src.isContinuous(), + isMaskContinuous = _mask.isContinuous(); const ocl::Device &defDev = ocl::Device::getDefault(); int groups = defDev.maxComputeUnits(); if (defDev.isIntel()) @@ -943,13 +944,14 @@ static bool ocl_meanStdDev( InputArray _src, OutputArray _mean, OutputArray _sdv char cvt[2][40]; String opts = format("-D srcT=%s -D srcT1=%s -D dstT=%s -D dstT1=%s -D sqddepth=%d" - " -D sqdstT=%s -D sqdstT1=%s -D convertToSDT=%s -D cn=%d%s" + " -D sqdstT=%s -D sqdstT1=%s -D convertToSDT=%s -D cn=%d%s%s" " -D convertToDT=%s -D WGS=%d -D WGS2_ALIGNED=%d%s%s", ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(dtype), ocl::typeToStr(ddepth), sqddepth, ocl::typeToStr(sqdtype), ocl::typeToStr(sqddepth), ocl::convertTypeStr(depth, sqddepth, cn, cvt[0]), cn, isContinuous ? " -D HAVE_SRC_CONT" : "", + isMaskContinuous ? " -D HAVE_MASK_CONT" : "", ocl::convertTypeStr(depth, ddepth, cn, cvt[1]), (int)wgs, wgs2_aligned, haveMask ? " -D HAVE_MASK" : "", doubleSupport ? " -D DOUBLE_SUPPORT" : ""); @@ -1025,7 +1027,7 @@ static bool ocl_meanStdDev( InputArray _src, OutputArray _mean, OutputArray _sdv void cv::meanStdDev( InputArray _src, OutputArray _mean, OutputArray _sdv, InputArray _mask ) { - CV_OCL_RUN(_src.isUMat() && _src.dims() <= 2, + CV_OCL_RUN(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2, ocl_meanStdDev(_src, _mean, _sdv, _mask)) Mat src = _src.getMat(), mask = _mask.getMat(); @@ -1571,7 +1573,7 @@ void cv::minMaxIdx(InputArray _src, double* minVal, CV_Assert( (cn == 1 && (_mask.empty() || _mask.type() == CV_8U)) || (cn > 1 && _mask.empty() && !minIdx && !maxIdx) ); - CV_OCL_RUN(_src.isUMat() && _src.dims() <= 2 && (_mask.empty() || _src.size() == _mask.size()), + CV_OCL_RUN(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2 && (_mask.empty() || _src.size() == _mask.size()), ocl_minMaxIdx(_src, minVal, maxVal, minIdx, maxIdx, _mask)) Mat src = _src.getMat(), mask = _mask.getMat(); @@ -2234,7 +2236,7 @@ double cv::norm( InputArray _src, int normType, InputArray _mask ) #ifdef HAVE_OPENCL double _result = 0; - CV_OCL_RUN_(_src.isUMat() && _src.dims() <= 2, + CV_OCL_RUN_(OCL_PERFORMANCE_CHECK(_src.isUMat()) && _src.dims() <= 2, ocl_norm(_src, normType, _mask, _result), _result) #endif @@ -2283,7 +2285,7 @@ double cv::norm( InputArray _src, int normType, InputArray _mask ) setIppErrorStatus(); } - typedef IppStatus (CV_STDCALL* ippiMaskNormFuncC3)(const void *, int, const void *, int, IppiSize, int, Ipp64f *); + /*typedef IppStatus (CV_STDCALL* ippiMaskNormFuncC3)(const void *, int, const void *, int, IppiSize, int, Ipp64f *); ippiMaskNormFuncC3 ippFuncC3 = normType == NORM_INF ? (type == CV_8UC3 ? (ippiMaskNormFuncC3)ippiNorm_Inf_8u_C3CMR : @@ -2318,7 +2320,7 @@ double cv::norm( InputArray _src, int normType, InputArray _mask ) return normType == NORM_L2SQR ? (double)(norm * norm) : (double)norm; } setIppErrorStatus(); - } + }*/ } else { @@ -2594,7 +2596,7 @@ double cv::norm( InputArray _src1, InputArray _src2, int normType, InputArray _m #ifdef HAVE_OPENCL double _result = 0; - CV_OCL_RUN_(_src1.isUMat(), + CV_OCL_RUN_(OCL_PERFORMANCE_CHECK(_src1.isUMat()), ocl_norm(_src1, _src2, normType, _mask, _result), _result) #endif @@ -2724,7 +2726,7 @@ double cv::norm( InputArray _src1, InputArray _src2, int normType, InputArray _m 0) : normType == NORM_L1 ? (type == CV_8UC1 ? (ippiMaskNormDiffFuncC1)ippiNormDiff_L1_8u_C1MR : - type == CV_8SC1 ? (ippiMaskNormDiffFuncC1)ippiNormDiff_L1_8s_C1MR : + //type == CV_8SC1 ? (ippiMaskNormDiffFuncC1)ippiNormDiff_L1_8s_C1MR : type == CV_16UC1 ? (ippiMaskNormDiffFuncC1)ippiNormDiff_L1_16u_C1MR : type == CV_32FC1 ? (ippiMaskNormDiffFuncC1)ippiNormDiff_L1_32f_C1MR : 0) : @@ -2741,7 +2743,7 @@ double cv::norm( InputArray _src1, InputArray _src2, int normType, InputArray _m return normType == NORM_L2SQR ? (double)(norm * norm) : (double)norm; setIppErrorStatus(); } - typedef IppStatus (CV_STDCALL* ippiMaskNormDiffFuncC3)(const void *, int, const void *, int, const void *, int, IppiSize, int, Ipp64f *); + /*typedef IppStatus (CV_STDCALL* ippiMaskNormDiffFuncC3)(const void *, int, const void *, int, const void *, int, IppiSize, int, Ipp64f *); ippiMaskNormDiffFuncC3 ippFuncC3 = normType == NORM_INF ? (type == CV_8UC3 ? (ippiMaskNormDiffFuncC3)ippiNormDiff_Inf_8u_C3CMR : @@ -2776,7 +2778,7 @@ double cv::norm( InputArray _src1, InputArray _src2, int normType, InputArray _m return normType == NORM_L2SQR ? (double)(norm * norm) : (double)norm; } setIppErrorStatus(); - } + }*/ } else { diff --git a/modules/core/test/ocl/test_arithm.cpp b/modules/core/test/ocl/test_arithm.cpp index a7a09cabb7dc40b8117aa505ae0212b9e82c6bf3..b0905b19f06b7d0b00f83830de1e19a5d1f0186d 100644 --- a/modules/core/test/ocl/test_arithm.cpp +++ b/modules/core/test/ocl/test_arithm.cpp @@ -157,6 +157,7 @@ PARAM_TEST_CASE(ArithmTestBase, MatDepth, Channels, bool) Border maskBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(mask, mask_roi, roiSize, maskBorder, CV_8UC1, 0, 2); cv::threshold(mask, mask, 0.5, 255., CV_8UC1); + *mask.ptr(0) = 255; // prevent test case with mask filled 0 only val = cv::Scalar(rng.uniform(-100.0, 100.0), rng.uniform(-100.0, 100.0), rng.uniform(-100.0, 100.0), rng.uniform(-100.0, 100.0)); @@ -829,7 +830,7 @@ OCL_TEST_P(Pow, Mat) { static const double pows[] = { -4, -1, -2.5, 0, 1, 2, 3.7, 4 }; - for (int j = 0; j < test_loop_times; j++) + for (int j = 0; j < 1/*test_loop_times*/; j++) for (int k = 0, size = sizeof(pows) / sizeof(double); k < size; ++k) { SCOPED_TRACE(pows[k]); @@ -1203,7 +1204,7 @@ OCL_TEST_P(MinMaxIdx_Mask, Mat) static bool relativeError(double actual, double expected, double eps) { - return std::abs(actual - expected) / actual < eps; + return std::abs(actual - expected) < eps*(1 + std::abs(actual)); } typedef ArithmTestBase Norm; @@ -1230,7 +1231,7 @@ OCL_TEST_P(Norm, NORM_INF_1arg_mask) OCL_OFF(const double cpuRes = cv::norm(src1_roi, NORM_INF, mask_roi)); OCL_ON(const double gpuRes = cv::norm(usrc1_roi, NORM_INF, umask_roi)); - EXPECT_NEAR(cpuRes, gpuRes, 0.1); + EXPECT_NEAR(cpuRes, gpuRes, 0.2); } } @@ -1302,7 +1303,7 @@ OCL_TEST_P(Norm, NORM_INF_2args) OCL_OFF(const double cpuRes = cv::norm(src1_roi, src2_roi, type)); OCL_ON(const double gpuRes = cv::norm(usrc1_roi, usrc2_roi, type)); - EXPECT_NEAR(cpuRes, gpuRes, 0.1); + EXPECT_NEAR(cpuRes, gpuRes, 0.2); } } @@ -1419,7 +1420,7 @@ OCL_TEST_P(UMatDot, Mat) OCL_OFF(const double cpuRes = src1_roi.dot(src2_roi)); OCL_ON(const double gpuRes = usrc1_roi.dot(usrc2_roi)); - EXPECT_PRED3(relativeError, cpuRes, gpuRes, 1e-6); + EXPECT_PRED3(relativeError, cpuRes, gpuRes, 1e-5); } } @@ -1749,7 +1750,7 @@ OCL_TEST_P(ReduceAvg, Mat) OCL_OFF(cv::reduce(src_roi, dst_roi, dim, CV_REDUCE_AVG, dtype)); OCL_ON(cv::reduce(usrc_roi, udst_roi, dim, CV_REDUCE_AVG, dtype)); - double eps = ddepth <= CV_32S ? 1 : 5e-6; + double eps = ddepth <= CV_32S ? 1 : 6e-6; OCL_EXPECT_MATS_NEAR(dst, eps); } } diff --git a/modules/core/test/ocl/test_channels.cpp b/modules/core/test/ocl/test_channels.cpp index 7565273e7a09650cd76183c77c7bbfa77cc2add8..53d7de5d528857269a231d9b63de8b5cd56e018d 100644 --- a/modules/core/test/ocl/test_channels.cpp +++ b/modules/core/test/ocl/test_channels.cpp @@ -105,6 +105,7 @@ PARAM_TEST_CASE(Merge, MatDepth, int, bool) UMAT_UPLOAD_INPUT_PARAMETER(src3); UMAT_UPLOAD_INPUT_PARAMETER(src4); + src_roi.clear(); usrc_roi.clear(); // for test_loop_times > 1 src_roi.push_back(src1_roi), usrc_roi.push_back(usrc1_roi); if (nsrc >= 2) src_roi.push_back(src2_roi), usrc_roi.push_back(usrc2_roi); diff --git a/modules/core/test/ocl/test_matrix_operation.cpp b/modules/core/test/ocl/test_matrix_operation.cpp index ee591e9bd972af065b4f1c98e49b3f99bb31917d..252db01d168703d610406e5db993df21bc41d5ce 100644 --- a/modules/core/test/ocl/test_matrix_operation.cpp +++ b/modules/core/test/ocl/test_matrix_operation.cpp @@ -96,7 +96,7 @@ OCL_TEST_P(ConvertTo, Accuracy) OCL_OFF(src_roi.convertTo(dst_roi, dstType, alpha, beta)); OCL_ON(usrc_roi.convertTo(udst_roi, dstType, alpha, beta)); - double eps = src_depth >= CV_32F || CV_MAT_DEPTH(dstType) >= CV_32F ? 1e-4 : 1; + double eps = CV_MAT_DEPTH(dstType) >= CV_32F ? 2e-4 : 1; OCL_EXPECT_MATS_NEAR(dst, eps); } } @@ -121,7 +121,7 @@ PARAM_TEST_CASE(CopyTo, MatDepth, Channels, bool, bool) use_mask = GET_PARAM(3); } - void generateTestData() + void generateTestData(bool one_cn_mask = false) { const int type = CV_MAKE_TYPE(depth, cn); @@ -132,9 +132,11 @@ PARAM_TEST_CASE(CopyTo, MatDepth, Channels, bool, bool) if (use_mask) { Border maskBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); - int mask_cn = randomDouble(0.0, 2.0) > 1.0 ? cn : 1; + int mask_cn = 1; + if (!one_cn_mask && randomDouble(0.0, 2.0) > 1.0) + mask_cn = cn; randomSubMat(mask, mask_roi, roiSize, maskBorder, CV_8UC(mask_cn), 0, 2); - cv::threshold(mask, mask, 0.5, 255., CV_8UC1); + cv::threshold(mask, mask, 0.5, 255., THRESH_BINARY); } Border dstBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); @@ -177,7 +179,7 @@ OCL_TEST_P(SetTo, Accuracy) { for (int j = 0; j < test_loop_times; j++) { - generateTestData(); + generateTestData(true); // see modules/core/src/umatrix.cpp Ln:791 => CV_Assert( mask.size() == size() && mask.type() == CV_8UC1 ); if (use_mask) { diff --git a/modules/cudabgsegm/CMakeLists.txt b/modules/cudabgsegm/CMakeLists.txt index 3a882824b119eab181bf0ca3ec7961774ff38ff8..41517b6c698c19a1aebd30558b7750b6de11f78b 100644 --- a/modules/cudabgsegm/CMakeLists.txt +++ b/modules/cudabgsegm/CMakeLists.txt @@ -6,4 +6,4 @@ set(the_description "CUDA-accelerated Background Segmentation") ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-declarations) -ocv_define_module(cudabgsegm opencv_video OPTIONAL opencv_legacy opencv_imgproc opencv_cudaarithm opencv_cudafilters opencv_cudaimgproc) +ocv_define_module(cudabgsegm opencv_video OPTIONAL opencv_imgproc opencv_cudaarithm opencv_cudafilters opencv_cudaimgproc) diff --git a/modules/cudabgsegm/perf/perf_bgsegm.cpp b/modules/cudabgsegm/perf/perf_bgsegm.cpp index 02fc9a8ee9bdff4435e3cb5d2b31a7afcb764474..6e1ab467885c73c8f9ef937536c555f12b3ca822 100644 --- a/modules/cudabgsegm/perf/perf_bgsegm.cpp +++ b/modules/cudabgsegm/perf/perf_bgsegm.cpp @@ -42,10 +42,6 @@ #include "perf_precomp.hpp" -#ifdef HAVE_OPENCV_CUDALEGACY -# include "opencv2/cudalegacy.hpp" -#endif - #ifdef HAVE_OPENCV_CUDAIMGPROC # include "opencv2/cudaimgproc.hpp" #endif @@ -72,18 +68,6 @@ using namespace perf; #if BUILD_WITH_VIDEO_INPUT_SUPPORT -#ifdef HAVE_OPENCV_CUDALEGACY - -namespace cv -{ - template<> void DefaultDeleter::operator ()(CvBGStatModel* obj) const - { - cvReleaseBGStatModel(&obj); - } -} - -#endif - DEF_PARAM_TEST_1(Video, string); PERF_TEST_P(Video, FGDStatModel, @@ -150,48 +134,7 @@ PERF_TEST_P(Video, FGDStatModel, } else { -#ifdef HAVE_OPENCV_CUDALEGACY - IplImage ipl_frame = frame; - cv::Ptr model(cvCreateFGDStatModel(&ipl_frame)); - - int i = 0; - - // collect performance data - for (; i < numIters; ++i) - { - cap >> frame; - ASSERT_FALSE(frame.empty()); - - ipl_frame = frame; - - startTimer(); - if(!next()) - break; - - cvUpdateBGStatModel(&ipl_frame, model); - - stopTimer(); - } - - // process last frame in sequence to get data for sanity test - for (; i < numIters; ++i) - { - cap >> frame; - ASSERT_FALSE(frame.empty()); - - ipl_frame = frame; - - cvUpdateBGStatModel(&ipl_frame, model); - } - - const cv::Mat background = cv::cvarrToMat(model->background); - const cv::Mat foreground = cv::cvarrToMat(model->foreground); - - CPU_SANITY_CHECK(background); - CPU_SANITY_CHECK(foreground); -#else FAIL_NO_CPU(); -#endif } } diff --git a/modules/cudabgsegm/test/test_bgsegm.cpp b/modules/cudabgsegm/test/test_bgsegm.cpp index 34f3dcc9ab2555229768a974466ca753d4d9311a..89fd69474c0e37c49f04bde165579e863998dc81 100644 --- a/modules/cudabgsegm/test/test_bgsegm.cpp +++ b/modules/cudabgsegm/test/test_bgsegm.cpp @@ -42,10 +42,6 @@ #include "test_precomp.hpp" -#ifdef HAVE_OPENCV_CUDALEGACY -# include "opencv2/cudalegacy.hpp" -#endif - #ifdef HAVE_CUDA using namespace cvtest; @@ -63,80 +59,6 @@ using namespace cvtest; # define BUILD_WITH_VIDEO_INPUT_SUPPORT 0 #endif -////////////////////////////////////////////////////// -// FGDStatModel - -#if BUILD_WITH_VIDEO_INPUT_SUPPORT && defined(HAVE_OPENCV_CUDALEGACY) - -namespace cv -{ - template<> void DefaultDeleter::operator ()(CvBGStatModel* obj) const - { - cvReleaseBGStatModel(&obj); - } -} - -PARAM_TEST_CASE(FGDStatModel, cv::cuda::DeviceInfo, std::string) -{ - cv::cuda::DeviceInfo devInfo; - std::string inputFile; - - virtual void SetUp() - { - devInfo = GET_PARAM(0); - cv::cuda::setDevice(devInfo.deviceID()); - - inputFile = std::string(cvtest::TS::ptr()->get_data_path()) + "video/" + GET_PARAM(1); - } -}; - -CUDA_TEST_P(FGDStatModel, Update) -{ - cv::VideoCapture cap(inputFile); - ASSERT_TRUE(cap.isOpened()); - - cv::Mat frame; - cap >> frame; - ASSERT_FALSE(frame.empty()); - - IplImage ipl_frame = frame; - cv::Ptr model(cvCreateFGDStatModel(&ipl_frame)); - - cv::cuda::GpuMat d_frame(frame); - cv::Ptr d_fgd = cv::cuda::createBackgroundSubtractorFGD(); - cv::cuda::GpuMat d_foreground, d_background; - std::vector< std::vector > foreground_regions; - d_fgd->apply(d_frame, d_foreground); - - for (int i = 0; i < 5; ++i) - { - cap >> frame; - ASSERT_FALSE(frame.empty()); - - ipl_frame = frame; - int gold_count = cvUpdateBGStatModel(&ipl_frame, model); - - d_frame.upload(frame); - d_fgd->apply(d_frame, d_foreground); - d_fgd->getBackgroundImage(d_background); - d_fgd->getForegroundRegions(foreground_regions); - int count = (int) foreground_regions.size(); - - cv::Mat gold_background = cv::cvarrToMat(model->background); - cv::Mat gold_foreground = cv::cvarrToMat(model->foreground); - - ASSERT_MAT_NEAR(gold_background, d_background, 1.0); - ASSERT_MAT_NEAR(gold_foreground, d_foreground, 0.0); - ASSERT_EQ(gold_count, count); - } -} - -INSTANTIATE_TEST_CASE_P(CUDA_BgSegm, FGDStatModel, testing::Combine( - ALL_DEVICES, - testing::Values(std::string("768x576.avi")))); - -#endif - ////////////////////////////////////////////////////// // MOG diff --git a/modules/cudaoptflow/CMakeLists.txt b/modules/cudaoptflow/CMakeLists.txt index b7a2109fbb046e35cda2986464524bf7c8213393..f2d3e3da0b248c2a11e3586e74463e01abde03db 100644 --- a/modules/cudaoptflow/CMakeLists.txt +++ b/modules/cudaoptflow/CMakeLists.txt @@ -6,4 +6,4 @@ set(the_description "CUDA-accelerated Optical Flow") ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-declarations) -ocv_define_module(cudaoptflow opencv_video opencv_legacy opencv_cudaarithm opencv_cudawarping opencv_cudaimgproc OPTIONAL opencv_cudalegacy) +ocv_define_module(cudaoptflow opencv_video opencv_cudaarithm opencv_cudawarping opencv_cudaimgproc OPTIONAL opencv_cudalegacy) diff --git a/modules/cudaoptflow/perf/perf_optflow.cpp b/modules/cudaoptflow/perf/perf_optflow.cpp index 6c312ad0be386067383842f419df4d18e3ff8694..71ab8950824544b77f6e6acc7190864621e8bbb1 100644 --- a/modules/cudaoptflow/perf/perf_optflow.cpp +++ b/modules/cudaoptflow/perf/perf_optflow.cpp @@ -41,7 +41,6 @@ //M*/ #include "perf_precomp.hpp" -#include "opencv2/legacy.hpp" using namespace std; using namespace testing; @@ -389,24 +388,6 @@ PERF_TEST_P(ImagePair, OpticalFlowDual_TVL1, ////////////////////////////////////////////////////// // OpticalFlowBM -void calcOpticalFlowBM(const cv::Mat& prev, const cv::Mat& curr, - cv::Size bSize, cv::Size shiftSize, cv::Size maxRange, int usePrevious, - cv::Mat& velx, cv::Mat& vely) -{ - cv::Size sz((curr.cols - bSize.width + shiftSize.width)/shiftSize.width, (curr.rows - bSize.height + shiftSize.height)/shiftSize.height); - - velx.create(sz, CV_32FC1); - vely.create(sz, CV_32FC1); - - CvMat cvprev = prev; - CvMat cvcurr = curr; - - CvMat cvvelx = velx; - CvMat cvvely = vely; - - cvCalcOpticalFlowBM(&cvprev, &cvcurr, bSize, shiftSize, maxRange, usePrevious, &cvvelx, &cvvely); -} - PERF_TEST_P(ImagePair, OpticalFlowBM, Values(make_pair("gpu/opticalflow/frame0.png", "gpu/opticalflow/frame1.png"))) { @@ -435,12 +416,7 @@ PERF_TEST_P(ImagePair, OpticalFlowBM, } else { - cv::Mat u, v; - - TEST_CYCLE() calcOpticalFlowBM(frame0, frame1, block_size, shift_size, max_range, false, u, v); - - CPU_SANITY_CHECK(u); - CPU_SANITY_CHECK(v); + FAIL_NO_CPU(); } } diff --git a/modules/cudaoptflow/test/test_optflow.cpp b/modules/cudaoptflow/test/test_optflow.cpp index 110fed0339c058dd5c834525683ae79f836e4553..1de40510ddc82ef72f89d2f6d73faf97782dae4e 100644 --- a/modules/cudaoptflow/test/test_optflow.cpp +++ b/modules/cudaoptflow/test/test_optflow.cpp @@ -41,7 +41,6 @@ //M*/ #include "test_precomp.hpp" -#include "opencv2/legacy.hpp" #ifdef HAVE_CUDA @@ -370,65 +369,6 @@ INSTANTIATE_TEST_CASE_P(CUDA_OptFlow, OpticalFlowDual_TVL1, testing::Combine( ALL_DEVICES, WHOLE_SUBMAT)); -////////////////////////////////////////////////////// -// OpticalFlowBM - -namespace -{ - void calcOpticalFlowBM(const cv::Mat& prev, const cv::Mat& curr, - cv::Size bSize, cv::Size shiftSize, cv::Size maxRange, int usePrevious, - cv::Mat& velx, cv::Mat& vely) - { - cv::Size sz((curr.cols - bSize.width + shiftSize.width)/shiftSize.width, (curr.rows - bSize.height + shiftSize.height)/shiftSize.height); - - velx.create(sz, CV_32FC1); - vely.create(sz, CV_32FC1); - - CvMat cvprev = prev; - CvMat cvcurr = curr; - - CvMat cvvelx = velx; - CvMat cvvely = vely; - - cvCalcOpticalFlowBM(&cvprev, &cvcurr, bSize, shiftSize, maxRange, usePrevious, &cvvelx, &cvvely); - } -} - -struct OpticalFlowBM : testing::TestWithParam -{ -}; - -CUDA_TEST_P(OpticalFlowBM, Accuracy) -{ - cv::cuda::DeviceInfo devInfo = GetParam(); - cv::cuda::setDevice(devInfo.deviceID()); - - cv::Mat frame0 = readImage("opticalflow/rubberwhale1.png", cv::IMREAD_GRAYSCALE); - ASSERT_FALSE(frame0.empty()); - cv::resize(frame0, frame0, cv::Size(), 0.5, 0.5); - - cv::Mat frame1 = readImage("opticalflow/rubberwhale2.png", cv::IMREAD_GRAYSCALE); - ASSERT_FALSE(frame1.empty()); - cv::resize(frame1, frame1, cv::Size(), 0.5, 0.5); - - cv::Size block_size(8, 8); - cv::Size shift_size(1, 1); - cv::Size max_range(8, 8); - - cv::cuda::GpuMat d_velx, d_vely, buf; - cv::cuda::calcOpticalFlowBM(loadMat(frame0), loadMat(frame1), - block_size, shift_size, max_range, false, - d_velx, d_vely, buf); - - cv::Mat velx, vely; - calcOpticalFlowBM(frame0, frame1, block_size, shift_size, max_range, false, velx, vely); - - EXPECT_MAT_NEAR(velx, d_velx, 0); - EXPECT_MAT_NEAR(vely, d_vely, 0); -} - -INSTANTIATE_TEST_CASE_P(CUDA_OptFlow, OpticalFlowBM, ALL_DEVICES); - ////////////////////////////////////////////////////// // FastOpticalFlowBM diff --git a/modules/imgproc/src/color.cpp b/modules/imgproc/src/color.cpp index fe460ee75a32727d5210062b18349b8822d2df93..351ee74e43b15540a15ffde8083e600a9def0a89 100644 --- a/modules/imgproc/src/color.cpp +++ b/modules/imgproc/src/color.cpp @@ -2038,6 +2038,10 @@ struct Luv2RGB_f float G = X*C3 + Y*C4 + Z*C5; float B = X*C6 + Y*C7 + Z*C8; + R = std::min(std::max(R, 0.f), 1.f); + G = std::min(std::max(G, 0.f), 1.f); + B = std::min(std::max(B, 0.f), 1.f); + if( gammaTab ) { R = splineInterpolate(R*gscale, gammaTab, GAMMA_TAB_SIZE); diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 9326fa1932eb84e1b4fdda753c1d406d96b8f165..ff730ee941115b4b20dd6ba8f688d1f4f00d6528 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -66,6 +66,11 @@ public: return 0; } + int bayer2RGBA(const T*, int, T*, int, int) const + { + return 0; + } + int bayer2RGB_EA(const T*, int, T*, int, int) const { return 0; @@ -218,6 +223,11 @@ public: return (int)(bayer - (bayer_end - width)); } + int bayer2RGBA(const uchar*, int, uchar*, int, int) const + { + return 0; + } + int bayer2RGB_EA(const uchar* bayer, int bayer_step, uchar* dst, int width, int blue) const { if (!use_simd) @@ -323,6 +333,165 @@ public: bool use_simd; }; +#elif CV_NEON +class SIMDBayerInterpolator_8u +{ +public: + SIMDBayerInterpolator_8u() + { + } + + int bayer2Gray(const uchar* bayer, int bayer_step, uchar* dst, + int width, int bcoeff, int gcoeff, int rcoeff) const + { + /* + B G B G | B G B G | B G B G | B G B G + G R G R | G R G R | G R G R | G R G R + B G B G | B G B G | B G B G | B G B G + */ + + uint16x8_t masklo = vdupq_n_u16(255); + const uchar* bayer_end = bayer + width; + + for( ; bayer <= bayer_end - 18; bayer += 14, dst += 14 ) + { + uint16x8_t r0 = vld1q_u16((const ushort*)bayer); + uint16x8_t r1 = vld1q_u16((const ushort*)(bayer + bayer_step)); + uint16x8_t r2 = vld1q_u16((const ushort*)(bayer + bayer_step*2)); + + uint16x8_t b1_ = vaddq_u16(vandq_u16(r0, masklo), vandq_u16(r2, masklo)); + uint16x8_t b1 = vextq_u16(b1_, b1_, 1); + uint16x8_t b0 = vaddq_u16(b1_, b1); + // b0 = b0 b2 b4 ... + // b1 = b1 b3 b5 ... + + uint16x8_t g0 = vaddq_u16(vshrq_n_u16(r0, 8), vshrq_n_u16(r2, 8)); + uint16x8_t g1 = vandq_u16(r1, masklo); + g0 = vaddq_u16(g0, vaddq_u16(g1, vextq_u16(g1, g1, 1))); + g1 = vshlq_n_u16(vextq_u16(g1, g1, 1), 2); + // g0 = b0 b2 b4 ... + // g1 = b1 b3 b5 ... + + r0 = vshrq_n_u16(r1, 8); + r1 = vaddq_u16(r0, vextq_u16(r0, r0, 1)); + r0 = vshlq_n_u16(r0, 2); + // r0 = r0 r2 r4 ... + // r1 = r1 r3 r5 ... + + b0 = vreinterpretq_u16_s16(vqdmulhq_n_s16(vreinterpretq_s16_u16(b0), (short)(rcoeff*2))); + b1 = vreinterpretq_u16_s16(vqdmulhq_n_s16(vreinterpretq_s16_u16(b1), (short)(rcoeff*4))); + + g0 = vreinterpretq_u16_s16(vqdmulhq_n_s16(vreinterpretq_s16_u16(g0), (short)(gcoeff*2))); + g1 = vreinterpretq_u16_s16(vqdmulhq_n_s16(vreinterpretq_s16_u16(g1), (short)(gcoeff*2))); + + r0 = vreinterpretq_u16_s16(vqdmulhq_n_s16(vreinterpretq_s16_u16(r0), (short)(bcoeff*2))); + r1 = vreinterpretq_u16_s16(vqdmulhq_n_s16(vreinterpretq_s16_u16(r1), (short)(bcoeff*4))); + + g0 = vaddq_u16(vaddq_u16(g0, b0), r0); + g1 = vaddq_u16(vaddq_u16(g1, b1), r1); + + uint8x8x2_t p = vzip_u8(vrshrn_n_u16(g0, 2), vrshrn_n_u16(g1, 2)); + vst1_u8(dst, p.val[0]); + vst1_u8(dst + 8, p.val[1]); + } + + return (int)(bayer - (bayer_end - width)); + } + + int bayer2RGB(const uchar* bayer, int bayer_step, uchar* dst, int width, int blue) const + { + /* + B G B G | B G B G | B G B G | B G B G + G R G R | G R G R | G R G R | G R G R + B G B G | B G B G | B G B G | B G B G + */ + uint16x8_t masklo = vdupq_n_u16(255); + uint8x16x3_t pix; + const uchar* bayer_end = bayer + width; + + for( ; bayer <= bayer_end - 18; bayer += 14, dst += 42 ) + { + uint16x8_t r0 = vld1q_u16((const ushort*)bayer); + uint16x8_t r1 = vld1q_u16((const ushort*)(bayer + bayer_step)); + uint16x8_t r2 = vld1q_u16((const ushort*)(bayer + bayer_step*2)); + + uint16x8_t b1 = vaddq_u16(vandq_u16(r0, masklo), vandq_u16(r2, masklo)); + uint16x8_t nextb1 = vextq_u16(b1, b1, 1); + uint16x8_t b0 = vaddq_u16(b1, nextb1); + // b0 b1 b2 ... + uint8x8x2_t bb = vzip_u8(vrshrn_n_u16(b0, 2), vrshrn_n_u16(nextb1, 1)); + pix.val[1-blue] = vcombine_u8(bb.val[0], bb.val[1]); + + uint16x8_t g0 = vaddq_u16(vshrq_n_u16(r0, 8), vshrq_n_u16(r2, 8)); + uint16x8_t g1 = vandq_u16(r1, masklo); + g0 = vaddq_u16(g0, vaddq_u16(g1, vextq_u16(g1, g1, 1))); + g1 = vextq_u16(g1, g1, 1); + // g0 g1 g2 ... + uint8x8x2_t gg = vzip_u8(vrshrn_n_u16(g0, 2), vmovn_u16(g1)); + pix.val[1] = vcombine_u8(gg.val[0], gg.val[1]); + + r0 = vshrq_n_u16(r1, 8); + r1 = vaddq_u16(r0, vextq_u16(r0, r0, 1)); + // r0 r1 r2 ... + uint8x8x2_t rr = vzip_u8(vmovn_u16(r0), vrshrn_n_u16(r1, 1)); + pix.val[1+blue] = vcombine_u8(rr.val[0], rr.val[1]); + + vst3q_u8(dst-1, pix); + } + + return (int)(bayer - (bayer_end - width)); + } + + int bayer2RGBA(const uchar* bayer, int bayer_step, uchar* dst, int width, int blue) const + { + /* + B G B G | B G B G | B G B G | B G B G + G R G R | G R G R | G R G R | G R G R + B G B G | B G B G | B G B G | B G B G + */ + uint16x8_t masklo = vdupq_n_u16(255); + uint8x16x4_t pix; + const uchar* bayer_end = bayer + width; + pix.val[3] = vdupq_n_u8(255); + + for( ; bayer <= bayer_end - 18; bayer += 14, dst += 56 ) + { + uint16x8_t r0 = vld1q_u16((const ushort*)bayer); + uint16x8_t r1 = vld1q_u16((const ushort*)(bayer + bayer_step)); + uint16x8_t r2 = vld1q_u16((const ushort*)(bayer + bayer_step*2)); + + uint16x8_t b1 = vaddq_u16(vandq_u16(r0, masklo), vandq_u16(r2, masklo)); + uint16x8_t nextb1 = vextq_u16(b1, b1, 1); + uint16x8_t b0 = vaddq_u16(b1, nextb1); + // b0 b1 b2 ... + uint8x8x2_t bb = vzip_u8(vrshrn_n_u16(b0, 2), vrshrn_n_u16(nextb1, 1)); + pix.val[1-blue] = vcombine_u8(bb.val[0], bb.val[1]); + + uint16x8_t g0 = vaddq_u16(vshrq_n_u16(r0, 8), vshrq_n_u16(r2, 8)); + uint16x8_t g1 = vandq_u16(r1, masklo); + g0 = vaddq_u16(g0, vaddq_u16(g1, vextq_u16(g1, g1, 1))); + g1 = vextq_u16(g1, g1, 1); + // g0 g1 g2 ... + uint8x8x2_t gg = vzip_u8(vrshrn_n_u16(g0, 2), vmovn_u16(g1)); + pix.val[1] = vcombine_u8(gg.val[0], gg.val[1]); + + r0 = vshrq_n_u16(r1, 8); + r1 = vaddq_u16(r0, vextq_u16(r0, r0, 1)); + // r0 r1 r2 ... + uint8x8x2_t rr = vzip_u8(vmovn_u16(r0), vrshrn_n_u16(r1, 1)); + pix.val[1+blue] = vcombine_u8(rr.val[0], rr.val[1]); + + vst4q_u8(dst-1, pix); + } + + return (int)(bayer - (bayer_end - width)); + } + + int bayer2RGB_EA(const uchar*, int, uchar*, int, int) const + { + return 0; + } +}; #else typedef SIMDBayerStubInterpolator_ SIMDBayerInterpolator_8u; #endif @@ -559,7 +728,9 @@ public: } // simd optimization only for dcn == 3 - int delta = dcn == 4 ? 0 : vecOp.bayer2RGB(bayer, bayer_step, dst, size.width, blue); + int delta = dcn == 4 ? + vecOp.bayer2RGBA(bayer, bayer_step, dst, size.width, blue) : + vecOp.bayer2RGB(bayer, bayer_step, dst, size.width, blue); bayer += delta; dst += delta*dcn; diff --git a/modules/imgproc/src/morph.cpp b/modules/imgproc/src/morph.cpp index 4f696b4209d342e28ba7e7b373954945d3aa99aa..520d26d6c782d407e644d67d6d1c5145dc920bee 100644 --- a/modules/imgproc/src/morph.cpp +++ b/modules/imgproc/src/morph.cpp @@ -1221,7 +1221,7 @@ static bool IPPMorphReplicate(int op, const Mat &src, Mat &dst, const Mat &kerne IPP_MORPH_CASE(CV_32FC3, 32f_C3R, 32f); IPP_MORPH_CASE(CV_32FC4, 32f_C4R, 32f); default: - return false; + ; } #undef IPP_MORPH_CASE @@ -1253,14 +1253,11 @@ static bool IPPMorphReplicate(int op, const Mat &src, Mat &dst, const Mat &kerne IPP_MORPH_CASE(CV_32FC3, 32f_C3R, 32f); IPP_MORPH_CASE(CV_32FC4, 32f_C4R, 32f); default: - return false; + ; } #undef IPP_MORPH_CASE - -#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ >= 8 - return false; /// It disables false positive warning in GCC 4.8 and further -#endif } + return false; } static bool IPPMorphOp(int op, InputArray _src, OutputArray _dst, @@ -1339,22 +1336,190 @@ static bool IPPMorphOp(int op, InputArray _src, OutputArray _dst, #ifdef HAVE_OPENCL +#define ROUNDUP(sz, n) ((sz) + (n) - 1 - (((sz) + (n) - 1) % (n))) + +static bool ocl_morphSmall( InputArray _src, OutputArray _dst, InputArray _kernel, Point anchor, int borderType, + int op, int actual_op = -1, InputArray _extraMat = noArray()) +{ + const ocl::Device & dev = ocl::Device::getDefault(); + int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type), esz = CV_ELEM_SIZE(type); + bool doubleSupport = dev.doubleFPConfig() > 0; + + if (cn > 4 || (!doubleSupport && depth == CV_64F) || + _src.offset() % esz != 0 || _src.step() % esz != 0) + return false; + + bool haveExtraMat = !_extraMat.empty(); + CV_Assert(actual_op <= 3 || haveExtraMat); + + Size ksize = _kernel.size(); + if (anchor.x < 0) + anchor.x = ksize.width / 2; + if (anchor.y < 0) + anchor.y = ksize.height / 2; + + Size size = _src.size(), wholeSize; + bool isolated = (borderType & BORDER_ISOLATED) != 0; + borderType &= ~BORDER_ISOLATED; + int wdepth = depth, wtype = type; + if (depth == CV_8U) + { + wdepth = CV_32S; + wtype = CV_MAKETYPE(wdepth, cn); + } + char cvt[2][40]; + + const char * const borderMap[] = { "BORDER_CONSTANT", "BORDER_REPLICATE", + "BORDER_REFLECT", 0, "BORDER_REFLECT_101" }; + size_t globalsize[2] = { size.width, size.height }; + + UMat src = _src.getUMat(); + if (!isolated) + { + Point ofs; + src.locateROI(wholeSize, ofs); + } + + int h = isolated ? size.height : wholeSize.height; + int w = isolated ? size.width : wholeSize.width; + if (w < ksize.width || h < ksize.height) + return false; + + // Figure out what vector size to use for loading the pixels. + int pxLoadNumPixels = cn != 1 || size.width % 4 ? 1 : 4; + int pxLoadVecSize = cn * pxLoadNumPixels; + + // Figure out how many pixels per work item to compute in X and Y + // directions. Too many and we run out of registers. + int pxPerWorkItemX = 1, pxPerWorkItemY = 1; + if (cn <= 2 && ksize.width <= 4 && ksize.height <= 4) + { + pxPerWorkItemX = size.width % 8 ? size.width % 4 ? size.width % 2 ? 1 : 2 : 4 : 8; + pxPerWorkItemY = size.height % 2 ? 1 : 2; + } + else if (cn < 4 || (ksize.width <= 4 && ksize.height <= 4)) + { + pxPerWorkItemX = size.width % 2 ? 1 : 2; + pxPerWorkItemY = size.height % 2 ? 1 : 2; + } + globalsize[0] = size.width / pxPerWorkItemX; + globalsize[1] = size.height / pxPerWorkItemY; + + // Need some padding in the private array for pixels + int privDataWidth = ROUNDUP(pxPerWorkItemX + ksize.width - 1, pxLoadNumPixels); + + // Make the global size a nice round number so the runtime can pick + // from reasonable choices for the workgroup size + const int wgRound = 256; + globalsize[0] = ROUNDUP(globalsize[0], wgRound); + + if (actual_op < 0) + actual_op = op; + + // build processing + String processing; + Mat kernel8u; + _kernel.getMat().convertTo(kernel8u, CV_8U); + for (int y = 0; y < kernel8u.rows; ++y) + for (int x = 0; x < kernel8u.cols; ++x) + if (kernel8u.at(y, x) != 0) + processing += format("PROCESS(%d,%d)", y, x); + + + static const char * const op2str[] = { "OP_ERODE", "OP_DILATE", NULL, NULL, "OP_GRADIENT", "OP_TOPHAT", "OP_BLACKHAT" }; + String opts = format("-D cn=%d " + "-D ANCHOR_X=%d -D ANCHOR_Y=%d -D KERNEL_SIZE_X=%d -D KERNEL_SIZE_Y=%d " + "-D PX_LOAD_VEC_SIZE=%d -D PX_LOAD_NUM_PX=%d -D DEPTH_%d " + "-D PX_PER_WI_X=%d -D PX_PER_WI_Y=%d -D PRIV_DATA_WIDTH=%d -D %s -D %s " + "-D PX_LOAD_X_ITERATIONS=%d -D PX_LOAD_Y_ITERATIONS=%d " + "-D srcT=%s -D srcT1=%s -D dstT=srcT -D dstT1=srcT1 -D WT=%s -D WT1=%s " + "-D convertToWT=%s -D convertToDstT=%s -D PROCESS_ELEM_=%s -D %s%s", + cn, anchor.x, anchor.y, ksize.width, ksize.height, + pxLoadVecSize, pxLoadNumPixels, depth, + pxPerWorkItemX, pxPerWorkItemY, privDataWidth, borderMap[borderType], + isolated ? "BORDER_ISOLATED" : "NO_BORDER_ISOLATED", + privDataWidth / pxLoadNumPixels, pxPerWorkItemY + ksize.height - 1, + ocl::typeToStr(type), ocl::typeToStr(depth), + haveExtraMat ? ocl::typeToStr(wtype):"srcT",//to prevent overflow - WT + haveExtraMat ? ocl::typeToStr(wdepth):"srcT1",//to prevent overflow - WT1 + haveExtraMat ? ocl::convertTypeStr(depth, wdepth, cn, cvt[0]) : "noconvert",//to prevent overflow - src to WT + haveExtraMat ? ocl::convertTypeStr(wdepth, depth, cn, cvt[1]) : "noconvert",//to prevent overflow - WT to dst + processing.c_str(), op2str[op], + actual_op == op ? "" : cv::format(" -D %s", op2str[actual_op]).c_str()); + + ocl::Kernel kernel("filterSmall", cv::ocl::imgproc::filterSmall_oclsrc, opts); + if (kernel.empty()) + return false; + + _dst.create(size, type); + UMat dst = _dst.getUMat(); + + UMat source; + if(src.u != dst.u) + source = src; + else + { + Point ofs; + int cols = src.cols, rows = src.rows; + src.locateROI(wholeSize, ofs); + src.adjustROI(ofs.y, wholeSize.height - rows - ofs.y, ofs.x, wholeSize.width - cols - ofs.x); + src.copyTo(source); + + src.adjustROI(-ofs.y, -wholeSize.height + rows + ofs.y, -ofs.x, -wholeSize.width + cols + ofs.x); + source.adjustROI(-ofs.y, -wholeSize.height + rows + ofs.y, -ofs.x, -wholeSize.width + cols + ofs.x); + source.locateROI(wholeSize, ofs); + } + + UMat extraMat = _extraMat.getUMat(); + + int idxArg = kernel.set(0, ocl::KernelArg::PtrReadOnly(source)); + idxArg = kernel.set(idxArg, (int)source.step); + int srcOffsetX = (int)((source.offset % source.step) / source.elemSize()); + int srcOffsetY = (int)(source.offset / source.step); + int srcEndX = isolated ? srcOffsetX + size.width : wholeSize.width; + int srcEndY = isolated ? srcOffsetY + size.height : wholeSize.height; + idxArg = kernel.set(idxArg, srcOffsetX); + idxArg = kernel.set(idxArg, srcOffsetY); + idxArg = kernel.set(idxArg, srcEndX); + idxArg = kernel.set(idxArg, srcEndY); + idxArg = kernel.set(idxArg, ocl::KernelArg::WriteOnly(dst)); + + if (haveExtraMat) + { + idxArg = kernel.set(idxArg, ocl::KernelArg::ReadOnlyNoSize(extraMat)); + } + + return kernel.run(2, globalsize, NULL, false); + +} + static bool ocl_morphOp(InputArray _src, OutputArray _dst, InputArray _kernel, Point anchor, int iterations, int op, int borderType, const Scalar &, int actual_op = -1, InputArray _extraMat = noArray()) { const ocl::Device & dev = ocl::Device::getDefault(); - int type = _src.type(), depth = CV_MAT_DEPTH(type), cn = CV_MAT_CN(type); - bool doubleSupport = dev.doubleFPConfig() > 0; + int type = _src.type(), depth = CV_MAT_DEPTH(type), + cn = CV_MAT_CN(type), esz = CV_ELEM_SIZE(type); + Mat kernel = _kernel.getMat(); + Size ksize = kernel.data ? kernel.size() : Size(3, 3), ssize = _src.size(); + bool doubleSupport = dev.doubleFPConfig() > 0; if ((depth == CV_64F && !doubleSupport) || borderType != BORDER_CONSTANT) return false; - Mat kernel = _kernel.getMat(); bool haveExtraMat = !_extraMat.empty(); - Size ksize = kernel.data ? kernel.size() : Size(3, 3), ssize = _src.size(); CV_Assert(actual_op <= 3 || haveExtraMat); + // try to use OpenCL kernel adopted for small morph kernel + if (dev.isIntel() && !(dev.type() & ocl::Device::TYPE_CPU) && + ((ksize.width < 5 && ksize.height < 5 && esz <= 4) || + (ksize.width == 5 && ksize.height == 5 && cn == 1)) && + (iterations == 1)) + { + if (ocl_morphSmall(_src, _dst, _kernel, anchor, borderType, op, actual_op, _extraMat)) + return true; + } + if (iterations == 0 || kernel.rows*kernel.cols == 1) { _src.copyTo(_dst); diff --git a/modules/imgproc/src/opencl/cvtcolor.cl b/modules/imgproc/src/opencl/cvtcolor.cl index da835e08f3f997fe9a06b3cdf8d63f08f3f1baa6..2846357d52d3a07b51bfab057c2c8cf6be0625e1 100644 --- a/modules/imgproc/src/opencl/cvtcolor.cl +++ b/modules/imgproc/src/opencl/cvtcolor.cl @@ -441,18 +441,18 @@ __kernel void YCrCb2RGB(__global const uchar* src, int src_step, int src_offset, __global DATA_TYPE * dstptr = (__global DATA_TYPE*)(dst + dst_index); DATA_TYPE_4 src_pix = vload4(0, srcptr); - DATA_TYPE y = src_pix.x, cr = src_pix.y, cb = src_pix.z; + DATA_TYPE yp = src_pix.x, cr = src_pix.y, cb = src_pix.z; #ifdef DEPTH_5 __constant float * coeff = c_YCrCb2RGBCoeffs_f; - float r = fma(coeff[0], cr - HALF_MAX, y); - float g = fma(coeff[1], cr - HALF_MAX, fma(coeff[2], cb - HALF_MAX, y)); - float b = fma(coeff[3], cb - HALF_MAX, y); + float r = fma(coeff[0], cr - HALF_MAX, yp); + float g = fma(coeff[1], cr - HALF_MAX, fma(coeff[2], cb - HALF_MAX, yp)); + float b = fma(coeff[3], cb - HALF_MAX, yp); #else __constant int * coeff = c_YCrCb2RGBCoeffs_i; - int r = y + CV_DESCALE(coeff[0] * (cr - HALF_MAX), yuv_shift); - int g = y + CV_DESCALE(mad24(coeff[1], cr - HALF_MAX, coeff[2] * (cb - HALF_MAX)), yuv_shift); - int b = y + CV_DESCALE(coeff[3] * (cb - HALF_MAX), yuv_shift); + int r = yp + CV_DESCALE(coeff[0] * (cr - HALF_MAX), yuv_shift); + int g = yp + CV_DESCALE(mad24(coeff[1], cr - HALF_MAX, coeff[2] * (cb - HALF_MAX)), yuv_shift); + int b = yp + CV_DESCALE(coeff[3] * (cb - HALF_MAX), yuv_shift); #endif dstptr[(bidx^2)] = SAT_CAST(r); @@ -1796,6 +1796,10 @@ __kernel void Luv2BGR(__global const uchar * srcptr, int src_step, int src_offse float G = fma(X, coeffs[3], fma(Y, coeffs[4], Z * coeffs[5])); float B = fma(X, coeffs[6], fma(Y, coeffs[7], Z * coeffs[8])); + R = clamp(R, 0.f, 1.f); + G = clamp(G, 0.f, 1.f); + B = clamp(B, 0.f, 1.f); + #ifdef SRGB R = splineInterpolate(R*GammaTabScale, gammaTab, GAMMA_TAB_SIZE); G = splineInterpolate(G*GammaTabScale, gammaTab, GAMMA_TAB_SIZE); @@ -1853,6 +1857,10 @@ __kernel void Luv2BGR(__global const uchar * src, int src_step, int src_offset, float G = fma(X, coeffs[3], fma(Y, coeffs[4], Z * coeffs[5])); float B = fma(X, coeffs[6], fma(Y, coeffs[7], Z * coeffs[8])); + R = clamp(R, 0.f, 1.f); + G = clamp(G, 0.f, 1.f); + B = clamp(B, 0.f, 1.f); + #ifdef SRGB R = splineInterpolate(R*GammaTabScale, gammaTab, GAMMA_TAB_SIZE); G = splineInterpolate(G*GammaTabScale, gammaTab, GAMMA_TAB_SIZE); diff --git a/modules/imgproc/src/opencl/boxFilterSmall.cl b/modules/imgproc/src/opencl/filterSmall.cl similarity index 71% rename from modules/imgproc/src/opencl/boxFilterSmall.cl rename to modules/imgproc/src/opencl/filterSmall.cl index ff47d18e4b9828b10af682b81bb3f6545af908c9..c996fb833ed50a4799d48bba172fec24af25e108 100755 --- a/modules/imgproc/src/opencl/boxFilterSmall.cl +++ b/modules/imgproc/src/opencl/filterSmall.cl @@ -153,35 +153,10 @@ inline bool isBorder(const struct RectCoords bounds, int2 coord, int numPixels) } #endif -inline WT getBorderPixel(const struct RectCoords bounds, int2 coord, - __global const uchar * srcptr, int srcstep) -{ -#ifdef BORDER_CONSTANT - return (WT)(0); -#else - int selected_col = coord.x; - int selected_row = coord.y; - - EXTRAPOLATE(selected_col, selected_row, - bounds.x1, bounds.y1, - bounds.x2, bounds.y2); - - __global const uchar* ptr = srcptr + mad24(selected_row, srcstep, selected_col * SRCSIZE); - return convertToWT(loadpix(ptr)); -#endif -} - -inline WT readSrcPixelSingle(int2 pos, __global const uchar * srcptr, - int srcstep, const struct RectCoords srcCoords) -{ - if (!isBorder(srcCoords, pos, 1)) - { - __global const uchar * ptr = srcptr + mad24(pos.y, srcstep, pos.x * SRCSIZE); - return convertToWT(loadpix(ptr)); - } - else - return getBorderPixel(srcCoords, pos, srcptr, srcstep); -} +#define float1 float +#define uchar1 uchar +#define int1 int +#define uint1 unit #define __CAT(x, y) x##y #define CAT(x, y) __CAT(x, y) @@ -191,7 +166,7 @@ inline WT readSrcPixelSingle(int2 pos, __global const uchar * srcptr, #define PX_LOAD_FLOAT_VEC_TYPE CAT(WT1, PX_LOAD_VEC_SIZE) #define PX_LOAD_FLOAT_VEC_CONV CAT(convert_, PX_LOAD_FLOAT_VEC_TYPE) #define PX_LOAD CAT(vload, PX_LOAD_VEC_SIZE) -#define float1 float + inline PX_LOAD_FLOAT_VEC_TYPE readSrcPixelGroup(int2 pos, __global const uchar * srcptr, int srcstep, const struct RectCoords srcCoords) @@ -218,12 +193,150 @@ inline PX_LOAD_FLOAT_VEC_TYPE readSrcPixelGroup(int2 pos, __global const uchar * #define LOOP(N, VAR, STMT) CAT(LOOP, N)((VAR), (STMT)) -__kernel void boxFilterSmall(__global const uchar * srcptr, int src_step, int srcOffsetX, int srcOffsetY, int srcEndX, int srcEndY, - __global uchar * dstptr, int dst_step, int dst_offset, int rows, int cols +#ifdef OP_BOX_FILTER +#define PROCESS_ELEM \ + WT total_sum = (WT)(0); \ + int sy = 0; \ + LOOP(KERNEL_SIZE_Y, sy, \ + { \ + int sx = 0; \ + LOOP(KERNEL_SIZE_X, sx, \ + { \ + total_sum += privateData[py + sy][px + sx]; \ + }); \ + }) + +#elif defined OP_FILTER2D + +#define DIG(a) a, +__constant WT1 kernelData[] = { COEFF }; + +#define PROCESS_ELEM \ + WT total_sum = 0; \ + int sy = 0; \ + int kernelIndex = 0; \ + LOOP(KERNEL_SIZE_Y, sy, \ + { \ + int sx = 0; \ + LOOP(KERNEL_SIZE_X, sx, \ + { \ + total_sum = fma(kernelData[kernelIndex++], privateData[py + sy][px + sx], total_sum); \ + }); \ + }) + +#elif defined OP_ERODE || defined OP_DILATE + +#ifdef DEPTH_0 +#define MIN_VAL 0 +#define MAX_VAL UCHAR_MAX +#elif defined DEPTH_1 +#define MIN_VAL SCHAR_MIN +#define MAX_VAL SCHAR_MAX +#elif defined DEPTH_2 +#define MIN_VAL 0 +#define MAX_VAL USHRT_MAX +#elif defined DEPTH_3 +#define MIN_VAL SHRT_MIN +#define MAX_VAL SHRT_MAX +#elif defined DEPTH_4 +#define MIN_VAL INT_MIN +#define MAX_VAL INT_MAX +#elif defined DEPTH_5 +#define MIN_VAL (-FLT_MAX) +#define MAX_VAL FLT_MAX +#elif defined DEPTH_6 +#define MIN_VAL (-DBL_MAX) +#define MAX_VAL DBL_MAX +#endif + +#ifdef OP_ERODE +#define VAL (WT)MAX_VAL +#elif defined OP_DILATE +#define VAL (WT)MIN_VAL +#else +#error "Unknown operation" +#endif + +#define convert_float1 convert_float +#define convert_uchar1 convert_uchar +#define convert_int1 convert_int +#define convert_uint1 convert_uint + +#ifdef OP_ERODE +#if defined INTEL_DEVICE && defined DEPTH_0 +// workaround for bug in Intel HD graphics drivers (10.18.10.3496 or older) +#define WA_CONVERT_1 CAT(convert_uint, cn) +#define WA_CONVERT_2 CAT(convert_, srcT) +#define MORPH_OP(A, B) WA_CONVERT_2(min(WA_CONVERT_1(A), WA_CONVERT_1(B))) +#else +#define MORPH_OP(A, B) min((A), (B)) +#endif +#endif +#ifdef OP_DILATE +#define MORPH_OP(A, B) max((A), (B)) +#endif + +#define PROCESS(_y, _x) \ + total_sum = convertToWT(MORPH_OP(convertToWT(total_sum), convertToWT(privateData[py + _y][px + _x]))); + +#define PROCESS_ELEM \ + WT total_sum = convertToWT(VAL); \ + PROCESS_ELEM_ + +#else +#error "No processing is specified" +#endif + +#if defined OP_GRADIENT || defined OP_TOPHAT || defined OP_BLACKHAT +#define EXTRA_PARAMS , __global const uchar * matptr, int mat_step, int mat_offset +#else +#define EXTRA_PARAMS +#endif + +inline WT getBorderPixel(const struct RectCoords bounds, int2 coord, + __global const uchar * srcptr, int srcstep) +{ +#ifdef BORDER_CONSTANT +#ifdef OP_ERODE + return (WT)(MAX_VAL); +#elif defined OP_DILATE + return (WT)(MIN_VAL); +#else + return (WT)(0); +#endif +#else + + int selected_col = coord.x; + int selected_row = coord.y; + + EXTRAPOLATE(selected_col, selected_row, + bounds.x1, bounds.y1, + bounds.x2, bounds.y2); + + __global const uchar* ptr = srcptr + mad24(selected_row, srcstep, selected_col * SRCSIZE); + return convertToWT(loadpix(ptr)); +#endif +} + +inline WT readSrcPixelSingle(int2 pos, __global const uchar * srcptr, + int srcstep, const struct RectCoords srcCoords) +{ + if (!isBorder(srcCoords, pos, 1)) + { + __global const uchar * ptr = srcptr + mad24(pos.y, srcstep, pos.x * SRCSIZE); + return convertToWT(loadpix(ptr)); + } + else + return getBorderPixel(srcCoords, pos, srcptr, srcstep); +} + + +__kernel void filterSmall(__global const uchar * srcptr, int src_step, int srcOffsetX, int srcOffsetY, int srcEndX, int srcEndY, + __global uchar * dstptr, int dst_step, int dst_offset, int rows, int cols #ifdef NORMALIZE - , float alpha + , float alpha #endif - ) + EXTRA_PARAMS ) { // for non-isolated border: offsetX, offsetY, wholeX, wholeY const struct RectCoords srcCoords = { srcOffsetX, srcOffsetY, srcEndX, srcEndY }; @@ -282,24 +395,27 @@ __kernel void boxFilterSmall(__global const uchar * srcptr, int src_step, int sr LOOP(PX_PER_WI_X, px, { int x = startX + px; - int sy = 0; - int kernelIndex = 0; - WT total_sum = (WT)(0); - - LOOP(KERNEL_SIZE_Y, sy, - { - int sx = 0; - LOOP(KERNEL_SIZE_X, sx, - { - total_sum += privateData[py + sy][px + sx]; - }); - }); - - __global dstT * dstPtr = (__global dstT *)(dstptr + mad24(y, dst_step, mad24(x, DSTSIZE, dst_offset))); + PROCESS_ELEM; + int dst_index = mad24(y, dst_step, mad24(x, DSTSIZE, dst_offset)); + __global dstT * dstPtr = (__global dstT *)(dstptr + dst_index); #ifdef NORMALIZE total_sum *= (WT)(alpha); #endif +#if defined OP_GRADIENT || defined OP_TOPHAT || defined OP_BLACKHAT + //for this type of operations SRCSIZE == DSTSIZE + int mat_index = mad24(y, mat_step, mad24(x, SRCSIZE, mat_offset)); + WT value = convertToWT(loadpix(matptr + mat_index)); + +#ifdef OP_GRADIENT + storepix(convertToDstT(convertToWT(total_sum) - convertToWT(value)), dstPtr ); +#elif defined OP_TOPHAT + storepix(convertToDstT(convertToWT(value) - convertToWT(total_sum)), dstPtr ); +#elif defined OP_BLACKHAT + storepix(convertToDstT(convertToWT(total_sum) - convertToWT(value)), dstPtr ); +#endif +#else // erode or dilate, or open-close storepix(convertToDstT(total_sum), dstPtr); +#endif }); }); } diff --git a/modules/imgproc/src/opencl/integral_sum.cl b/modules/imgproc/src/opencl/integral_sum.cl index 49a3bde95548756f1148360a5b57d1dc7027e78c..3c51c1a28baf2b0fe8f195af64d0b83f8ccb91ec 100644 --- a/modules/imgproc/src/opencl/integral_sum.cl +++ b/modules/imgproc/src/opencl/integral_sum.cl @@ -132,8 +132,11 @@ kernel void integral_sum_rows(__global const uchar *buf_ptr, int buf_step, int b } dst_sq_offset += dst_sq_step; - dst_sq = (__global sumSQT *)(dst_sq_ptr + mad24(x, dst_sq_step, dst_sq_offset)); - dst_sq[0] = 0; + if (x < rows - 1) + { + dst_sq = (__global sumSQT *)(dst_sq_ptr + mad24(x, dst_sq_step, dst_sq_offset)); + dst_sq[0] = 0; + } int buf_sq_index = mad24((int)sizeof(sumSQT), x, buf_sq_offset); sumSQT accum_sq = 0; diff --git a/modules/imgproc/src/opencl/pyr_down.cl b/modules/imgproc/src/opencl/pyr_down.cl index 2358775e7a19d979f5040ad39aa6c21425cd12e0..4db1a8d8115d0e59cdaedae2be22f46870623a30 100644 --- a/modules/imgproc/src/opencl/pyr_down.cl +++ b/modules/imgproc/src/opencl/pyr_down.cl @@ -89,19 +89,56 @@ #define MAD(x,y,z) mad((x),(y),(z)) #endif +#define LOAD_LOCAL(col_gl, col_lcl) \ + sum0 = co3* SRC(col_gl, EXTRAPOLATE_(src_y - 2, src_rows)); \ + sum0 = MAD(co2, SRC(col_gl, EXTRAPOLATE_(src_y - 1, src_rows)), sum0); \ + temp = SRC(col_gl, EXTRAPOLATE_(src_y, src_rows)); \ + sum0 = MAD(co1, temp, sum0); \ + sum1 = co3 * temp; \ + temp = SRC(col_gl, EXTRAPOLATE_(src_y + 1, src_rows)); \ + sum0 = MAD(co2, temp, sum0); \ + sum1 = MAD(co2, temp, sum1); \ + temp = SRC(col_gl, EXTRAPOLATE_(src_y + 2, src_rows)); \ + sum0 = MAD(co3, temp, sum0); \ + sum1 = MAD(co1, temp, sum1); \ + smem[0][col_lcl] = sum0; \ + sum1 = MAD(co2, SRC(col_gl, EXTRAPOLATE_(src_y + 3, src_rows)), sum1); \ + sum1 = MAD(co3, SRC(col_gl, EXTRAPOLATE_(src_y + 4, src_rows)), sum1); \ + smem[1][col_lcl] = sum1; + + +#if kercn == 4 +#define LOAD_LOCAL4(col_gl, col_lcl) \ + sum40 = co3* SRC4(col_gl, EXTRAPOLATE_(src_y - 2, src_rows)); \ + sum40 = MAD(co2, SRC4(col_gl, EXTRAPOLATE_(src_y - 1, src_rows)), sum40); \ + temp4 = SRC4(col_gl, EXTRAPOLATE_(src_y, src_rows)); \ + sum40 = MAD(co1, temp4, sum40); \ + sum41 = co3 * temp4; \ + temp4 = SRC4(col_gl, EXTRAPOLATE_(src_y + 1, src_rows)); \ + sum40 = MAD(co2, temp4, sum40); \ + sum41 = MAD(co2, temp4, sum41); \ + temp4 = SRC4(col_gl, EXTRAPOLATE_(src_y + 2, src_rows)); \ + sum40 = MAD(co3, temp4, sum40); \ + sum41 = MAD(co1, temp4, sum41); \ + vstore4(sum40, col_lcl, (__local float*) &smem[0][2]); \ + sum41 = MAD(co2, SRC4(col_gl, EXTRAPOLATE_(src_y + 3, src_rows)), sum41); \ + sum41 = MAD(co3, SRC4(col_gl, EXTRAPOLATE_(src_y + 4, src_rows)), sum41); \ + vstore4(sum41, col_lcl, (__local float*) &smem[1][2]); +#endif + #define noconvert __kernel void pyrDown(__global const uchar * src, int src_step, int src_offset, int src_rows, int src_cols, __global uchar * dst, int dst_step, int dst_offset, int dst_rows, int dst_cols) { const int x = get_global_id(0)*kercn; - const int y = get_group_id(1); + const int y = 2*get_global_id(1); - __local FT smem[LOCAL_SIZE + 4]; + __local FT smem[2][LOCAL_SIZE + 4]; __global uchar * dstData = dst + dst_offset; __global const uchar * srcData = src + src_offset; - FT sum; + FT sum0, sum1, temp; FT co1 = 0.375f; FT co2 = 0.25f; FT co3 = 0.0625f; @@ -109,134 +146,68 @@ __kernel void pyrDown(__global const uchar * src, int src_step, int src_offset, const int src_y = 2*y; int col; - if (src_y >= 2 && src_y < src_rows - 2) + if (src_y >= 2 && src_y < src_rows - 4) { +#define EXTRAPOLATE_(val, maxVal) val #if kercn == 1 col = EXTRAPOLATE(x, src_cols); - - sum = co3* SRC(col, src_y - 2); - sum = MAD(co2, SRC(col, src_y - 1), sum); - sum = MAD(co1, SRC(col, src_y ), sum); - sum = MAD(co2, SRC(col, src_y + 1), sum); - sum = MAD(co3, SRC(col, src_y + 2), sum); - - smem[2 + get_local_id(0)] = sum; + LOAD_LOCAL(col, 2 + get_local_id(0)) #else if (x < src_cols-4) { - float4 sum4; - sum4 = co3* SRC4(x, src_y - 2); - sum4 = MAD(co2, SRC4(x, src_y - 1), sum4); - sum4 = MAD(co1, SRC4(x, src_y ), sum4); - sum4 = MAD(co2, SRC4(x, src_y + 1), sum4); - sum4 = MAD(co3, SRC4(x, src_y + 2), sum4); - - vstore4(sum4, get_local_id(0), (__local float*) &smem[2]); + float4 sum40, sum41, temp4; + LOAD_LOCAL4(x, get_local_id(0)) } else { for (int i=0; i<4; i++) { col = EXTRAPOLATE(x+i, src_cols); - sum = co3* SRC(col, src_y - 2); - sum = MAD(co2, SRC(col, src_y - 1), sum); - sum = MAD(co1, SRC(col, src_y ), sum); - sum = MAD(co2, SRC(col, src_y + 1), sum); - sum = MAD(co3, SRC(col, src_y + 2), sum); - - smem[2 + 4*get_local_id(0)+i] = sum; + LOAD_LOCAL(col, 2 + 4 * get_local_id(0) + i) } } #endif if (get_local_id(0) < 2) { col = EXTRAPOLATE((int)(get_group_id(0)*LOCAL_SIZE + get_local_id(0) - 2), src_cols); - - sum = co3* SRC(col, src_y - 2); - sum = MAD(co2, SRC(col, src_y - 1), sum); - sum = MAD(co1, SRC(col, src_y ), sum); - sum = MAD(co2, SRC(col, src_y + 1), sum); - sum = MAD(co3, SRC(col, src_y + 2), sum); - - smem[get_local_id(0)] = sum; + LOAD_LOCAL(col, get_local_id(0)) } - - if (get_local_id(0) > 1 && get_local_id(0) < 4) + else if (get_local_id(0) < 4) { col = EXTRAPOLATE((int)((get_group_id(0)+1)*LOCAL_SIZE + get_local_id(0) - 2), src_cols); - - sum = co3* SRC(col, src_y - 2); - sum = MAD(co2, SRC(col, src_y - 1), sum); - sum = MAD(co1, SRC(col, src_y ), sum); - sum = MAD(co2, SRC(col, src_y + 1), sum); - sum = MAD(co3, SRC(col, src_y + 2), sum); - - smem[LOCAL_SIZE + get_local_id(0)] = sum; + LOAD_LOCAL(col, LOCAL_SIZE + get_local_id(0)) } } else // need extrapolate y { +#define EXTRAPOLATE_(val, maxVal) EXTRAPOLATE(val, maxVal) #if kercn == 1 col = EXTRAPOLATE(x, src_cols); - - sum = co3* SRC(col, EXTRAPOLATE(src_y - 2, src_rows)); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y - 1, src_rows)), sum); - sum = MAD(co1, SRC(col, EXTRAPOLATE(src_y , src_rows)), sum); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y + 1, src_rows)), sum); - sum = MAD(co3, SRC(col, EXTRAPOLATE(src_y + 2, src_rows)), sum); - - smem[2 + get_local_id(0)] = sum; + LOAD_LOCAL(col, 2 + get_local_id(0)) #else if (x < src_cols-4) { - float4 sum4; - sum4 = co3* SRC4(x, EXTRAPOLATE(src_y - 2, src_rows)); - sum4 = MAD(co2, SRC4(x, EXTRAPOLATE(src_y - 1, src_rows)), sum4); - sum4 = MAD(co1, SRC4(x, EXTRAPOLATE(src_y , src_rows)), sum4); - sum4 = MAD(co2, SRC4(x, EXTRAPOLATE(src_y + 1, src_rows)), sum4); - sum4 = MAD(co3, SRC4(x, EXTRAPOLATE(src_y + 2, src_rows)), sum4); - - vstore4(sum4, get_local_id(0), (__local float*) &smem[2]); + float4 sum40, sum41, temp4; + LOAD_LOCAL4(x, get_local_id(0)) } else { for (int i=0; i<4; i++) { col = EXTRAPOLATE(x+i, src_cols); - sum = co3* SRC(col, EXTRAPOLATE(src_y - 2, src_rows)); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y - 1, src_rows)), sum); - sum = MAD(co1, SRC(col, EXTRAPOLATE(src_y , src_rows)), sum); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y + 1, src_rows)), sum); - sum = MAD(co3, SRC(col, EXTRAPOLATE(src_y + 2, src_rows)), sum); - - smem[2 + 4*get_local_id(0)+i] = sum; + LOAD_LOCAL(col, 2 + 4*get_local_id(0) + i) } } #endif if (get_local_id(0) < 2) { col = EXTRAPOLATE((int)(get_group_id(0)*LOCAL_SIZE + get_local_id(0) - 2), src_cols); - - sum = co3* SRC(col, EXTRAPOLATE(src_y - 2, src_rows)); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y - 1, src_rows)), sum); - sum = MAD(co1, SRC(col, EXTRAPOLATE(src_y , src_rows)), sum); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y + 1, src_rows)), sum); - sum = MAD(co3, SRC(col, EXTRAPOLATE(src_y + 2, src_rows)), sum); - - smem[get_local_id(0)] = sum; + LOAD_LOCAL(col, get_local_id(0)) } - - if (get_local_id(0) > 1 && get_local_id(0) < 4) + else if (get_local_id(0) < 4) { col = EXTRAPOLATE((int)((get_group_id(0)+1)*LOCAL_SIZE + get_local_id(0) - 2), src_cols); - - sum = co3* SRC(col, EXTRAPOLATE(src_y - 2, src_rows)); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y - 1, src_rows)), sum); - sum = MAD(co1, SRC(col, EXTRAPOLATE(src_y , src_rows)), sum); - sum = MAD(co2, SRC(col, EXTRAPOLATE(src_y + 1, src_rows)), sum); - sum = MAD(co3, SRC(col, EXTRAPOLATE(src_y + 2, src_rows)), sum); - - smem[LOCAL_SIZE + get_local_id(0)] = sum; + LOAD_LOCAL(col, LOCAL_SIZE + get_local_id(0)) } } @@ -247,50 +218,68 @@ __kernel void pyrDown(__global const uchar * src, int src_step, int src_offset, { const int tid2 = get_local_id(0) * 2; - sum = 0.f; + const int dst_x = (get_group_id(0) * get_local_size(0) + tid2) / 2; + + if (dst_x < dst_cols) + { + for (int yin = y, y1 = min(dst_rows, y + 2); yin < y1; yin++) + { #if cn == 1 #if fdepth <= 5 - sum = sum + dot(vload4(0, (__local float*) (&smem)+tid2), (float4)(co3, co2, co1, co2)); + FT sum = dot(vload4(0, (__local float*) (&smem) + tid2 + (yin - y) * (LOCAL_SIZE + 4)), (float4)(co3, co2, co1, co2)); #else - sum = sum + dot(vload4(0, (__local double*) (&smem)+tid2), (double4)(co3, co2, co1, co2)); + FT sum = dot(vload4(0, (__local double*) (&smem) + tid2 + (yin - y) * (LOCAL_SIZE + 4)), (double4)(co3, co2, co1, co2)); #endif #else - sum = MAD(co3, smem[2 + tid2 - 2], sum); - sum = MAD(co2, smem[2 + tid2 - 1], sum); - sum = MAD(co1, smem[2 + tid2 ], sum); - sum = MAD(co2, smem[2 + tid2 + 1], sum); + FT sum = co3 * smem[yin - y][2 + tid2 - 2]; + sum = MAD(co2, smem[yin - y][2 + tid2 - 1], sum); + sum = MAD(co1, smem[yin - y][2 + tid2 ], sum); + sum = MAD(co2, smem[yin - y][2 + tid2 + 1], sum); #endif - sum = MAD(co3, smem[2 + tid2 + 2], sum); - - const int dst_x = (get_group_id(0) * get_local_size(0) + tid2) / 2; - - if (dst_x < dst_cols) - storepix(convertToT(sum), dstData + y * dst_step + dst_x * PIXSIZE); + sum = MAD(co3, smem[yin - y][2 + tid2 + 2], sum); + storepix(convertToT(sum), dstData + yin * dst_step + dst_x * PIXSIZE); + } + } } #else int tid4 = get_local_id(0) * 4; - - sum = co3* smem[2 + tid4 + 2]; - sum = MAD(co3, smem[2 + tid4 - 2], sum); - sum = MAD(co2, smem[2 + tid4 - 1], sum); - sum = MAD(co1, smem[2 + tid4 ], sum); - sum = MAD(co2, smem[2 + tid4 + 1], sum); - int dst_x = (get_group_id(0) * LOCAL_SIZE + tid4) / 2; + if (dst_x < dst_cols - 1) + { + for (int yin = y, y1 = min(dst_rows, y + 2); yin < y1; yin++) + { - if (dst_x < dst_cols) - storepix(convertToT(sum), dstData + mad24(y, dst_step, dst_x * PIXSIZE)); - - tid4 += 2; - dst_x += 1; + FT sum = co3* smem[yin - y][2 + tid4 + 2]; + sum = MAD(co3, smem[yin - y][2 + tid4 - 2], sum); + sum = MAD(co2, smem[yin - y][2 + tid4 - 1], sum); + sum = MAD(co1, smem[yin - y][2 + tid4 ], sum); + sum = MAD(co2, smem[yin - y][2 + tid4 + 1], sum); + storepix(convertToT(sum), dstData + mad24(yin, dst_step, dst_x * PIXSIZE)); + + dst_x ++; + sum = co3* smem[yin - y][2 + tid4 + 4]; + sum = MAD(co3, smem[yin - y][2 + tid4 ], sum); + sum = MAD(co2, smem[yin - y][2 + tid4 + 1], sum); + sum = MAD(co1, smem[yin - y][2 + tid4 + 2], sum); + sum = MAD(co2, smem[yin - y][2 + tid4 + 3], sum); + storepix(convertToT(sum), dstData + mad24(yin, dst_step, dst_x * PIXSIZE)); + dst_x --; + } - sum = co3* smem[2 + tid4 + 2]; - sum = MAD(co3, smem[2 + tid4 - 2], sum); - sum = MAD(co2, smem[2 + tid4 - 1], sum); - sum = MAD(co1, smem[2 + tid4 ], sum); - sum = MAD(co2, smem[2 + tid4 + 1], sum); + } + else if (dst_x < dst_cols) + { + for (int yin = y, y1 = min(dst_rows, y + 2); yin < y1; yin++) + { + FT sum = co3* smem[yin - y][2 + tid4 + 2]; + sum = MAD(co3, smem[yin - y][2 + tid4 - 2], sum); + sum = MAD(co2, smem[yin - y][2 + tid4 - 1], sum); + sum = MAD(co1, smem[yin - y][2 + tid4 ], sum); + sum = MAD(co2, smem[yin - y][2 + tid4 + 1], sum); - if (dst_x < dst_cols) - storepix(convertToT(sum), dstData + mad24(y, dst_step, dst_x * PIXSIZE)); + storepix(convertToT(sum), dstData + mad24(yin, dst_step, dst_x * PIXSIZE)); + } + } #endif + } diff --git a/modules/imgproc/src/pyramids.cpp b/modules/imgproc/src/pyramids.cpp index cbbe39930185ee07b64eee13298c2a451390f763..2714e08f30c59cf825db7c54fb58c3f61d0a02c7 100644 --- a/modules/imgproc/src/pyramids.cpp +++ b/modules/imgproc/src/pyramids.cpp @@ -445,7 +445,7 @@ static bool ocl_pyrDown( InputArray _src, OutputArray _dst, const Size& _dsz, in k.args(ocl::KernelArg::ReadOnly(src), ocl::KernelArg::WriteOnly(dst)); size_t localThreads[2] = { local_size/kercn, 1 }; - size_t globalThreads[2] = { (src.cols + (kercn-1))/kercn, dst.rows }; + size_t globalThreads[2] = { (src.cols + (kercn-1))/kercn, (dst.rows + 1) / 2 }; return k.run(2, globalThreads, localThreads, false); } diff --git a/modules/imgproc/src/smooth.cpp b/modules/imgproc/src/smooth.cpp index 66ff429cf3a65f8002dde429755fdf766dcf5e9c..907a6591ba7e1d1722bc7d86f72518a379740e9c 100644 --- a/modules/imgproc/src/smooth.cpp +++ b/modules/imgproc/src/smooth.cpp @@ -720,7 +720,7 @@ static bool ocl_boxFilter( InputArray _src, OutputArray _dst, int ddepth, "-D PX_PER_WI_X=%d -D PX_PER_WI_Y=%d -D PRIV_DATA_WIDTH=%d -D %s -D %s " "-D PX_LOAD_X_ITERATIONS=%d -D PX_LOAD_Y_ITERATIONS=%d " "-D srcT=%s -D srcT1=%s -D dstT=%s -D dstT1=%s -D WT=%s -D WT1=%s " - "-D convertToWT=%s -D convertToDstT=%s%s%s", + "-D convertToWT=%s -D convertToDstT=%s%s%s -D OP_BOX_FILTER", cn, anchor.x, anchor.y, ksize.width, ksize.height, pxLoadVecSize, pxLoadNumPixels, pxPerWorkItemX, pxPerWorkItemY, privDataWidth, borderMap[borderType], @@ -734,7 +734,7 @@ static bool ocl_boxFilter( InputArray _src, OutputArray _dst, int ddepth, - if (!kernel.create("boxFilterSmall", cv::ocl::imgproc::boxFilterSmall_oclsrc, build_options)) + if (!kernel.create("filterSmall", cv::ocl::imgproc::filterSmall_oclsrc, build_options)) return false; } else diff --git a/modules/imgproc/test/ocl/test_blend.cpp b/modules/imgproc/test/ocl/test_blend.cpp index 7b62b97172afbe38858b81ac420ac70291236607..6d8a15fb2b9873f1f85a2cd233bcd225c1c6dd0f 100644 --- a/modules/imgproc/test/ocl/test_blend.cpp +++ b/modules/imgproc/test/ocl/test_blend.cpp @@ -117,7 +117,7 @@ OCL_TEST_P(BlendLinear, Accuracy) OCL_OFF(cv::blendLinear(src1_roi, src2_roi, weights1_roi, weights2_roi, dst_roi)); OCL_ON(cv::blendLinear(usrc1_roi, usrc2_roi, uweights1_roi, uweights2_roi, udst_roi)); - Near(depth <= CV_32S ? 1.0 : 0.2); + Near(depth <= CV_32S ? 1.0 : 0.5); } } diff --git a/modules/imgproc/test/ocl/test_boxfilter.cpp b/modules/imgproc/test/ocl/test_boxfilter.cpp index 63f4ebff207b434d335debd5e5fad6c1d4c76f29..4940dff79986ff014125d4791bd92c8edebd7ec7 100644 --- a/modules/imgproc/test/ocl/test_boxfilter.cpp +++ b/modules/imgproc/test/ocl/test_boxfilter.cpp @@ -109,7 +109,7 @@ OCL_TEST_P(BoxFilter, Mat) OCL_OFF(cv::boxFilter(src_roi, dst_roi, -1, ksize, anchor, normalize, borderType)); OCL_ON(cv::boxFilter(usrc_roi, udst_roi, -1, ksize, anchor, normalize, borderType)); - Near(depth <= CV_32S ? 1 : 1e-3); + Near(depth <= CV_32S ? 1 : 3e-3); } } diff --git a/modules/imgproc/test/ocl/test_color.cpp b/modules/imgproc/test/ocl/test_color.cpp index 82bf2c06f1d6d5f6a5ab283357db2706a16fb558..5f3a2f73f992c5f1fe1cc7e2c540169d6ffaf60e 100644 --- a/modules/imgproc/test/ocl/test_color.cpp +++ b/modules/imgproc/test/ocl/test_color.cpp @@ -302,14 +302,14 @@ OCL_TEST_P(CvtColor8u32f, Lab2LRGBA) { performTest(3, 4, CVTCODE(Lab2LRGB), dept // RGB -> Luv -OCL_TEST_P(CvtColor8u32f, BGR2Luv) { performTest(3, 3, CVTCODE(BGR2Luv), depth == CV_8U ? 1 : 1e-2); } -OCL_TEST_P(CvtColor8u32f, RGB2Luv) { performTest(3, 3, CVTCODE(RGB2Luv), depth == CV_8U ? 1 : 1e-2); } -OCL_TEST_P(CvtColor8u32f, LBGR2Luv) { performTest(3, 3, CVTCODE(LBGR2Luv), depth == CV_8U ? 1 : 4e-3); } -OCL_TEST_P(CvtColor8u32f, LRGB2Luv) { performTest(3, 3, CVTCODE(LRGB2Luv), depth == CV_8U ? 1 : 5e-3); } -OCL_TEST_P(CvtColor8u32f, BGRA2Luv) { performTest(4, 3, CVTCODE(BGR2Luv), depth == CV_8U ? 1 : 8e-3); } -OCL_TEST_P(CvtColor8u32f, RGBA2Luv) { performTest(4, 3, CVTCODE(RGB2Luv), depth == CV_8U ? 1 : 9e-3); } -OCL_TEST_P(CvtColor8u32f, LBGRA2Luv) { performTest(4, 3, CVTCODE(LBGR2Luv), depth == CV_8U ? 1 : 5e-3); } -OCL_TEST_P(CvtColor8u32f, LRGBA2Luv) { performTest(4, 3, CVTCODE(LRGB2Luv), depth == CV_8U ? 1 : 5e-3); } +OCL_TEST_P(CvtColor8u32f, BGR2Luv) { performTest(3, 3, CVTCODE(BGR2Luv), depth == CV_8U ? 1 : 1.5e-2); } +OCL_TEST_P(CvtColor8u32f, RGB2Luv) { performTest(3, 3, CVTCODE(RGB2Luv), depth == CV_8U ? 1 : 1.5e-2); } +OCL_TEST_P(CvtColor8u32f, LBGR2Luv) { performTest(3, 3, CVTCODE(LBGR2Luv), depth == CV_8U ? 1 : 6e-3); } +OCL_TEST_P(CvtColor8u32f, LRGB2Luv) { performTest(3, 3, CVTCODE(LRGB2Luv), depth == CV_8U ? 1 : 6e-3); } +OCL_TEST_P(CvtColor8u32f, BGRA2Luv) { performTest(4, 3, CVTCODE(BGR2Luv), depth == CV_8U ? 1 : 2e-2); } +OCL_TEST_P(CvtColor8u32f, RGBA2Luv) { performTest(4, 3, CVTCODE(RGB2Luv), depth == CV_8U ? 1 : 2e-2); } +OCL_TEST_P(CvtColor8u32f, LBGRA2Luv) { performTest(4, 3, CVTCODE(LBGR2Luv), depth == CV_8U ? 1 : 6e-3); } +OCL_TEST_P(CvtColor8u32f, LRGBA2Luv) { performTest(4, 3, CVTCODE(LRGB2Luv), depth == CV_8U ? 1 : 6e-3); } OCL_TEST_P(CvtColor8u32f, Luv2BGR) { performTest(3, 3, CVTCODE(Luv2BGR), depth == CV_8U ? 1 : 7e-5); } OCL_TEST_P(CvtColor8u32f, Luv2RGB) { performTest(3, 3, CVTCODE(Luv2RGB), depth == CV_8U ? 1 : 7e-5); } diff --git a/modules/imgproc/test/ocl/test_filters.cpp b/modules/imgproc/test/ocl/test_filters.cpp index 1fe29278866790deac8e4f587b7751b56b9e0c3c..61f38a6b8b30201a05914bccbc0628dc6b23c804 100644 --- a/modules/imgproc/test/ocl/test_filters.cpp +++ b/modules/imgproc/test/ocl/test_filters.cpp @@ -275,14 +275,68 @@ OCL_TEST_P(Dilate, Mat) ///////////////////////////////////////////////////////////////////////////////////////////////// // MorphologyEx +IMPLEMENT_PARAM_CLASS(MorphOp, int) +PARAM_TEST_CASE(MorphologyEx, MatType, + int, // kernel size + MorphOp, // MORPH_OP + int, // iterations + bool) +{ + int type, ksize, op, iterations; + bool useRoi; + + TEST_DECLARE_INPUT_PARAMETER(src); + TEST_DECLARE_OUTPUT_PARAMETER(dst); + + virtual void SetUp() + { + type = GET_PARAM(0); + ksize = GET_PARAM(1); + op = GET_PARAM(2); + iterations = GET_PARAM(3); + useRoi = GET_PARAM(4); + } + + void random_roi(int minSize = 1) + { + if (minSize == 0) + minSize = ksize; + + Size roiSize = randomSize(minSize, MAX_VALUE); + + Border srcBorder = randomBorder(0, useRoi ? MAX_VALUE : 0); + randomSubMat(src, src_roi, roiSize, srcBorder, type, 5, 256); + + Border dstBorder = randomBorder(0, useRoi ? MAX_VALUE : 0); + randomSubMat(dst, dst_roi, roiSize, dstBorder, type, -60, 70); + + UMAT_UPLOAD_INPUT_PARAMETER(src); + UMAT_UPLOAD_OUTPUT_PARAMETER(dst); + } + + void Near() + { + int depth = CV_MAT_DEPTH(type); + bool isFP = depth >= CV_32F; -typedef FilterTestBase MorphologyEx; + if (isFP) + Near(1e-6, true); + else + Near(1, false); + } + + void Near(double threshold, bool relative) + { + if (relative) + OCL_EXPECT_MATS_NEAR_RELATIVE(dst, threshold); + else + OCL_EXPECT_MATS_NEAR(dst, threshold); + } +}; OCL_TEST_P(MorphologyEx, Mat) { Size kernelSize(ksize, ksize); - int iterations = (int)param; - int op = size.height; for (int j = 0; j < test_loop_times; j++) { @@ -377,12 +431,10 @@ OCL_INSTANTIATE_TEST_CASE_P(Filter, Dilate, Combine( OCL_INSTANTIATE_TEST_CASE_P(Filter, MorphologyEx, Combine( Values(CV_8UC1, CV_8UC3, CV_8UC4, CV_32FC1, CV_32FC3, CV_32FC4), - Values(3, 5, 7), - Values(Size(0, 2), Size(0, 3), Size(0, 4), Size(0, 5), Size(0, 6)), // used as generator of operations - Values((BorderType)BORDER_CONSTANT), - Values(1.0, 2.0, 3.0), - Bool(), - Values(1))); // not used + Values(3, 5, 7), // kernel size + Values((MorphOp)MORPH_OPEN, (MorphOp)MORPH_CLOSE, (MorphOp)MORPH_GRADIENT, (MorphOp)MORPH_TOPHAT, (MorphOp)MORPH_BLACKHAT), // used as generator of operations + Values(1, 2, 3), + Bool())); } } // namespace cvtest::ocl diff --git a/modules/java/CMakeLists.txt b/modules/java/CMakeLists.txt index 1362aecad9b6ee6b7e3832cae12756c8a90e9016..7d1acb93bff4be45cdf64e9c28a5a15eef40967a 100644 --- a/modules/java/CMakeLists.txt +++ b/modules/java/CMakeLists.txt @@ -6,7 +6,7 @@ if(IOS OR NOT PYTHON_DEFAULT_AVAILABLE OR NOT ANT_EXECUTABLE OR NOT (JNI_FOUND O endif() set(the_description "The java bindings") -ocv_add_module(java BINDINGS opencv_core opencv_imgproc OPTIONAL opencv_objdetect opencv_features2d opencv_video opencv_imgcodecs opencv_videoio opencv_ml opencv_calib3d opencv_photo opencv_nonfree opencv_contrib) +ocv_add_module(java BINDINGS opencv_core opencv_imgproc OPTIONAL opencv_objdetect opencv_features2d opencv_video opencv_imgcodecs opencv_videoio opencv_calib3d opencv_photo opencv_nonfree opencv_contrib) ocv_module_include_directories("${CMAKE_CURRENT_SOURCE_DIR}/generator/src/cpp") if(NOT ANDROID) diff --git a/modules/java/generator/config/ml.filelist b/modules/java/generator/config/ml.filelist new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/java/generator/rst_parser.py b/modules/java/generator/rst_parser.py index 80b09ac40bd3a9c88d10e3a5aa7e8859194e2c44..78114aedb12d4af5fe66f8d6c88175c6851914c4 100755 --- a/modules/java/generator/rst_parser.py +++ b/modules/java/generator/rst_parser.py @@ -2,7 +2,7 @@ from __future__ import print_function import os, sys, re, string, fnmatch -allmodules = ["core", "flann", "imgproc", "ml", "imgcodecs", "videoio", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "cuda", "androidcamera", "java", "python", "stitching", "ts", "photo", "nonfree", "videostab", "softcascade", "superres"] +allmodules = ["core", "flann", "imgproc", "imgcodecs", "videoio", "highgui", "video", "features2d", "calib3d", "objdetect", "legacy", "contrib", "cuda", "androidcamera", "java", "python", "stitching", "ts", "photo", "nonfree", "videostab", "softcascade", "superres"] verbose = False show_warnings = True show_errors = True diff --git a/modules/java/generator/src/cpp/jni_part.cpp b/modules/java/generator/src/cpp/jni_part.cpp index ccd870cdf9c78a82f2755f0a1770719a3ca75ebc..a4ac0d553a64db30f5a9a5204952d07618be9b6c 100644 --- a/modules/java/generator/src/cpp/jni_part.cpp +++ b/modules/java/generator/src/cpp/jni_part.cpp @@ -14,10 +14,6 @@ # include "opencv2/video.hpp" #endif -#ifdef HAVE_OPENCV_ML -# include "opencv2/ml.hpp" -#endif - #ifdef HAVE_OPENCV_CONTRIB # include "opencv2/contrib.hpp" #endif @@ -41,10 +37,7 @@ JNI_OnLoad(JavaVM* vm, void* ) #ifdef HAVE_OPENCV_VIDEO init &= cv::initModule_video(); #endif -#ifdef HAVE_OPENCV_ML - init &= cv::initModule_ml(); -#endif - #ifdef HAVE_OPENCV_CONTRIB +#ifdef HAVE_OPENCV_CONTRIB init &= cv::initModule_contrib(); #endif diff --git a/modules/ml/doc/boosting.rst b/modules/ml/doc/boosting.rst index 7c5bc83fce1ca3f896d0e03d2302f877f1489c68..2ba4a031f4afbd68174ba3cf5638955862716bdd 100644 --- a/modules/ml/doc/boosting.rst +++ b/modules/ml/doc/boosting.rst @@ -63,41 +63,30 @@ training examples are recomputed at each training iteration. Examples deleted at .. [FHT98] Friedman, J. H., Hastie, T. and Tibshirani, R. Additive Logistic Regression: a Statistical View of Boosting. Technical Report, Dept. of Statistics*, Stanford University, 1998. -CvBoostParams +Boost::Params ------------- -.. ocv:struct:: CvBoostParams : public CvDTreeParams +.. ocv:struct:: Boost::Params : public DTree::Params Boosting training parameters. - There is one structure member that you can set directly: - - .. ocv:member:: int split_criteria - - Splitting criteria used to choose optimal splits during a weak tree construction. Possible values are: - - * **CvBoost::DEFAULT** Use the default for the particular boosting method, see below. - * **CvBoost::GINI** Use Gini index. This is default option for Real AdaBoost; may be also used for Discrete AdaBoost. - * **CvBoost::MISCLASS** Use misclassification rate. This is default option for Discrete AdaBoost; may be also used for Real AdaBoost. - * **CvBoost::SQERR** Use least squares criteria. This is default and the only option for LogitBoost and Gentle AdaBoost. - -The structure is derived from :ocv:class:`CvDTreeParams` but not all of the decision tree parameters are supported. In particular, cross-validation is not supported. +The structure is derived from ``DTrees::Params`` but not all of the decision tree parameters are supported. In particular, cross-validation is not supported. All parameters are public. You can initialize them by a constructor and then override some of them directly if you want. -CvBoostParams::CvBoostParams +Boost::Params::Params ---------------------------- The constructors. -.. ocv:function:: CvBoostParams::CvBoostParams() +.. ocv:function:: Boost::Params::Params() -.. ocv:function:: CvBoostParams::CvBoostParams( int boost_type, int weak_count, double weight_trim_rate, int max_depth, bool use_surrogates, const float* priors ) +.. ocv:function:: Boost::Params::Params( int boostType, int weakCount, double weightTrimRate, int maxDepth, bool useSurrogates, const Mat& priors ) :param boost_type: Type of the boosting algorithm. Possible values are: - * **CvBoost::DISCRETE** Discrete AdaBoost. - * **CvBoost::REAL** Real AdaBoost. It is a technique that utilizes confidence-rated predictions and works well with categorical data. - * **CvBoost::LOGIT** LogitBoost. It can produce good regression fits. - * **CvBoost::GENTLE** Gentle AdaBoost. It puts less weight on outlier data points and for that reason is often good with regression data. + * **Boost::DISCRETE** Discrete AdaBoost. + * **Boost::REAL** Real AdaBoost. It is a technique that utilizes confidence-rated predictions and works well with categorical data. + * **Boost::LOGIT** LogitBoost. It can produce good regression fits. + * **Boost::GENTLE** Gentle AdaBoost. It puts less weight on outlier data points and for that reason is often good with regression data. Gentle AdaBoost and Real AdaBoost are often the preferable choices. @@ -105,131 +94,54 @@ The constructors. :param weight_trim_rate: A threshold between 0 and 1 used to save computational time. Samples with summary weight :math:`\leq 1 - weight\_trim\_rate` do not participate in the *next* iteration of training. Set this parameter to 0 to turn off this functionality. -See :ocv:func:`CvDTreeParams::CvDTreeParams` for description of other parameters. +See ``DTrees::Params`` for description of other parameters. Default parameters are: :: - CvBoostParams::CvBoostParams() + Boost::Params::Params() { - boost_type = CvBoost::REAL; - weak_count = 100; - weight_trim_rate = 0.95; - cv_folds = 0; - max_depth = 1; + boostType = Boost::REAL; + weakCount = 100; + weightTrimRate = 0.95; + CVFolds = 0; + maxDepth = 1; } -CvBoostTree ------------ -.. ocv:class:: CvBoostTree : public CvDTree - -The weak tree classifier, a component of the boosted tree classifier :ocv:class:`CvBoost`, is a derivative of :ocv:class:`CvDTree`. Normally, there is no need to use the weak classifiers directly. However, they can be accessed as elements of the sequence ``CvBoost::weak``, retrieved by :ocv:func:`CvBoost::get_weak_predictors`. - -.. note:: In case of LogitBoost and Gentle AdaBoost, each weak predictor is a regression tree, rather than a classification tree. Even in case of Discrete AdaBoost and Real AdaBoost, the ``CvBoostTree::predict`` return value (:ocv:member:`CvDTreeNode::value`) is not an output class label. A negative value "votes" for class #0, a positive value - for class #1. The votes are weighted. The weight of each individual tree may be increased or decreased using the method ``CvBoostTree::scale``. - -CvBoost +Boost ------- -.. ocv:class:: CvBoost : public CvStatModel - -Boosted tree classifier derived from :ocv:class:`CvStatModel`. - -CvBoost::CvBoost ----------------- -Default and training constructors. - -.. ocv:function:: CvBoost::CvBoost() - -.. ocv:function:: CvBoost::CvBoost( const Mat& trainData, int tflag, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), const Mat& varType=Mat(), const Mat& missingDataMask=Mat(), CvBoostParams params=CvBoostParams() ) - -.. ocv:function:: CvBoost::CvBoost( const CvMat* trainData, int tflag, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, const CvMat* varType=0, const CvMat* missingDataMask=0, CvBoostParams params=CvBoostParams() ) - -.. ocv:pyfunction:: cv2.Boost([trainData, tflag, responses[, varIdx[, sampleIdx[, varType[, missingDataMask[, params]]]]]]) -> - - -The constructors follow conventions of :ocv:func:`CvStatModel::CvStatModel`. See :ocv:func:`CvStatModel::train` for parameters descriptions. - -CvBoost::train --------------- -Trains a boosted tree classifier. - -.. ocv:function:: bool CvBoost::train( const Mat& trainData, int tflag, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), const Mat& varType=Mat(), const Mat& missingDataMask=Mat(), CvBoostParams params=CvBoostParams(), bool update=false ) - -.. ocv:function:: bool CvBoost::train( const CvMat* trainData, int tflag, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, const CvMat* varType=0, const CvMat* missingDataMask=0, CvBoostParams params=CvBoostParams(), bool update=false ) - -.. ocv:function:: bool CvBoost::train( CvMLData* data, CvBoostParams params=CvBoostParams(), bool update=false ) - -.. ocv:pyfunction:: cv2.Boost.train(trainData, tflag, responses[, varIdx[, sampleIdx[, varType[, missingDataMask[, params[, update]]]]]]) -> retval - - :param update: Specifies whether the classifier needs to be updated (``true``, the new weak tree classifiers added to the existing ensemble) or the classifier needs to be rebuilt from scratch (``false``). +.. ocv:class:: Boost : public DTrees -The train method follows the common template of :ocv:func:`CvStatModel::train`. The responses must be categorical, which means that boosted trees cannot be built for regression, and there should be two classes. +Boosted tree classifier derived from ``DTrees`` -CvBoost::predict +Boost::create ---------------- -Predicts a response for an input sample. +Creates the empty model -.. ocv:function:: float CvBoost::predict( const cv::Mat& sample, const cv::Mat& missing=Mat(), const cv::Range& slice=Range::all(), bool rawMode=false, bool returnSum=false ) const +.. ocv:function:: Ptr Boost::create(const Params& params=Params()) -.. ocv:function:: float CvBoost::predict( const CvMat* sample, const CvMat* missing=0, CvMat* weak_responses=0, CvSlice slice=CV_WHOLE_SEQ, bool raw_mode=false, bool return_sum=false ) const - -.. ocv:pyfunction:: cv2.Boost.predict(sample[, missing[, slice[, rawMode[, returnSum]]]]) -> retval - - :param sample: Input sample. - - :param missing: Optional mask of missing measurements. To handle missing measurements, the weak classifiers must include surrogate splits (see ``CvDTreeParams::use_surrogates``). - - :param weak_responses: Optional output parameter, a floating-point vector with responses of each individual weak classifier. The number of elements in the vector must be equal to the slice length. - - :param slice: Continuous subset of the sequence of weak classifiers to be used for prediction. By default, all the weak classifiers are used. - - :param rawMode: Normally, it should be set to ``false``. - - :param returnSum: If ``true`` then return sum of votes instead of the class label. - -The method runs the sample through the trees in the ensemble and returns the output class label based on the weighted voting. - -CvBoost::prune --------------- -Removes the specified weak classifiers. - -.. ocv:function:: void CvBoost::prune( CvSlice slice ) - -.. ocv:pyfunction:: cv2.Boost.prune(slice) -> None - - :param slice: Continuous subset of the sequence of weak classifiers to be removed. - -The method removes the specified weak classifiers from the sequence. - -.. note:: Do not confuse this method with the pruning of individual decision trees, which is currently not supported. - - -CvBoost::calc_error -------------------- -Returns error of the boosted tree classifier. - -.. ocv:function:: float CvBoost::calc_error( CvMLData* _data, int type , std::vector *resp = 0 ) - -The method is identical to :ocv:func:`CvDTree::calc_error` but uses the boosted tree classifier as predictor. +Use ``StatModel::train`` to train the model, ``StatModel::train(traindata, params)`` to create and train the model, ``StatModel::load(filename)`` to load the pre-trained model. +Boost::getBParams +----------------- +Returns the boosting parameters -CvBoost::get_weak_predictors ----------------------------- -Returns the sequence of weak tree classifiers. +.. ocv:function:: Params Boost::getBParams() const -.. ocv:function:: CvSeq* CvBoost::get_weak_predictors() +The method returns the training parameters. -The method returns the sequence of weak classifiers. Each element of the sequence is a pointer to the :ocv:class:`CvBoostTree` class or to some of its derivatives. +Boost::setBParams +----------------- +Sets the boosting parameters -CvBoost::get_params -------------------- -Returns current parameters of the boosted tree classifier. +.. ocv:function:: void Boost::setBParams( const Params& p ) -.. ocv:function:: const CvBoostParams& CvBoost::get_params() const + :param p: Training parameters of type Boost::Params. +The method sets the training parameters. -CvBoost::get_data ------------------ -Returns used train data of the boosted tree classifier. +Prediction with Boost +--------------------- -.. ocv:function:: const CvDTreeTrainData* CvBoost::get_data() const +StatModel::predict(samples, results, flags) should be used. Pass ``flags=StatModel::RAW_OUTPUT`` to get the raw sum from Boost classifier. diff --git a/modules/ml/doc/decision_trees.rst b/modules/ml/doc/decision_trees.rst index de6fc99d6303c93ed861d540d2925ab176a66cf2..474cca2e329a78aee5de7298ab6ad81247b503f1 100644 --- a/modules/ml/doc/decision_trees.rst +++ b/modules/ml/doc/decision_trees.rst @@ -3,10 +3,7 @@ Decision Trees The ML classes discussed in this section implement Classification and Regression Tree algorithms described in [Breiman84]_. -The class -:ocv:class:`CvDTree` represents a single decision tree that may be used alone or as a base class in tree ensembles (see -:ref:`Boosting` and -:ref:`Random Trees` ). +The class ``cv::ml::DTrees`` represents a single decision tree or a collection of decision trees. It's also a base class for ``RTrees`` and ``Boost``. A decision tree is a binary tree (tree where each non-leaf node has two child nodes). It can be used either for classification or for regression. For classification, each tree leaf is marked with a class label; multiple leaves may have the same label. For regression, a constant is also assigned to each tree leaf, so the approximation function is piecewise constant. @@ -55,123 +52,107 @@ Besides the prediction that is an obvious use of decision trees, the tree can be Importance of each variable is computed over all the splits on this variable in the tree, primary and surrogate ones. Thus, to compute variable importance correctly, the surrogate splits must be enabled in the training parameters, even if there is no missing data. -CvDTreeSplit ------------- -.. ocv:struct:: CvDTreeSplit - +DTrees::Split +------------- +.. ocv:class:: DTrees::Split - The structure represents a possible decision tree node split. It has public members: + The class represents split in a decision tree. It has public members: - .. ocv:member:: int var_idx + .. ocv:member:: int varIdx Index of variable on which the split is created. - .. ocv:member:: int inversed + .. ocv:member:: bool inversed - If it is not null then inverse split rule is used that is left and right branches are exchanged in the rule expressions below. + If true, then the inverse split rule is used (i.e. left and right branches are exchanged in the rule expressions below). .. ocv:member:: float quality - The split quality, a positive number. It is used to choose the best primary split, then to choose and sort the surrogate splits. After the tree is constructed, it is also used to compute variable importance. - - .. ocv:member:: CvDTreeSplit* next + The split quality, a positive number. It is used to choose the best split. - Pointer to the next split in the node list of splits. + .. ocv:member:: int next - .. ocv:member:: int[] subset - - Bit array indicating the value subset in case of split on a categorical variable. The rule is: :: - - if var_value in subset - then next_node <- left - else next_node <- right + Index of the next split in the list of splits for the node - .. ocv:member:: float ord::c + .. ocv:member:: float c The threshold value in case of split on an ordered variable. The rule is: :: - if var_value < ord.c - then next_node<-left - else next_node<-right + if var_value < c + then next_node<-left + else next_node<-right - .. ocv:member:: int ord::split_point + .. ocv:member:: int subsetOfs - Used internally by the training algorithm. + Offset of the bitset used by the split on a categorical variable. The rule is: :: -CvDTreeNode ------------ -.. ocv:struct:: CvDTreeNode - - - The structure represents a node in a decision tree. It has public members: - - .. ocv:member:: int class_idx - - Class index normalized to 0..class_count-1 range and assigned to the node. It is used internally in classification trees and tree ensembles. + if bitset[var_value] == 1 + then next_node <- left + else next_node <- right - .. ocv:member:: int Tn +DTrees::Node +------------ +.. ocv:class:: DTrees::Node - Tree index in a ordered sequence of pruned trees. The indices are used during and after the pruning procedure. The root node has the maximum value ``Tn`` of the whole tree, child nodes have ``Tn`` less than or equal to the parent's ``Tn``, and nodes with :math:`Tn \leq CvDTree::pruned\_tree\_idx` are not used at prediction stage (the corresponding branches are considered as cut-off), even if they have not been physically deleted from the tree at the pruning stage. + The class represents a decision tree node. It has public members: .. ocv:member:: double value Value at the node: a class label in case of classification or estimated function value in case of regression. - .. ocv:member:: CvDTreeNode* parent - - Pointer to the parent node. + .. ocv:member:: int classIdx - .. ocv:member:: CvDTreeNode* left + Class index normalized to 0..class_count-1 range and assigned to the node. It is used internally in classification trees and tree ensembles. - Pointer to the left child node. + .. ocv:member:: int parent - .. ocv:member:: CvDTreeNode* right + Index of the parent node - Pointer to the right child node. + .. ocv:member:: int left - .. ocv:member:: CvDTreeSplit* split + Index of the left child node - Pointer to the first (primary) split in the node list of splits. + .. ocv:member:: int right - .. ocv:member:: int sample_count + Index of right child node. - The number of samples that fall into the node at the training stage. It is used to resolve the difficult cases - when the variable for the primary split is missing and all the variables for other surrogate splits are missing too. In this case the sample is directed to the left if ``left->sample_count > right->sample_count`` and to the right otherwise. + .. ocv:member:: int defaultDir - .. ocv:member:: int depth + Default direction where to go (-1: left or +1: right). It helps in the case of missing values. - Depth of the node. The root node depth is 0, the child nodes depth is the parent's depth + 1. + .. ocv:member:: int split -Other numerous fields of ``CvDTreeNode`` are used internally at the training stage. + Index of the first split -CvDTreeParams -------------- -.. ocv:struct:: CvDTreeParams +DTrees::Params +--------------- +.. ocv:class:: DTrees::Params The structure contains all the decision tree training parameters. You can initialize it by default constructor and then override any parameters directly before training, or the structure may be fully initialized using the advanced variant of the constructor. -CvDTreeParams::CvDTreeParams +DTrees::Params::Params ---------------------------- -The constructors. +The constructors -.. ocv:function:: CvDTreeParams::CvDTreeParams() +.. ocv:function:: DTrees::Params::Params() -.. ocv:function:: CvDTreeParams::CvDTreeParams( int max_depth, int min_sample_count, float regression_accuracy, bool use_surrogates, int max_categories, int cv_folds, bool use_1se_rule, bool truncate_pruned_tree, const float* priors ) +.. ocv:function:: DTrees::Params::Params( int maxDepth, int minSampleCount, double regressionAccuracy, bool useSurrogates, int maxCategories, int CVFolds, bool use1SERule, bool truncatePrunedTree, const Mat& priors ) - :param max_depth: The maximum possible depth of the tree. That is the training algorithms attempts to split a node while its depth is less than ``max_depth``. The actual depth may be smaller if the other termination criteria are met (see the outline of the training procedure in the beginning of the section), and/or if the tree is pruned. + :param maxDepth: The maximum possible depth of the tree. That is the training algorithms attempts to split a node while its depth is less than ``maxDepth``. The root node has zero depth. The actual depth may be smaller if the other termination criteria are met (see the outline of the training procedure in the beginning of the section), and/or if the tree is pruned. - :param min_sample_count: If the number of samples in a node is less than this parameter then the node will not be split. + :param minSampleCount: If the number of samples in a node is less than this parameter then the node will not be split. - :param regression_accuracy: Termination criteria for regression trees. If all absolute differences between an estimated value in a node and values of train samples in this node are less than this parameter then the node will not be split. + :param regressionAccuracy: Termination criteria for regression trees. If all absolute differences between an estimated value in a node and values of train samples in this node are less than this parameter then the node will not be split further. - :param use_surrogates: If true then surrogate splits will be built. These splits allow to work with missing data and compute variable importance correctly. + :param useSurrogates: If true then surrogate splits will be built. These splits allow to work with missing data and compute variable importance correctly. .. note:: currently it's not implemented. - :param max_categories: Cluster possible values of a categorical variable into ``K`` :math:`\leq` ``max_categories`` clusters to find a suboptimal split. If a discrete variable, on which the training procedure tries to make a split, takes more than ``max_categories`` values, the precise best subset estimation may take a very long time because the algorithm is exponential. Instead, many decision trees engines (including ML) try to find sub-optimal split in this case by clustering all the samples into ``max_categories`` clusters that is some categories are merged together. The clustering is applied only in ``n``>2-class classification problems for categorical variables with ``N > max_categories`` possible values. In case of regression and 2-class classification the optimal split can be found efficiently without employing clustering, thus the parameter is not used in these cases. + :param maxCategories: Cluster possible values of a categorical variable into ``K<=maxCategories`` clusters to find a suboptimal split. If a discrete variable, on which the training procedure tries to make a split, takes more than ``maxCategories`` values, the precise best subset estimation may take a very long time because the algorithm is exponential. Instead, many decision trees engines (including our implementation) try to find sub-optimal split in this case by clustering all the samples into ``maxCategories`` clusters that is some categories are merged together. The clustering is applied only in ``n > 2``-class classification problems for categorical variables with ``N > max_categories`` possible values. In case of regression and 2-class classification the optimal split can be found efficiently without employing clustering, thus the parameter is not used in these cases. - :param cv_folds: If ``cv_folds > 1`` then prune a tree with ``K``-fold cross-validation where ``K`` is equal to ``cv_folds``. + :param CVFolds: If ``CVFolds > 1`` then algorithms prunes the built decision tree using ``K``-fold cross-validation procedure where ``K`` is equal to ``CVFolds``. - :param use_1se_rule: If true then a pruning will be harsher. This will make a tree more compact and more resistant to the training data noise but a bit less accurate. + :param use1SERule: If true then a pruning will be harsher. This will make a tree more compact and more resistant to the training data noise but a bit less accurate. - :param truncate_pruned_tree: If true then pruned branches are physically removed from the tree. Otherwise they are retained and it is possible to get results from the original unpruned (or pruned less aggressively) tree by decreasing ``CvDTree::pruned_tree_idx`` parameter. + :param truncatePrunedTree: If true then pruned branches are physically removed from the tree. Otherwise they are retained and it is possible to get results from the original unpruned (or pruned less aggressively) tree. :param priors: The array of a priori class probabilities, sorted by the class label value. The parameter can be used to tune the decision tree preferences toward a certain class. For example, if you want to detect some rare anomaly occurrence, the training base will likely contain much more normal cases than anomalies, so a very good classification performance will be achieved just by considering every case as normal. To avoid this, the priors can be specified, where the anomaly probability is artificially increased (up to 0.5 or even greater), so the weight of the misclassified anomalies becomes much bigger, and the tree is adjusted properly. You can also think about this parameter as weights of prediction categories which determine relative weights that you give to misclassification. That is, if the weight of the first category is 1 and the weight of the second category is 10, then each mistake in predicting the second category is equivalent to making 10 mistakes in predicting the first category. @@ -179,142 +160,82 @@ The default constructor initializes all the parameters with the default values t :: - CvDTreeParams() : max_categories(10), max_depth(INT_MAX), min_sample_count(10), - cv_folds(10), use_surrogates(true), use_1se_rule(true), - truncate_pruned_tree(true), regression_accuracy(0.01f), priors(0) - {} - - -CvDTreeTrainData ----------------- -.. ocv:struct:: CvDTreeTrainData - -Decision tree training data and shared data for tree ensembles. The structure is mostly used internally for storing both standalone trees and tree ensembles efficiently. Basically, it contains the following types of information: - -#. Training parameters, an instance of :ocv:class:`CvDTreeParams`. - -#. Training data preprocessed to find the best splits more efficiently. For tree ensembles, this preprocessed data is reused by all trees. Additionally, the training data characteristics shared by all trees in the ensemble are stored here: variable types, the number of classes, a class label compression map, and so on. - -#. Buffers, memory storages for tree nodes, splits, and other elements of the constructed trees. - -There are two ways of using this structure. In simple cases (for example, a standalone tree or the ready-to-use "black box" tree ensemble from machine learning, like -:ref:`Random Trees` or -:ref:`Boosting` ), there is no need to care or even to know about the structure. You just construct the needed statistical model, train it, and use it. The ``CvDTreeTrainData`` structure is constructed and used internally. However, for custom tree algorithms or another sophisticated cases, the structure may be constructed and used explicitly. The scheme is the following: - -#. - The structure is initialized using the default constructor, followed by ``set_data``, or it is built using the full form of constructor. The parameter ``_shared`` must be set to ``true``. - -#. - One or more trees are trained using this data (see the special form of the method :ocv:func:`CvDTree::train`). + DTrees::Params::Params() + { + maxDepth = INT_MAX; + minSampleCount = 10; + regressionAccuracy = 0.01f; + useSurrogates = false; + maxCategories = 10; + CVFolds = 10; + use1SERule = true; + truncatePrunedTree = true; + priors = Mat(); + } -#. - The structure is released as soon as all the trees using it are released. -CvDTree -------- -.. ocv:class:: CvDTree : public CvStatModel +DTrees +------ -The class implements a decision tree as described in the beginning of this section. +.. ocv:class:: DTrees : public StatModel +The class represents a single decision tree or a collection of decision trees. The current public interface of the class allows user to train only a single decision tree, however the class is capable of storing multiple decision trees and using them for prediction (by summing responses or using a voting schemes), and the derived from DTrees classes (such as ``RTrees`` and ``Boost``) use this capability to implement decision tree ensembles. -CvDTree::train --------------- -Trains a decision tree. - -.. ocv:function:: bool CvDTree::train( const Mat& trainData, int tflag, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), const Mat& varType=Mat(), const Mat& missingDataMask=Mat(), CvDTreeParams params=CvDTreeParams() ) - -.. ocv:function:: bool CvDTree::train( const CvMat* trainData, int tflag, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, const CvMat* varType=0, const CvMat* missingDataMask=0, CvDTreeParams params=CvDTreeParams() ) - -.. ocv:function:: bool CvDTree::train( CvMLData* trainData, CvDTreeParams params=CvDTreeParams() ) - -.. ocv:function:: bool CvDTree::train( CvDTreeTrainData* trainData, const CvMat* subsampleIdx ) - -.. ocv:pyfunction:: cv2.DTree.train(trainData, tflag, responses[, varIdx[, sampleIdx[, varType[, missingDataMask[, params]]]]]) -> retval - -There are four ``train`` methods in :ocv:class:`CvDTree`: - -* The **first two** methods follow the generic :ocv:func:`CvStatModel::train` conventions. It is the most complete form. Both data layouts (``tflag=CV_ROW_SAMPLE`` and ``tflag=CV_COL_SAMPLE``) are supported, as well as sample and variable subsets, missing measurements, arbitrary combinations of input and output variable types, and so on. The last parameter contains all of the necessary training parameters (see the :ocv:class:`CvDTreeParams` description). - -* The **third** method uses :ocv:class:`CvMLData` to pass training data to a decision tree. - -* The **last** method ``train`` is mostly used for building tree ensembles. It takes the pre-constructed :ocv:class:`CvDTreeTrainData` instance and an optional subset of the training set. The indices in ``subsampleIdx`` are counted relatively to the ``_sample_idx`` , passed to the ``CvDTreeTrainData`` constructor. For example, if ``_sample_idx=[1, 5, 7, 100]`` , then ``subsampleIdx=[0,3]`` means that the samples ``[1, 100]`` of the original training set are used. - -The function is parallelized with the TBB library. - - - -CvDTree::predict +DTrees::create ---------------- -Returns the leaf node of a decision tree corresponding to the input vector. - -.. ocv:function:: CvDTreeNode* CvDTree::predict( const Mat& sample, const Mat& missingDataMask=Mat(), bool preprocessedInput=false ) const - -.. ocv:function:: CvDTreeNode* CvDTree::predict( const CvMat* sample, const CvMat* missingDataMask=0, bool preprocessedInput=false ) const +Creates the empty model -.. ocv:pyfunction:: cv2.DTree.predict(sample[, missingDataMask[, preprocessedInput]]) -> retval +.. ocv:function:: Ptr DTrees::create(const Params& params=Params()) - :param sample: Sample for prediction. +The static method creates empty decision tree with the specified parameters. It should be then trained using ``train`` method (see ``StatModel::train``). Alternatively, you can load the model from file using ``StatModel::load(filename)``. - :param missingDataMask: Optional input missing measurement mask. +DTrees::getDParams +------------------ +Returns the training parameters - :param preprocessedInput: This parameter is normally set to ``false``, implying a regular input. If it is ``true``, the method assumes that all the values of the discrete input variables have been already normalized to :math:`0` to :math:`num\_of\_categories_i-1` ranges since the decision tree uses such normalized representation internally. It is useful for faster prediction with tree ensembles. For ordered input variables, the flag is not used. +.. ocv:function:: Params DTrees::getDParams() const -The method traverses the decision tree and returns the reached leaf node as output. The prediction result, either the class label or the estimated function value, may be retrieved as the ``value`` field of the :ocv:class:`CvDTreeNode` structure, for example: ``dtree->predict(sample,mask)->value``. +The method returns the training parameters. - - -CvDTree::calc_error +DTrees::setDParams ------------------- -Returns error of the decision tree. - -.. ocv:function:: float CvDTree::calc_error( CvMLData* trainData, int type, std::vector *resp = 0 ) - - :param trainData: Data for the decision tree. - - :param type: Type of error. Possible values are: - - * **CV_TRAIN_ERROR** Error on train samples. - - * **CV_TEST_ERROR** Error on test samples. +Sets the training parameters - :param resp: If it is not null then size of this vector will be set to the number of samples and each element will be set to result of prediction on the corresponding sample. +.. ocv:function:: void DTrees::setDParams( const Params& p ) -The method calculates error of the decision tree. In case of classification it is the percentage of incorrectly classified samples and in case of regression it is the mean of squared errors on samples. + :param p: Training parameters of type DTrees::Params. +The method sets the training parameters. -CvDTree::getVarImportance -------------------------- -Returns the variable importance array. -.. ocv:function:: Mat CvDTree::getVarImportance() - -.. ocv:function:: const CvMat* CvDTree::get_var_importance() +DTrees::getRoots +------------------- +Returns indices of root nodes -.. ocv:pyfunction:: cv2.DTree.getVarImportance() -> retval +.. ocv:function:: std::vector& DTrees::getRoots() const -CvDTree::get_root ------------------ -Returns the root of the decision tree. +DTrees::getNodes +------------------- +Returns all the nodes -.. ocv:function:: const CvDTreeNode* CvDTree::get_root() const +.. ocv:function:: std::vector& DTrees::getNodes() const +all the node indices, mentioned above (left, right, parent, root indices) are indices in the returned vector -CvDTree::get_pruned_tree_idx ----------------------------- -Returns the ``CvDTree::pruned_tree_idx`` parameter. - -.. ocv:function:: int CvDTree::get_pruned_tree_idx() const +DTrees::getSplits +------------------- +Returns all the splits -The parameter ``DTree::pruned_tree_idx`` is used to prune a decision tree. See the ``CvDTreeNode::Tn`` parameter. +.. ocv:function:: std::vector& DTrees::getSplits() const -CvDTree::get_data ------------------ -Returns used train data of the decision tree. +all the split indices, mentioned above (split, next etc.) are indices in the returned vector -.. ocv:function:: CvDTreeTrainData* CvDTree::get_data() const +DTrees::getSubsets +------------------- +Returns all the bitsets for categorical splits -Example: building a tree for classifying mushrooms. See the ``mushroom.cpp`` sample that demonstrates how to build and use the -decision tree. +.. ocv:function:: std::vector& DTrees::getSubsets() const +``Split::subsetOfs`` is an offset in the returned vector .. [Breiman84] Breiman, L., Friedman, J. Olshen, R. and Stone, C. (1984), *Classification and Regression Trees*, Wadsworth. diff --git a/modules/ml/doc/ertrees.rst b/modules/ml/doc/ertrees.rst deleted file mode 100644 index 7e6d03e7fc6adda9a93c09df719e34ccb9fbef68..0000000000000000000000000000000000000000 --- a/modules/ml/doc/ertrees.rst +++ /dev/null @@ -1,15 +0,0 @@ -Extremely randomized trees -========================== - -Extremely randomized trees have been introduced by Pierre Geurts, Damien Ernst and Louis Wehenkel in the article "Extremely randomized trees", 2006 [http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.65.7485&rep=rep1&type=pdf]. The algorithm of growing Extremely randomized trees is similar to :ref:`Random Trees` (Random Forest), but there are two differences: - -#. Extremely randomized trees don't apply the bagging procedure to construct a set of the training samples for each tree. The same input training set is used to train all trees. - -#. Extremely randomized trees pick a node split very extremely (both a variable index and variable splitting value are chosen randomly), whereas Random Forest finds the best split (optimal one by variable index and variable splitting value) among random subset of variables. - - -CvERTrees ----------- -.. ocv:class:: CvERTrees : public CvRTrees - - The class implements the Extremely randomized trees algorithm. ``CvERTrees`` is inherited from :ocv:class:`CvRTrees` and has the same interface, so see description of :ocv:class:`CvRTrees` class to get details. To set the training parameters of Extremely randomized trees the same class :ocv:struct:`CvRTParams` is used. diff --git a/modules/ml/doc/expectation_maximization.rst b/modules/ml/doc/expectation_maximization.rst index b79dea820b7b5bfadb4fdc8d66eff83ce78ed92d..4b54007bb93d7fe68949598c6d5189ba53ec6ecc 100644 --- a/modules/ml/doc/expectation_maximization.rst +++ b/modules/ml/doc/expectation_maximization.rst @@ -66,7 +66,7 @@ Alternatively, the algorithm may start with the M-step when the initial values f :math:`p_{i,k}` can be provided. Another alternative when :math:`p_{i,k}` are unknown is to use a simpler clustering algorithm to pre-cluster the input samples and thus obtain initial :math:`p_{i,k}` . Often (including machine learning) the -:ocv:func:`kmeans` algorithm is used for that purpose. +``k-means`` algorithm is used for that purpose. One of the main problems of the EM algorithm is a large number of parameters to estimate. The majority of the parameters reside in @@ -91,18 +91,21 @@ already a good enough approximation). EM -- -.. ocv:class:: EM : public Algorithm +.. ocv:class:: EM : public StatModel -The class implements the EM algorithm as described in the beginning of this section. It is inherited from :ocv:class:`Algorithm`. +The class implements the EM algorithm as described in the beginning of this section. +EM::Params +---------- +.. ocv:class:: EM::Params -EM::EM ------- -The constructor of the class +The class describes EM training parameters. -.. ocv:function:: EM::EM(int nclusters=EM::DEFAULT_NCLUSTERS, int covMatType=EM::COV_MAT_DIAGONAL, const TermCriteria& termCrit=TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, EM::DEFAULT_MAX_ITERS, FLT_EPSILON) ) +EM::Params::Params +------------------ +The constructor -.. ocv:pyfunction:: cv2.EM([nclusters[, covMatType[, termCrit]]]) -> +.. ocv:function:: EM::Params::Params( int nclusters=DEFAULT_NCLUSTERS, int covMatType=EM::COV_MAT_DIAGONAL,const TermCriteria& termCrit=TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, EM::DEFAULT_MAX_ITERS, 1e-6)) :param nclusters: The number of mixture components in the Gaussian mixture model. Default value of the parameter is ``EM::DEFAULT_NCLUSTERS=5``. Some of EM implementation could determine the optimal number of mixtures within a specified value range, but that is not the case in ML yet. @@ -116,21 +119,26 @@ The constructor of the class :param termCrit: The termination criteria of the EM algorithm. The EM algorithm can be terminated by the number of iterations ``termCrit.maxCount`` (number of M-steps) or when relative change of likelihood logarithm is less than ``termCrit.epsilon``. Default maximum number of iterations is ``EM::DEFAULT_MAX_ITERS=100``. -EM::train ---------- -Estimates the Gaussian mixture parameters from a samples set. -.. ocv:function:: bool EM::train(InputArray samples, OutputArray logLikelihoods=noArray(), OutputArray labels=noArray(), OutputArray probs=noArray()) +EM::create +---------- +Creates empty EM model -.. ocv:function:: bool EM::trainE(InputArray samples, InputArray means0, InputArray covs0=noArray(), InputArray weights0=noArray(), OutputArray logLikelihoods=noArray(), OutputArray labels=noArray(), OutputArray probs=noArray()) +.. ocv:function:: Ptr EM::create(const Params& params=Params()) -.. ocv:function:: bool EM::trainM(InputArray samples, InputArray probs0, OutputArray logLikelihoods=noArray(), OutputArray labels=noArray(), OutputArray probs=noArray()) + :param params: EM parameters -.. ocv:pyfunction:: cv2.EM.train(samples[, logLikelihoods[, labels[, probs]]]) -> retval, logLikelihoods, labels, probs +The model should be trained then using ``StatModel::train(traindata, flags)`` method. Alternatively, you can use one of the ``EM::train*`` methods or load it from file using ``StatModel::load(filename)``. -.. ocv:pyfunction:: cv2.EM.trainE(samples, means0[, covs0[, weights0[, logLikelihoods[, labels[, probs]]]]]) -> retval, logLikelihoods, labels, probs +EM::train +--------- +Static methods that estimate the Gaussian mixture parameters from a samples set -.. ocv:pyfunction:: cv2.EM.trainM(samples, probs0[, logLikelihoods[, labels[, probs]]]) -> retval, logLikelihoods, labels, probs +.. ocv:function:: Ptr EM::train(InputArray samples, OutputArray logLikelihoods=noArray(), OutputArray labels=noArray(), OutputArray probs=noArray(), const Params& params=Params()) + +.. ocv:function:: bool EM::train_startWithE(InputArray samples, InputArray means0, InputArray covs0=noArray(), InputArray weights0=noArray(), OutputArray logLikelihoods=noArray(), OutputArray labels=noArray(), OutputArray probs=noArray(), const Params& params=Params()) + +.. ocv:function:: bool EM::train_startWithM(InputArray samples, InputArray probs0, OutputArray logLikelihoods=noArray(), OutputArray labels=noArray(), OutputArray probs=noArray(), const Params& params=Params()) :param samples: Samples from which the Gaussian mixture model will be estimated. It should be a one-channel matrix, each row of which is a sample. If the matrix does not have ``CV_64F`` type it will be converted to the inner matrix of such type for the further computing. @@ -148,6 +156,8 @@ Estimates the Gaussian mixture parameters from a samples set. :param probs: The optional output matrix that contains posterior probabilities of each Gaussian mixture component given the each sample. It has :math:`nsamples \times nclusters` size and ``CV_64FC1`` type. + :param params: The Gaussian mixture params, see ``EM::Params`` description above. + Three versions of training method differ in the initialization of Gaussian mixture model parameters and start step: * **train** - Starts with Expectation step. Initial values of the model parameters will be estimated by the k-means algorithm. @@ -167,15 +177,13 @@ Unlike many of the ML models, EM is an unsupervised learning algorithm and it do :math:`\texttt{labels}_i=\texttt{arg max}_k(p_{i,k}), i=1..N` (indices of the most probable mixture component for each sample). The trained model can be used further for prediction, just like any other classifier. The trained model is similar to the -:ocv:class:`CvNormalBayesClassifier`. +``NormalBayesClassifier``. -EM::predict ------------ +EM::predict2 +------------ Returns a likelihood logarithm value and an index of the most probable mixture component for the given sample. -.. ocv:function:: Vec2d EM::predict(InputArray sample, OutputArray probs=noArray()) const - -.. ocv:pyfunction:: cv2.EM.predict(sample[, probs]) -> retval, probs +.. ocv:function:: Vec2d EM::predict2(InputArray sample, OutputArray probs=noArray()) const :param sample: A sample for classification. It should be a one-channel matrix of :math:`1 \times dims` or :math:`dims \times 1` size. @@ -183,28 +191,29 @@ Returns a likelihood logarithm value and an index of the most probable mixture c The method returns a two-element ``double`` vector. Zero element is a likelihood logarithm value for the sample. First element is an index of the most probable mixture component for the given sample. -CvEM::isTrained ---------------- -Returns ``true`` if the Gaussian mixture model was trained. -.. ocv:function:: bool EM::isTrained() const +EM::getMeans +------------ +Returns the cluster centers (means of the Gaussian mixture) + +.. ocv:function:: Mat EM::getMeans() const + +Returns matrix with the number of rows equal to the number of mixtures and number of columns equal to the space dimensionality. + + +EM::getWeights +-------------- +Returns weights of the mixtures + +.. ocv:function:: Mat EM::getWeights() const -.. ocv:pyfunction:: cv2.EM.isTrained() -> retval +Returns vector with the number of elements equal to the number of mixtures. -EM::read, EM::write -------------------- -See :ocv:func:`Algorithm::read` and :ocv:func:`Algorithm::write`. -EM::get, EM::set ----------------- -See :ocv:func:`Algorithm::get` and :ocv:func:`Algorithm::set`. The following parameters are available: +EM::getCovs +-------------- +Returns covariation matrices -* ``"nclusters"`` -* ``"covMatType"`` -* ``"maxIters"`` -* ``"epsilon"`` -* ``"weights"`` *(read-only)* -* ``"means"`` *(read-only)* -* ``"covs"`` *(read-only)* +.. ocv:function:: void EM::getCovs(std::vector& covs) const -.. +Returns vector of covariation matrices. Number of matrices is the number of gaussian mixtures, each matrix is a square floating-point matrix NxN, where N is the space dimensionality. diff --git a/modules/ml/doc/gradient_boosted_trees.rst b/modules/ml/doc/gradient_boosted_trees.rst deleted file mode 100644 index b83c47e4e1726e0139964c68f6694a4b2da41188..0000000000000000000000000000000000000000 --- a/modules/ml/doc/gradient_boosted_trees.rst +++ /dev/null @@ -1,272 +0,0 @@ -.. _Gradient Boosted Trees: - -Gradient Boosted Trees -====================== - -.. highlight:: cpp - -Gradient Boosted Trees (GBT) is a generalized boosting algorithm introduced by -Jerome Friedman: http://www.salfordsystems.com/doc/GreedyFuncApproxSS.pdf . -In contrast to the AdaBoost.M1 algorithm, GBT can deal with both multiclass -classification and regression problems. Moreover, it can use any -differential loss function, some popular ones are implemented. -Decision trees (:ocv:class:`CvDTree`) usage as base learners allows to process ordered -and categorical variables. - -.. _Training GBT: - -Training the GBT model ----------------------- - -Gradient Boosted Trees model represents an ensemble of single regression trees -built in a greedy fashion. Training procedure is an iterative process -similar to the numerical optimization via the gradient descent method. Summary loss -on the training set depends only on the current model predictions for the -training samples, in other words -:math:`\sum^N_{i=1}L(y_i, F(x_i)) \equiv \mathcal{L}(F(x_1), F(x_2), ... , F(x_N)) -\equiv \mathcal{L}(F)`. And the :math:`\mathcal{L}(F)` -gradient can be computed as follows: - -.. math:: - grad(\mathcal{L}(F)) = \left( \dfrac{\partial{L(y_1, F(x_1))}}{\partial{F(x_1)}}, - \dfrac{\partial{L(y_2, F(x_2))}}{\partial{F(x_2)}}, ... , - \dfrac{\partial{L(y_N, F(x_N))}}{\partial{F(x_N)}} \right) . - -At every training step, a single regression tree is built to predict an -antigradient vector components. Step length is computed corresponding to the -loss function and separately for every region determined by the tree leaf. It -can be eliminated by changing values of the leaves directly. - -See below the main scheme of the training process: - -#. - Find the best constant model. -#. - For :math:`i` in :math:`[1,M]`: - - #. - Compute the antigradient. - #. - Grow a regression tree to predict antigradient components. - #. - Change values in the tree leaves. - #. - Add the tree to the model. - - -The following loss functions are implemented for regression problems: - -* - Squared loss (``CvGBTrees::SQUARED_LOSS``): - :math:`L(y,f(x))=\dfrac{1}{2}(y-f(x))^2` -* - Absolute loss (``CvGBTrees::ABSOLUTE_LOSS``): - :math:`L(y,f(x))=|y-f(x)|` -* - Huber loss (``CvGBTrees::HUBER_LOSS``): - :math:`L(y,f(x)) = \left\{ \begin{array}{lr} - \delta\cdot\left(|y-f(x)|-\dfrac{\delta}{2}\right) & : |y-f(x)|>\delta\\ - \dfrac{1}{2}\cdot(y-f(x))^2 & : |y-f(x)|\leq\delta \end{array} \right.`, - - where :math:`\delta` is the :math:`\alpha`-quantile estimation of the - :math:`|y-f(x)|`. In the current implementation :math:`\alpha=0.2`. - - -The following loss functions are implemented for classification problems: - -* - Deviance or cross-entropy loss (``CvGBTrees::DEVIANCE_LOSS``): - :math:`K` functions are built, one function for each output class, and - :math:`L(y,f_1(x),...,f_K(x)) = -\sum^K_{k=0}1(y=k)\ln{p_k(x)}`, - where :math:`p_k(x)=\dfrac{\exp{f_k(x)}}{\sum^K_{i=1}\exp{f_i(x)}}` - is the estimation of the probability of :math:`y=k`. - -As a result, you get the following model: - -.. math:: f(x) = f_0 + \nu\cdot\sum^M_{i=1}T_i(x) , - -where :math:`f_0` is the initial guess (the best constant model) and :math:`\nu` -is a regularization parameter from the interval :math:`(0,1]`, further called -*shrinkage*. - -.. _Predicting with GBT: - -Predicting with the GBT Model ------------------------------ - -To get the GBT model prediction, you need to compute the sum of responses of -all the trees in the ensemble. For regression problems, it is the answer. -For classification problems, the result is :math:`\arg\max_{i=1..K}(f_i(x))`. - - -.. highlight:: cpp - - -CvGBTreesParams ---------------- -.. ocv:struct:: CvGBTreesParams : public CvDTreeParams - -GBT training parameters. - -The structure contains parameters for each single decision tree in the ensemble, -as well as the whole model characteristics. The structure is derived from -:ocv:class:`CvDTreeParams` but not all of the decision tree parameters are supported: -cross-validation, pruning, and class priorities are not used. - -CvGBTreesParams::CvGBTreesParams --------------------------------- -.. ocv:function:: CvGBTreesParams::CvGBTreesParams() - -.. ocv:function:: CvGBTreesParams::CvGBTreesParams( int loss_function_type, int weak_count, float shrinkage, float subsample_portion, int max_depth, bool use_surrogates ) - - :param loss_function_type: Type of the loss function used for training - (see :ref:`Training GBT`). It must be one of the - following types: ``CvGBTrees::SQUARED_LOSS``, ``CvGBTrees::ABSOLUTE_LOSS``, - ``CvGBTrees::HUBER_LOSS``, ``CvGBTrees::DEVIANCE_LOSS``. The first three - types are used for regression problems, and the last one for - classification. - - :param weak_count: Count of boosting algorithm iterations. ``weak_count*K`` is the total - count of trees in the GBT model, where ``K`` is the output classes count - (equal to one in case of a regression). - - :param shrinkage: Regularization parameter (see :ref:`Training GBT`). - - :param subsample_portion: Portion of the whole training set used for each algorithm iteration. - Subset is generated randomly. For more information see - http://www.salfordsystems.com/doc/StochasticBoostingSS.pdf. - - :param max_depth: Maximal depth of each decision tree in the ensemble (see :ocv:class:`CvDTree`). - - :param use_surrogates: If ``true``, surrogate splits are built (see :ocv:class:`CvDTree`). - -By default the following constructor is used: - -.. code-block:: cpp - - CvGBTreesParams(CvGBTrees::SQUARED_LOSS, 200, 0.01f, 0.8f, 3, false) - : CvDTreeParams( 3, 10, 0, false, 10, 0, false, false, 0 ) - -CvGBTrees ---------- -.. ocv:class:: CvGBTrees : public CvStatModel - -The class implements the Gradient boosted tree model as described in the beginning of this section. - -CvGBTrees::CvGBTrees --------------------- -Default and training constructors. - -.. ocv:function:: CvGBTrees::CvGBTrees() - -.. ocv:function:: CvGBTrees::CvGBTrees( const Mat& trainData, int tflag, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), const Mat& varType=Mat(), const Mat& missingDataMask=Mat(), CvGBTreesParams params=CvGBTreesParams() ) - -.. ocv:function:: CvGBTrees::CvGBTrees( const CvMat* trainData, int tflag, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, const CvMat* varType=0, const CvMat* missingDataMask=0, CvGBTreesParams params=CvGBTreesParams() ) - -.. ocv:pyfunction:: cv2.GBTrees([trainData, tflag, responses[, varIdx[, sampleIdx[, varType[, missingDataMask[, params]]]]]]) -> - -The constructors follow conventions of :ocv:func:`CvStatModel::CvStatModel`. See :ocv:func:`CvStatModel::train` for parameters descriptions. - -CvGBTrees::train ----------------- -Trains a Gradient boosted tree model. - -.. ocv:function:: bool CvGBTrees::train(const Mat& trainData, int tflag, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), const Mat& varType=Mat(), const Mat& missingDataMask=Mat(), CvGBTreesParams params=CvGBTreesParams(), bool update=false) - -.. ocv:function:: bool CvGBTrees::train( const CvMat* trainData, int tflag, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, const CvMat* varType=0, const CvMat* missingDataMask=0, CvGBTreesParams params=CvGBTreesParams(), bool update=false ) - -.. ocv:function:: bool CvGBTrees::train(CvMLData* data, CvGBTreesParams params=CvGBTreesParams(), bool update=false) - -.. ocv:pyfunction:: cv2.GBTrees.train(trainData, tflag, responses[, varIdx[, sampleIdx[, varType[, missingDataMask[, params[, update]]]]]]) -> retval - -The first train method follows the common template (see :ocv:func:`CvStatModel::train`). -Both ``tflag`` values (``CV_ROW_SAMPLE``, ``CV_COL_SAMPLE``) are supported. -``trainData`` must be of the ``CV_32F`` type. ``responses`` must be a matrix of type -``CV_32S`` or ``CV_32F``. In both cases it is converted into the ``CV_32F`` -matrix inside the training procedure. ``varIdx`` and ``sampleIdx`` must be a -list of indices (``CV_32S``) or a mask (``CV_8U`` or ``CV_8S``). ``update`` is -a dummy parameter. - -The second form of :ocv:func:`CvGBTrees::train` function uses :ocv:class:`CvMLData` as a -data set container. ``update`` is still a dummy parameter. - -All parameters specific to the GBT model are passed into the training function -as a :ocv:class:`CvGBTreesParams` structure. - - -CvGBTrees::predict ------------------- -Predicts a response for an input sample. - -.. ocv:function:: float CvGBTrees::predict(const Mat& sample, const Mat& missing=Mat(), const Range& slice = Range::all(), int k=-1) const - -.. ocv:function:: float CvGBTrees::predict( const CvMat* sample, const CvMat* missing=0, CvMat* weakResponses=0, CvSlice slice = CV_WHOLE_SEQ, int k=-1 ) const - -.. ocv:pyfunction:: cv2.GBTrees.predict(sample[, missing[, slice[, k]]]) -> retval - - :param sample: Input feature vector that has the same format as every training set - element. If not all the variables were actually used during training, - ``sample`` contains forged values at the appropriate places. - - :param missing: Missing values mask, which is a dimensional matrix of the same size as - ``sample`` having the ``CV_8U`` type. ``1`` corresponds to the missing value - in the same position in the ``sample`` vector. If there are no missing values - in the feature vector, an empty matrix can be passed instead of the missing mask. - - :param weakResponses: Matrix used to obtain predictions of all the trees. - The matrix has :math:`K` rows, - where :math:`K` is the count of output classes (1 for the regression case). - The matrix has as many columns as the ``slice`` length. - - :param slice: Parameter defining the part of the ensemble used for prediction. - If ``slice = Range::all()``, all trees are used. Use this parameter to - get predictions of the GBT models with different ensemble sizes learning - only one model. - - :param k: Number of tree ensembles built in case of the classification problem - (see :ref:`Training GBT`). Use this - parameter to change the output to sum of the trees' predictions in the - ``k``-th ensemble only. To get the total GBT model prediction, ``k`` value - must be -1. For regression problems, ``k`` is also equal to -1. - -The method predicts the response corresponding to the given sample -(see :ref:`Predicting with GBT`). -The result is either the class label or the estimated function value. The -:ocv:func:`CvGBTrees::predict` method enables using the parallel version of the GBT model -prediction if the OpenCV is built with the TBB library. In this case, predictions -of single trees are computed in a parallel fashion. - - -CvGBTrees::clear ----------------- -Clears the model. - -.. ocv:function:: void CvGBTrees::clear() - -.. ocv:pyfunction:: cv2.GBTrees.clear() -> None - -The function deletes the data set information and all the weak models and sets all internal -variables to the initial state. The function is called in :ocv:func:`CvGBTrees::train` and in the -destructor. - - -CvGBTrees::calc_error ---------------------- -Calculates a training or testing error. - -.. ocv:function:: float CvGBTrees::calc_error( CvMLData* _data, int type, std::vector *resp = 0 ) - - :param _data: Data set. - - :param type: Parameter defining the error that should be computed: train (``CV_TRAIN_ERROR``) or test - (``CV_TEST_ERROR``). - - :param resp: If non-zero, a vector of predictions on the corresponding data set is - returned. - -If the :ocv:class:`CvMLData` data is used to store the data set, :ocv:func:`CvGBTrees::calc_error` can be -used to get a training/testing error easily and (optionally) all predictions -on the training/testing set. If the Intel* TBB* library is used, the error is computed in a -parallel way, namely, predictions for different samples are computed at the same time. -In case of a regression problem, a mean squared error is returned. For -classifications, the result is a misclassification error in percent. diff --git a/modules/ml/doc/k_nearest_neighbors.rst b/modules/ml/doc/k_nearest_neighbors.rst index 05413c77856a1363ac4d1c1d2397a46efdfab6d5..6e1664145013b011318492ce9d52375f8d93ce4a 100644 --- a/modules/ml/doc/k_nearest_neighbors.rst +++ b/modules/ml/doc/k_nearest_neighbors.rst @@ -5,9 +5,9 @@ K-Nearest Neighbors The algorithm caches all training samples and predicts the response for a new sample by analyzing a certain number (**K**) of the nearest neighbors of the sample using voting, calculating weighted sum, and so on. The method is sometimes referred to as "learning by example" because for prediction it looks for the feature vector with a known response that is closest to the given vector. -CvKNearest +KNearest ---------- -.. ocv:class:: CvKNearest : public CvStatModel +.. ocv:class:: KNearest : public StatModel The class implements K-Nearest Neighbors model as described in the beginning of this section. @@ -17,65 +17,32 @@ The class implements K-Nearest Neighbors model as described in the beginning of * (Python) An example of grid search digit recognition using KNearest can be found at opencv_source/samples/python2/digits_adjust.py * (Python) An example of video digit recognition using KNearest can be found at opencv_source/samples/python2/digits_video.py -CvKNearest::CvKNearest +KNearest::create ---------------------- -Default and training constructors. +Creates the empty model -.. ocv:function:: CvKNearest::CvKNearest() +.. ocv:function:: Ptr KNearest::create(const Params& params=Params()) -.. ocv:function:: CvKNearest::CvKNearest( const Mat& trainData, const Mat& responses, const Mat& sampleIdx=Mat(), bool isRegression=false, int max_k=32 ) + :param params: The model parameters: default number of neighbors to use in predict method (in ``KNearest::findNearest`` this number must be passed explicitly) and the flag on whether classification or regression model should be trained. -.. ocv:function:: CvKNearest::CvKNearest( const CvMat* trainData, const CvMat* responses, const CvMat* sampleIdx=0, bool isRegression=false, int max_k=32 ) +The static method creates empty KNearest classifier. It should be then trained using ``train`` method (see ``StatModel::train``). Alternatively, you can load boost model from file using ``StatModel::load(filename)``. -See :ocv:func:`CvKNearest::train` for additional parameters descriptions. -CvKNearest::train ------------------ -Trains the model. - -.. ocv:function:: bool CvKNearest::train( const Mat& trainData, const Mat& responses, const Mat& sampleIdx=Mat(), bool isRegression=false, int maxK=32, bool updateBase=false ) - -.. ocv:function:: bool CvKNearest::train( const CvMat* trainData, const CvMat* responses, const CvMat* sampleIdx=0, bool is_regression=false, int maxK=32, bool updateBase=false ) - -.. ocv:pyfunction:: cv2.KNearest.train(trainData, responses[, sampleIdx[, isRegression[, maxK[, updateBase]]]]) -> retval - - :param isRegression: Type of the problem: ``true`` for regression and ``false`` for classification. - - :param maxK: Number of maximum neighbors that may be passed to the method :ocv:func:`CvKNearest::find_nearest`. - - :param updateBase: Specifies whether the model is trained from scratch (``update_base=false``), or it is updated using the new training data (``update_base=true``). In the latter case, the parameter ``maxK`` must not be larger than the original value. - -The method trains the K-Nearest model. It follows the conventions of the generic :ocv:func:`CvStatModel::train` approach with the following limitations: - -* Only ``CV_ROW_SAMPLE`` data layout is supported. -* Input variables are all ordered. -* Output variables can be either categorical ( ``is_regression=false`` ) or ordered ( ``is_regression=true`` ). -* Variable subsets (``var_idx``) and missing measurements are not supported. - -CvKNearest::find_nearest +KNearest::findNearest ------------------------ Finds the neighbors and predicts responses for input vectors. -.. ocv:function:: float CvKNearest::find_nearest( const Mat& samples, int k, Mat* results=0, const float** neighbors=0, Mat* neighborResponses=0, Mat* dist=0 ) const - -.. ocv:function:: float CvKNearest::find_nearest( const Mat& samples, int k, Mat& results, Mat& neighborResponses, Mat& dists) const +.. ocv:function:: float KNearest::findNearest( InputArray samples, int k, OutputArray results, OutputArray neighborResponses=noArray(), OutputArray dist=noArray() ) const -.. ocv:function:: float CvKNearest::find_nearest( const CvMat* samples, int k, CvMat* results=0, const float** neighbors=0, CvMat* neighborResponses=0, CvMat* dist=0 ) const + :param samples: Input samples stored by rows. It is a single-precision floating-point matrix of `` * k`` size. -.. ocv:pyfunction:: cv2.KNearest.find_nearest(samples, k[, results[, neighborResponses[, dists]]]) -> retval, results, neighborResponses, dists + :param k: Number of used nearest neighbors. Should be greater than 1. + :param results: Vector with results of prediction (regression or classification) for each input sample. It is a single-precision floating-point vector with ```` elements. - :param samples: Input samples stored by rows. It is a single-precision floating-point matrix of :math:`number\_of\_samples \times number\_of\_features` size. + :param neighborResponses: Optional output values for corresponding neighbors. It is a single-precision floating-point matrix of `` * k`` size. - :param k: Number of used nearest neighbors. It must satisfy constraint: :math:`k \le` :ocv:func:`CvKNearest::get_max_k`. - - :param results: Vector with results of prediction (regression or classification) for each input sample. It is a single-precision floating-point vector with ``number_of_samples`` elements. - - :param neighbors: Optional output pointers to the neighbor vectors themselves. It is an array of ``k*samples->rows`` pointers. - - :param neighborResponses: Optional output values for corresponding ``neighbors``. It is a single-precision floating-point matrix of :math:`number\_of\_samples \times k` size. - - :param dist: Optional output distances from the input vectors to the corresponding ``neighbors``. It is a single-precision floating-point matrix of :math:`number\_of\_samples \times k` size. + :param dist: Optional output distances from the input vectors to the corresponding neighbors. It is a single-precision floating-point matrix of `` * k`` size. For each input vector (a row of the matrix ``samples``), the method finds the ``k`` nearest neighbors. In case of regression, the predicted result is a mean value of the particular vector's neighbor responses. In case of classification, the class is determined by voting. @@ -87,110 +54,18 @@ If only a single input vector is passed, all output matrices are optional and th The function is parallelized with the TBB library. -CvKNearest::get_max_k +KNearest::getDefaultK +--------------------- +Returns the default number of neighbors + +.. ocv:function:: int KNearest::getDefaultK() const + +The function returns the default number of neighbors that is used in a simpler ``predict`` method, not ``findNearest``. + +KNearest::setDefaultK --------------------- -Returns the number of maximum neighbors that may be passed to the method :ocv:func:`CvKNearest::find_nearest`. - -.. ocv:function:: int CvKNearest::get_max_k() const - -CvKNearest::get_var_count -------------------------- -Returns the number of used features (variables count). - -.. ocv:function:: int CvKNearest::get_var_count() const - -CvKNearest::get_sample_count ----------------------------- -Returns the total number of train samples. - -.. ocv:function:: int CvKNearest::get_sample_count() const - -CvKNearest::is_regression -------------------------- -Returns type of the problem: ``true`` for regression and ``false`` for classification. - -.. ocv:function:: bool CvKNearest::is_regression() const - - - -The sample below (currently using the obsolete ``CvMat`` structures) demonstrates the use of the k-nearest classifier for 2D point classification: :: - - #include "ml.h" - #include "highgui.h" - - int main( int argc, char** argv ) - { - const int K = 10; - int i, j, k, accuracy; - float response; - int train_sample_count = 100; - CvRNG rng_state = cvRNG(-1); - CvMat* trainData = cvCreateMat( train_sample_count, 2, CV_32FC1 ); - CvMat* trainClasses = cvCreateMat( train_sample_count, 1, CV_32FC1 ); - IplImage* img = cvCreateImage( cvSize( 500, 500 ), 8, 3 ); - float _sample[2]; - CvMat sample = cvMat( 1, 2, CV_32FC1, _sample ); - cvZero( img ); - - CvMat trainData1, trainData2, trainClasses1, trainClasses2; - - // form the training samples - cvGetRows( trainData, &trainData1, 0, train_sample_count/2 ); - cvRandArr( &rng_state, &trainData1, CV_RAND_NORMAL, cvScalar(200,200), cvScalar(50,50) ); - - cvGetRows( trainData, &trainData2, train_sample_count/2, train_sample_count ); - cvRandArr( &rng_state, &trainData2, CV_RAND_NORMAL, cvScalar(300,300), cvScalar(50,50) ); - - cvGetRows( trainClasses, &trainClasses1, 0, train_sample_count/2 ); - cvSet( &trainClasses1, cvScalar(1) ); - - cvGetRows( trainClasses, &trainClasses2, train_sample_count/2, train_sample_count ); - cvSet( &trainClasses2, cvScalar(2) ); - - // learn classifier - CvKNearest knn( trainData, trainClasses, 0, false, K ); - CvMat* nearests = cvCreateMat( 1, K, CV_32FC1); - - for( i = 0; i < img->height; i++ ) - { - for( j = 0; j < img->width; j++ ) - { - sample.data.fl[0] = (float)j; - sample.data.fl[1] = (float)i; - - // estimate the response and get the neighbors' labels - response = knn.find_nearest(&sample,K,0,0,nearests,0); - - // compute the number of neighbors representing the majority - for( k = 0, accuracy = 0; k < K; k++ ) - { - if( nearests->data.fl[k] == response) - accuracy++; - } - // highlight the pixel depending on the accuracy (or confidence) - cvSet2D( img, i, j, response == 1 ? - (accuracy > 5 ? CV_RGB(180,0,0) : CV_RGB(180,120,0)) : - (accuracy > 5 ? CV_RGB(0,180,0) : CV_RGB(120,120,0)) ); - } - } - - // display the original training samples - for( i = 0; i < train_sample_count/2; i++ ) - { - CvPoint pt; - pt.x = cvRound(trainData1.data.fl[i*2]); - pt.y = cvRound(trainData1.data.fl[i*2+1]); - cvCircle( img, pt, 2, CV_RGB(255,0,0), CV_FILLED ); - pt.x = cvRound(trainData2.data.fl[i*2]); - pt.y = cvRound(trainData2.data.fl[i*2+1]); - cvCircle( img, pt, 2, CV_RGB(0,255,0), CV_FILLED ); - } - - cvNamedWindow( "classifier result", 1 ); - cvShowImage( "classifier result", img ); - cvWaitKey(0); - - cvReleaseMat( &trainClasses ); - cvReleaseMat( &trainData ); - return 0; - } +Returns the default number of neighbors + +.. ocv:function:: void KNearest::setDefaultK(int k) + +The function sets the default number of neighbors that is used in a simpler ``predict`` method, not ``findNearest``. diff --git a/modules/ml/doc/ml.rst b/modules/ml/doc/ml.rst index b83e7dedc3f27f3aeff68749a6211ac99c7fa548..86da3ac4ff1371984b6ef8cbd6ea835484731e42 100644 --- a/modules/ml/doc/ml.rst +++ b/modules/ml/doc/ml.rst @@ -15,9 +15,7 @@ Most of the classification and regression algorithms are implemented as C++ clas support_vector_machines decision_trees boosting - gradient_boosted_trees random_trees - ertrees expectation_maximization neural_networks mldata diff --git a/modules/ml/doc/mldata.rst b/modules/ml/doc/mldata.rst index c3092d1490fa5d1908335340827be272f3c7c84f..b710f29a8bbb28472ac9001072174162d561351a 100644 --- a/modules/ml/doc/mldata.rst +++ b/modules/ml/doc/mldata.rst @@ -1,279 +1,126 @@ -MLData +Training Data =================== .. highlight:: cpp -For the machine learning algorithms, the data set is often stored in a file of the ``.csv``-like format. The file contains a table of predictor and response values where each row of the table corresponds to a sample. Missing values are supported. The UC Irvine Machine Learning Repository (http://archive.ics.uci.edu/ml/) provides many data sets stored in such a format to the machine learning community. The class ``MLData`` is implemented to easily load the data for training one of the OpenCV machine learning algorithms. For float values, only the ``'.'`` separator is supported. The table can have a header and in such case the user have to set the number of the header lines to skip them duaring the file reading. +In machine learning algorithms there is notion of training data. Training data includes several components: -CvMLData --------- -.. ocv:class:: CvMLData +* A set of training samples. Each training sample is a vector of values (in Computer Vision it's sometimes referred to as feature vector). Usually all the vectors have the same number of components (features); OpenCV ml module assumes that. Each feature can be ordered (i.e. its values are floating-point numbers that can be compared with each other and strictly ordered, i.e. sorted) or categorical (i.e. its value belongs to a fixed set of values that can be integers, strings etc.). -Class for loading the data from a ``.csv`` file. -:: +* Optional set of responses corresponding to the samples. Training data with no responses is used in unsupervised learning algorithms that learn structure of the supplied data based on distances between different samples. Training data with responses is used in supervised learning algorithms, which learn the function mapping samples to responses. Usually the responses are scalar values, ordered (when we deal with regression problem) or categorical (when we deal with classification problem; in this case the responses are often called "labels"). Some algorithms, most noticeably Neural networks, can handle not only scalar, but also multi-dimensional or vector responses. - class CV_EXPORTS CvMLData - { - public: - CvMLData(); - virtual ~CvMLData(); +* Another optional component is the mask of missing measurements. Most algorithms require all the components in all the training samples be valid, but some other algorithms, such as decision tress, can handle the cases of missing measurements. - int read_csv(const char* filename); +* In the case of classification problem user may want to give different weights to different classes. This is useful, for example, when + * user wants to shift prediction accuracy towards lower false-alarm rate or higher hit-rate. + * user wants to compensate for significantly different amounts of training samples from different classes. - const CvMat* get_values() const; - const CvMat* get_responses(); - const CvMat* get_missing() const; +* In addition to that, each training sample may be given a weight, if user wants the algorithm to pay special attention to certain training samples and adjust the training model accordingly. - void set_response_idx( int idx ); - int get_response_idx() const; +* Also, user may wish not to use the whole training data at once, but rather use parts of it, e.g. to do parameter optimization via cross-validation procedure. +As you can see, training data can have rather complex structure; besides, it may be very big and/or not entirely available, so there is need to make abstraction for this concept. In OpenCV ml there is ``cv::ml::TrainData`` class for that. - void set_train_test_split( const CvTrainTestSplit * spl); - const CvMat* get_train_sample_idx() const; - const CvMat* get_test_sample_idx() const; - void mix_train_and_test_idx(); +TrainData +--------- +.. ocv:class:: TrainData - const CvMat* get_var_idx(); - void change_var_idx( int vi, bool state ); +Class encapsulating training data. Please note that the class only specifies the interface of training data, but not implementation. All the statistical model classes in ml take Ptr. In other words, you can create your own class derived from ``TrainData`` and supply smart pointer to the instance of this class into ``StatModel::train``. - const CvMat* get_var_types(); - void set_var_types( const char* str ); - - int get_var_type( int var_idx ) const; - void change_var_type( int var_idx, int type); - - void set_delimiter( char ch ); - char get_delimiter() const; - - void set_miss_ch( char ch ); - char get_miss_ch() const; - - const std::map& get_class_labels_map() const; - - protected: - ... - }; - -CvMLData::read_csv ------------------- -Reads the data set from a ``.csv``-like ``filename`` file and stores all read values in a matrix. +TrainData::loadFromCSV +---------------------- +Reads the dataset from a .csv file and returns the ready-to-use training data. -.. ocv:function:: int CvMLData::read_csv(const char* filename) +.. ocv:function:: Ptr loadFromCSV(const String& filename, int headerLineCount, int responseStartIdx=-1, int responseEndIdx=-1, const String& varTypeSpec=String(), char delimiter=',', char missch='?') :param filename: The input file name -While reading the data, the method tries to define the type of variables (predictors and responses): ordered or categorical. If a value of the variable is not numerical (except for the label for a missing value), the type of the variable is set to ``CV_VAR_CATEGORICAL``. If all existing values of the variable are numerical, the type of the variable is set to ``CV_VAR_ORDERED``. So, the default definition of variables types works correctly for all cases except the case of a categorical variable with numerical class labels. In this case, the type ``CV_VAR_ORDERED`` is set. You should change the type to ``CV_VAR_CATEGORICAL`` using the method :ocv:func:`CvMLData::change_var_type`. For categorical variables, a common map is built to convert a string class label to the numerical class label. Use :ocv:func:`CvMLData::get_class_labels_map` to obtain this map. + :param headerLineCount: The number of lines in the beginning to skip; besides the header, the function also skips empty lines and lines staring with '#' -Also, when reading the data, the method constructs the mask of missing values. For example, values are equal to `'?'`. + :param responseStartIdx: Index of the first output variable. If -1, the function considers the last variable as the response -CvMLData::get_values --------------------- -Returns a pointer to the matrix of predictors and response values + :param responseEndIdx: Index of the last output variable + 1. If -1, then there is single response variable at ``responseStartIdx``. -.. ocv:function:: const CvMat* CvMLData::get_values() const + :param varTypeSpec: The optional text string that specifies the variables' types. It has the format ``ord[n1-n2,n3,n4-n5,...]cat[n6,n7-n8,...]``. That is, variables from n1 to n2 (inclusive range), n3, n4 to n5 ... are considered ordered and n6, n7 to n8 ... are considered as categorical. The range [n1..n2] + [n3] + [n4..n5] + ... + [n6] + [n7..n8] should cover all the variables. If varTypeSpec is not specified, then algorithm uses the following rules: + 1. all input variables are considered ordered by default. If some column contains has non-numerical values, e.g. 'apple', 'pear', 'apple', 'apple', 'mango', the corresponding variable is considered categorical. + 2. if there are several output variables, they are all considered as ordered. Error is reported when non-numerical values are used. + 3. if there is a single output variable, then if its values are non-numerical or are all integers, then it's considered categorical. Otherwise, it's considered ordered. -The method returns a pointer to the matrix of predictor and response ``values`` or ``0`` if the data has not been loaded from the file yet. + :param delimiter: The character used to separate values in each line. -The row count of this matrix equals the sample count. The column count equals predictors ``+ 1`` for the response (if exists) count. This means that each row of the matrix contains values of one sample predictor and response. The matrix type is ``CV_32FC1``. + :param missch: The character used to specify missing measurements. It should not be a digit. Although it's a non-numerical value, it surely does not affect the decision of whether the variable ordered or categorical. -CvMLData::get_responses ------------------------ -Returns a pointer to the matrix of response values +TrainData::create +----------------- +Creates training data from in-memory arrays. -.. ocv:function:: const CvMat* CvMLData::get_responses() +.. ocv:function:: Ptr create(InputArray samples, int layout, InputArray responses, InputArray varIdx=noArray(), InputArray sampleIdx=noArray(), InputArray sampleWeights=noArray(), InputArray varType=noArray()) -The method returns a pointer to the matrix of response values or throws an exception if the data has not been loaded from the file yet. + :param samples: matrix of samples. It should have ``CV_32F`` type. -This is a single-column matrix of the type ``CV_32FC1``. Its row count is equal to the sample count, one column and . + :param layout: it's either ``ROW_SAMPLE``, which means that each training sample is a row of ``samples``, or ``COL_SAMPLE``, which means that each training sample occupies a column of ``samples``. -CvMLData::get_missing ---------------------- -Returns a pointer to the mask matrix of missing values + :param responses: matrix of responses. If the responses are scalar, they should be stored as a single row or as a single column. The matrix should have type ``CV_32F`` or ``CV_32S`` (in the former case the responses are considered as ordered by default; in the latter case - as categorical) -.. ocv:function:: const CvMat* CvMLData::get_missing() const + :param varIdx: vector specifying which variables to use for training. It can be an integer vector (``CV_32S``) containing 0-based variable indices or byte vector (``CV_8U``) containing a mask of active variables. -The method returns a pointer to the mask matrix of missing values or throws an exception if the data has not been loaded from the file yet. + :param sampleIdx: vector specifying which samples to use for training. It can be an integer vector (``CV_32S``) containing 0-based sample indices or byte vector (``CV_8U``) containing a mask of training samples. -This matrix has the same size as the ``values`` matrix (see :ocv:func:`CvMLData::get_values`) and the type ``CV_8UC1``. + :param sampleWeights: optional vector with weights for each sample. It should have ``CV_32F`` type. -CvMLData::set_response_idx --------------------------- -Specifies index of response column in the data matrix - -.. ocv:function:: void CvMLData::set_response_idx( int idx ) - -The method sets the index of a response column in the ``values`` matrix (see :ocv:func:`CvMLData::get_values`) or throws an exception if the data has not been loaded from the file yet. + :param varType: optional vector of type ``CV_8U`` and size + , containing types of each input and output variable. The ordered variables are denoted by value ``VAR_ORDERED``, and categorical - by ``VAR_CATEGORICAL``. -The old response columns become predictors. If ``idx < 0``, there is no response. -CvMLData::get_response_idx +TrainData::getTrainSamples -------------------------- -Returns index of the response column in the loaded data matrix - -.. ocv:function:: int CvMLData::get_response_idx() const - -The method returns the index of a response column in the ``values`` matrix (see :ocv:func:`CvMLData::get_values`) or throws an exception if the data has not been loaded from the file yet. - -If ``idx < 0``, there is no response. - - -CvMLData::set_train_test_split ------------------------------- -Divides the read data set into two disjoint training and test subsets. - -.. ocv:function:: void CvMLData::set_train_test_split( const CvTrainTestSplit * spl ) - -This method sets parameters for such a split using ``spl`` (see :ocv:class:`CvTrainTestSplit`) or throws an exception if the data has not been loaded from the file yet. - -CvMLData::get_train_sample_idx ------------------------------- -Returns the matrix of sample indices for a training subset - -.. ocv:function:: const CvMat* CvMLData::get_train_sample_idx() const - -The method returns the matrix of sample indices for a training subset. This is a single-row matrix of the type ``CV_32SC1``. If data split is not set, the method returns ``0``. If the data has not been loaded from the file yet, an exception is thrown. - -CvMLData::get_test_sample_idx ------------------------------ -Returns the matrix of sample indices for a testing subset - -.. ocv:function:: const CvMat* CvMLData::get_test_sample_idx() const - - -CvMLData::mix_train_and_test_idx --------------------------------- -Mixes the indices of training and test samples - -.. ocv:function:: void CvMLData::mix_train_and_test_idx() - -The method shuffles the indices of training and test samples preserving sizes of training and test subsets if the data split is set by :ocv:func:`CvMLData::get_values`. If the data has not been loaded from the file yet, an exception is thrown. - -CvMLData::get_var_idx ---------------------- -Returns the indices of the active variables in the data matrix - -.. ocv:function:: const CvMat* CvMLData::get_var_idx() - -The method returns the indices of variables (columns) used in the ``values`` matrix (see :ocv:func:`CvMLData::get_values`). - -It returns ``0`` if the used subset is not set. It throws an exception if the data has not been loaded from the file yet. Returned matrix is a single-row matrix of the type ``CV_32SC1``. Its column count is equal to the size of the used variable subset. - -CvMLData::change_var_idx ------------------------- -Enables or disables particular variable in the loaded data - -.. ocv:function:: void CvMLData::change_var_idx( int vi, bool state ) - -By default, after reading the data set all variables in the ``values`` matrix (see :ocv:func:`CvMLData::get_values`) are used. But you may want to use only a subset of variables and include/exclude (depending on ``state`` value) a variable with the ``vi`` index from the used subset. If the data has not been loaded from the file yet, an exception is thrown. - -CvMLData::get_var_types ------------------------ -Returns a matrix of the variable types. - -.. ocv:function:: const CvMat* CvMLData::get_var_types() - -The function returns a single-row matrix of the type ``CV_8UC1``, where each element is set to either ``CV_VAR_ORDERED`` or ``CV_VAR_CATEGORICAL``. The number of columns is equal to the number of variables. If data has not been loaded from file yet an exception is thrown. - -CvMLData::set_var_types ------------------------ -Sets the variables types in the loaded data. - -.. ocv:function:: void CvMLData::set_var_types( const char* str ) - -In the string, a variable type is followed by a list of variables indices. For example: ``"ord[0-17],cat[18]"``, ``"ord[0,2,4,10-12], cat[1,3,5-9,13,14]"``, ``"cat"`` (all variables are categorical), ``"ord"`` (all variables are ordered). - -CvMLData::get_header_lines_number ---------------------------------- -Returns a number of the table header lines. - -.. ocv:function:: int CvMLData::get_header_lines_number() const - -CvMLData::set_header_lines_number ---------------------------------- -Sets a number of the table header lines. - -.. ocv:function:: void CvMLData::set_header_lines_number( int n ) - -By default it is supposed that the table does not have a header, i.e. it contains only the data. - -CvMLData::get_var_type ----------------------- -Returns type of the specified variable - -.. ocv:function:: int CvMLData::get_var_type( int var_idx ) const - -The method returns the type of a variable by the index ``var_idx`` ( ``CV_VAR_ORDERED`` or ``CV_VAR_CATEGORICAL``). - -CvMLData::change_var_type -------------------------- -Changes type of the specified variable - -.. ocv:function:: void CvMLData::change_var_type( int var_idx, int type) - -The method changes type of variable with index ``var_idx`` from existing type to ``type`` ( ``CV_VAR_ORDERED`` or ``CV_VAR_CATEGORICAL``). +Returns matrix of train samples -CvMLData::set_delimiter ------------------------ -Sets the delimiter in the file used to separate input numbers +.. ocv:function:: Mat TrainData::getTrainSamples(int layout=ROW_SAMPLE, bool compressSamples=true, bool compressVars=true) const -.. ocv:function:: void CvMLData::set_delimiter( char ch ) + :param layout: The requested layout. If it's different from the initial one, the matrix is transposed. -The method sets the delimiter for variables in a file. For example: ``','`` (default), ``';'``, ``' '`` (space), or other characters. The floating-point separator ``'.'`` is not allowed. + :param compressSamples: if true, the function returns only the training samples (specified by sampleIdx) -CvMLData::get_delimiter ------------------------ -Returns the currently used delimiter character. + :param compressVars: if true, the function returns the shorter training samples, containing only the active variables. -.. ocv:function:: char CvMLData::get_delimiter() const +In current implementation the function tries to avoid physical data copying and returns the matrix stored inside TrainData (unless the transposition or compression is needed). -CvMLData::set_miss_ch ---------------------- -Sets the character used to specify missing values +TrainData::getTrainResponses +---------------------------- +Returns the vector of responses -.. ocv:function:: void CvMLData::set_miss_ch( char ch ) +.. ocv:function:: Mat TrainData::getTrainResponses() const -The method sets the character used to specify missing values. For example: ``'?'`` (default), ``'-'``. The floating-point separator ``'.'`` is not allowed. +The function returns ordered or the original categorical responses. Usually it's used in regression algorithms. -CvMLData::get_miss_ch ---------------------- -Returns the currently used missing value character. -.. ocv:function:: char CvMLData::get_miss_ch() const +TrainData::getClassLabels +---------------------------- +Returns the vector of class labels -CvMLData::get_class_labels_map -------------------------------- -Returns a map that converts strings to labels. +.. ocv:function:: Mat TrainData::getClassLabels() const -.. ocv:function:: const std::map& CvMLData::get_class_labels_map() const +The function returns vector of unique labels occurred in the responses. -The method returns a map that converts string class labels to the numerical class labels. It can be used to get an original class label as in a file. -CvTrainTestSplit ----------------- -.. ocv:struct:: CvTrainTestSplit +TrainData::getTrainNormCatResponses +----------------------------------- +Returns the vector of normalized categorical responses -Structure setting the split of a data set read by :ocv:class:`CvMLData`. -:: +.. ocv:function:: Mat TrainData::getTrainNormCatResponses() const - struct CvTrainTestSplit - { - CvTrainTestSplit(); - CvTrainTestSplit( int train_sample_count, bool mix = true); - CvTrainTestSplit( float train_sample_portion, bool mix = true); +The function returns vector of responses. Each response is integer from 0 to -1. The actual label value can be retrieved then from the class label vector, see ``TrainData::getClassLabels``. - union - { - int count; - float portion; - } train_sample_part; - int train_sample_part_mode; +TrainData::setTrainTestSplitRatio +----------------------------------- +Splits the training data into the training and test parts - bool mix; - }; +.. ocv:function:: void TrainData::setTrainTestSplitRatio(double ratio, bool shuffle=true) -There are two ways to construct a split: +The function selects a subset of specified relative size and then returns it as the training set. If the function is not called, all the data is used for training. Please, note that for each of ``TrainData::getTrain*`` there is corresponding ``TrainData::getTest*``, so that the test subset can be retrieved and processed as well. -* Set the training sample count (subset size) ``train_sample_count``. Other existing samples are located in a test subset. -* Set a training sample portion in ``[0,..1]``. The flag ``mix`` is used to mix training and test samples indices when the split is set. Otherwise, the data set is split in the storing order: the first part of samples of a given size is a training subset, the second part is a test subset. +Other methods +------------- +The class includes many other methods that can be used to access normalized categorical input variables, access training data by parts, so that does not have to fit into the memory etc. diff --git a/modules/ml/doc/neural_networks.rst b/modules/ml/doc/neural_networks.rst index 776bf243bd92178f4fd8b1ee52ea52b1a88538a1..557ef82c3553304ba08d07a9dcd8054d7c6b2daf 100644 --- a/modules/ml/doc/neural_networks.rst +++ b/modules/ml/doc/neural_networks.rst @@ -29,17 +29,17 @@ In other words, given the outputs Different activation functions may be used. ML implements three standard functions: * - Identity function ( ``CvANN_MLP::IDENTITY`` ): + Identity function ( ``ANN_MLP::IDENTITY`` ): :math:`f(x)=x` * - Symmetrical sigmoid ( ``CvANN_MLP::SIGMOID_SYM`` ): + Symmetrical sigmoid ( ``ANN_MLP::SIGMOID_SYM`` ): :math:`f(x)=\beta*(1-e^{-\alpha x})/(1+e^{-\alpha x}` ), which is the default choice for MLP. The standard sigmoid with :math:`\beta =1, \alpha =1` is shown below: .. image:: pics/sigmoid_bipolar.png * - Gaussian function ( ``CvANN_MLP::GAUSSIAN`` ): + Gaussian function ( ``ANN_MLP::GAUSSIAN`` ): :math:`f(x)=\beta e^{-\alpha x*x}` , which is not completely supported at the moment. In ML, all the neurons have the same activation functions, with the same free parameters ( @@ -95,60 +95,90 @@ The second (default) one is a batch RPROP algorithm. .. [RPROP93] M. Riedmiller and H. Braun, *A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm*, Proc. ICNN, San Francisco (1993). -CvANN_MLP_TrainParams +ANN_MLP::Params --------------------- -.. ocv:struct:: CvANN_MLP_TrainParams +.. ocv:class:: ANN_MLP::Params - Parameters of the MLP training algorithm. You can initialize the structure by a constructor or the individual parameters can be adjusted after the structure is created. + Parameters of the MLP and of the training algorithm. You can initialize the structure by a constructor or the individual parameters can be adjusted after the structure is created. + + The network structure: + + .. ocv:member:: Mat layerSizes + + The number of elements in each layer of network. The very first element specifies the number of elements in the input layer. The last element - number of elements in the output layer. + + .. ocv:member:: int activateFunc + + The activation function. Currently the only fully supported activation function is ``ANN_MLP::SIGMOID_SYM``. + + .. ocv:member:: double fparam1 + + The first parameter of activation function, 0 by default. + + .. ocv:member:: double fparam2 + + The second parameter of the activation function, 0 by default. + + .. note:: + + If you are using the default ``ANN_MLP::SIGMOID_SYM`` activation function with the default parameter values fparam1=0 and fparam2=0 then the function used is y = 1.7159*tanh(2/3 * x), so the output will range from [-1.7159, 1.7159], instead of [0,1]. The back-propagation algorithm parameters: - .. ocv:member:: double bp_dw_scale + .. ocv:member:: double bpDWScale Strength of the weight gradient term. The recommended value is about 0.1. - .. ocv:member:: double bp_moment_scale + .. ocv:member:: double bpMomentScale Strength of the momentum term (the difference between weights on the 2 previous iterations). This parameter provides some inertia to smooth the random fluctuations of the weights. It can vary from 0 (the feature is disabled) to 1 and beyond. The value 0.1 or so is good enough The RPROP algorithm parameters (see [RPROP93]_ for details): - .. ocv:member:: double rp_dw0 + .. ocv:member:: double prDW0 Initial value :math:`\Delta_0` of update-values :math:`\Delta_{ij}`. - .. ocv:member:: double rp_dw_plus + .. ocv:member:: double rpDWPlus Increase factor :math:`\eta^+`. It must be >1. - .. ocv:member:: double rp_dw_minus + .. ocv:member:: double rpDWMinus Decrease factor :math:`\eta^-`. It must be <1. - .. ocv:member:: double rp_dw_min + .. ocv:member:: double rpDWMin Update-values lower limit :math:`\Delta_{min}`. It must be positive. - .. ocv:member:: double rp_dw_max + .. ocv:member:: double rpDWMax Update-values upper limit :math:`\Delta_{max}`. It must be >1. -CvANN_MLP_TrainParams::CvANN_MLP_TrainParams +ANN_MLP::Params::Params -------------------------------------------- -The constructors. +Construct the parameter structure + +.. ocv:function:: ANN_MLP::Params() -.. ocv:function:: CvANN_MLP_TrainParams::CvANN_MLP_TrainParams() +.. ocv:function:: ANN_MLP::Params::Params( const Mat& layerSizes, int activateFunc, double fparam1, double fparam2, TermCriteria termCrit, int trainMethod, double param1, double param2=0 ) -.. ocv:function:: CvANN_MLP_TrainParams::CvANN_MLP_TrainParams( CvTermCriteria term_crit, int train_method, double param1, double param2=0 ) + :param layerSizes: Integer vector specifying the number of neurons in each layer including the input and output layers. - :param term_crit: Termination criteria of the training algorithm. You can specify the maximum number of iterations (``max_iter``) and/or how much the error could change between the iterations to make the algorithm continue (``epsilon``). + :param activateFunc: Parameter specifying the activation function for each neuron: one of ``ANN_MLP::IDENTITY``, ``ANN_MLP::SIGMOID_SYM``, and ``ANN_MLP::GAUSSIAN``. + + :param fparam1: The first parameter of the activation function, :math:`\alpha`. See the formulas in the introduction section. + + :param fparam2: The second parameter of the activation function, :math:`\beta`. See the formulas in the introduction section. + + :param termCrit: Termination criteria of the training algorithm. You can specify the maximum number of iterations (``maxCount``) and/or how much the error could change between the iterations to make the algorithm continue (``epsilon``). :param train_method: Training method of the MLP. Possible values are: - * **CvANN_MLP_TrainParams::BACKPROP** The back-propagation algorithm. + * **ANN_MLP_TrainParams::BACKPROP** The back-propagation algorithm. - * **CvANN_MLP_TrainParams::RPROP** The RPROP algorithm. + * **ANN_MLP_TrainParams::RPROP** The RPROP algorithm. :param param1: Parameter of the training method. It is ``rp_dw0`` for ``RPROP`` and ``bp_dw_scale`` for ``BACKPROP``. @@ -158,126 +188,54 @@ By default the RPROP algorithm is used: :: - CvANN_MLP_TrainParams::CvANN_MLP_TrainParams() + ANN_MLP_TrainParams::ANN_MLP_TrainParams() { - term_crit = cvTermCriteria( CV_TERMCRIT_ITER + CV_TERMCRIT_EPS, 1000, 0.01 ); + layerSizes = Mat(); + activateFun = SIGMOID_SYM; + fparam1 = fparam2 = 0; + term_crit = TermCriteria( TermCriteria::MAX_ITER + TermCriteria::EPS, 1000, 0.01 ); train_method = RPROP; - bp_dw_scale = bp_moment_scale = 0.1; - rp_dw0 = 0.1; rp_dw_plus = 1.2; rp_dw_minus = 0.5; - rp_dw_min = FLT_EPSILON; rp_dw_max = 50.; + bpDWScale = bpMomentScale = 0.1; + rpDW0 = 0.1; rpDWPlus = 1.2; rpDWMinus = 0.5; + rpDWMin = FLT_EPSILON; rpDWMax = 50.; } -CvANN_MLP +ANN_MLP --------- -.. ocv:class:: CvANN_MLP : public CvStatModel +.. ocv:class:: ANN_MLP : public StatModel MLP model. -Unlike many other models in ML that are constructed and trained at once, in the MLP model these steps are separated. First, a network with the specified topology is created using the non-default constructor or the method :ocv:func:`CvANN_MLP::create`. All the weights are set to zeros. Then, the network is trained using a set of input and output vectors. The training procedure can be repeated more than once, that is, the weights can be adjusted based on the new training data. +Unlike many other models in ML that are constructed and trained at once, in the MLP model these steps are separated. First, a network with the specified topology is created using the non-default constructor or the method :ocv:func:`ANN_MLP::create`. All the weights are set to zeros. Then, the network is trained using a set of input and output vectors. The training procedure can be repeated more than once, that is, the weights can be adjusted based on the new training data. -CvANN_MLP::CvANN_MLP +ANN_MLP::create -------------------- -The constructors. - -.. ocv:function:: CvANN_MLP::CvANN_MLP() - -.. ocv:function:: CvANN_MLP::CvANN_MLP( const CvMat* layerSizes, int activateFunc=CvANN_MLP::SIGMOID_SYM, double fparam1=0, double fparam2=0 ) - -.. ocv:pyfunction:: cv2.ANN_MLP([layerSizes[, activateFunc[, fparam1[, fparam2]]]]) -> - -The advanced constructor allows to create MLP with the specified topology. See :ocv:func:`CvANN_MLP::create` for details. - -CvANN_MLP::create ------------------ -Constructs MLP with the specified topology. - -.. ocv:function:: void CvANN_MLP::create( const Mat& layerSizes, int activateFunc=CvANN_MLP::SIGMOID_SYM, double fparam1=0, double fparam2=0 ) - -.. ocv:function:: void CvANN_MLP::create( const CvMat* layerSizes, int activateFunc=CvANN_MLP::SIGMOID_SYM, double fparam1=0, double fparam2=0 ) - -.. ocv:pyfunction:: cv2.ANN_MLP.create(layerSizes[, activateFunc[, fparam1[, fparam2]]]) -> None - - :param layerSizes: Integer vector specifying the number of neurons in each layer including the input and output layers. - - :param activateFunc: Parameter specifying the activation function for each neuron: one of ``CvANN_MLP::IDENTITY``, ``CvANN_MLP::SIGMOID_SYM``, and ``CvANN_MLP::GAUSSIAN``. - - :param fparam1: Free parameter of the activation function, :math:`\alpha`. See the formulas in the introduction section. - - :param fparam2: Free parameter of the activation function, :math:`\beta`. See the formulas in the introduction section. - -The method creates an MLP network with the specified topology and assigns the same activation function to all the neurons. - -CvANN_MLP::train ----------------- -Trains/updates MLP. - -.. ocv:function:: int CvANN_MLP::train( const Mat& inputs, const Mat& outputs, const Mat& sampleWeights, const Mat& sampleIdx=Mat(), CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(), int flags=0 ) - -.. ocv:function:: int CvANN_MLP::train( const CvMat* inputs, const CvMat* outputs, const CvMat* sampleWeights, const CvMat* sampleIdx=0, CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(), int flags=0 ) - -.. ocv:pyfunction:: cv2.ANN_MLP.train(inputs, outputs, sampleWeights[, sampleIdx[, params[, flags]]]) -> retval - - :param inputs: Floating-point matrix of input vectors, one vector per row. - - :param outputs: Floating-point matrix of the corresponding output vectors, one vector per row. - - :param sampleWeights: (RPROP only) Optional floating-point vector of weights for each sample. Some samples may be more important than others for training. You may want to raise the weight of certain classes to find the right balance between hit-rate and false-alarm rate, and so on. - - :param sampleIdx: Optional integer vector indicating the samples (rows of ``inputs`` and ``outputs``) that are taken into account. - - :param params: Training parameters. See the :ocv:class:`CvANN_MLP_TrainParams` description. - - :param flags: Various parameters to control the training algorithm. A combination of the following parameters is possible: - - * **UPDATE_WEIGHTS** Algorithm updates the network weights, rather than computes them from scratch. In the latter case the weights are initialized using the Nguyen-Widrow algorithm. - - * **NO_INPUT_SCALE** Algorithm does not normalize the input vectors. If this flag is not set, the training algorithm normalizes each input feature independently, shifting its mean value to 0 and making the standard deviation equal to 1. If the network is assumed to be updated frequently, the new training data could be much different from original one. In this case, you should take care of proper normalization. - - * **NO_OUTPUT_SCALE** Algorithm does not normalize the output vectors. If the flag is not set, the training algorithm normalizes each output feature independently, by transforming it to the certain range depending on the used activation function. - -This method applies the specified training algorithm to computing/adjusting the network weights. It returns the number of done iterations. - -The RPROP training algorithm is parallelized with the TBB library. - -If you are using the default ``cvANN_MLP::SIGMOID_SYM`` activation function then the output should be in the range [-1,1], instead of [0,1], for optimal results. - -CvANN_MLP::predict ------------------- -Predicts responses for input samples. - -.. ocv:function:: float CvANN_MLP::predict( const Mat& inputs, Mat& outputs ) const - -.. ocv:function:: float CvANN_MLP::predict( const CvMat* inputs, CvMat* outputs ) const - -.. ocv:pyfunction:: cv2.ANN_MLP.predict(inputs[, outputs]) -> retval, outputs +Creates empty model - :param inputs: Input samples. +.. ocv:function:: Ptr ANN_MLP::create(const Params& params=Params()) - :param outputs: Predicted responses for corresponding samples. +Use ``StatModel::train`` to train the model, ``StatModel::train(traindata, params)`` to create and train the model, ``StatModel::load(filename)`` to load the pre-trained model. Note that the train method has optional flags, and the following flags are handled by ``ANN_MLP``: -The method returns a dummy value which should be ignored. + * **UPDATE_WEIGHTS** Algorithm updates the network weights, rather than computes them from scratch. In the latter case the weights are initialized using the Nguyen-Widrow algorithm. -If you are using the default ``cvANN_MLP::SIGMOID_SYM`` activation function with the default parameter values fparam1=0 and fparam2=0 then the function used is y = 1.7159*tanh(2/3 * x), so the output will range from [-1.7159, 1.7159], instead of [0,1]. + * **NO_INPUT_SCALE** Algorithm does not normalize the input vectors. If this flag is not set, the training algorithm normalizes each input feature independently, shifting its mean value to 0 and making the standard deviation equal to 1. If the network is assumed to be updated frequently, the new training data could be much different from original one. In this case, you should take care of proper normalization. -CvANN_MLP::get_layer_count --------------------------- -Returns the number of layers in the MLP. + * **NO_OUTPUT_SCALE** Algorithm does not normalize the output vectors. If the flag is not set, the training algorithm normalizes each output feature independently, by transforming it to the certain range depending on the used activation function. -.. ocv:function:: int CvANN_MLP::get_layer_count() -CvANN_MLP::get_layer_sizes --------------------------- -Returns numbers of neurons in each layer of the MLP. +ANN_MLP::setParams +------------------- +Sets the new network parameters -.. ocv:function:: const CvMat* CvANN_MLP::get_layer_sizes() +.. ocv:function:: void ANN_MLP::setParams(const Params& params) -The method returns the integer vector specifying the number of neurons in each layer including the input and output layers of the MLP. + :param params: The new parameters -CvANN_MLP::get_weights ----------------------- -Returns neurons weights of the particular layer. +The existing network, if any, will be destroyed and new empty one will be created. It should be re-trained after that. -.. ocv:function:: double* CvANN_MLP::get_weights(int layer) +ANN_MLP::getParams +------------------- +Retrieves the current network parameters - :param layer: Index of the particular layer. +.. ocv:function:: Params ANN_MLP::getParams() const diff --git a/modules/ml/doc/normal_bayes_classifier.rst b/modules/ml/doc/normal_bayes_classifier.rst index dbd6ae229cc5b556eb90b9da18242a71770e94fb..e3aba21c3256e42fad4ab5d7dadeeb059b09d678 100644 --- a/modules/ml/doc/normal_bayes_classifier.rst +++ b/modules/ml/doc/normal_bayes_classifier.rst @@ -9,55 +9,26 @@ This simple classification model assumes that feature vectors from each class ar .. [Fukunaga90] K. Fukunaga. *Introduction to Statistical Pattern Recognition*. second ed., New York: Academic Press, 1990. -CvNormalBayesClassifier +NormalBayesClassifier ----------------------- -.. ocv:class:: CvNormalBayesClassifier : public CvStatModel +.. ocv:class:: NormalBayesClassifier : public StatModel Bayes classifier for normally distributed data. -CvNormalBayesClassifier::CvNormalBayesClassifier ------------------------------------------------- -Default and training constructors. +NormalBayesClassifier::create +----------------------------- +Creates empty model -.. ocv:function:: CvNormalBayesClassifier::CvNormalBayesClassifier() +.. ocv:function:: Ptr NormalBayesClassifier::create(const NormalBayesClassifier::Params& params=Params()) -.. ocv:function:: CvNormalBayesClassifier::CvNormalBayesClassifier( const Mat& trainData, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat() ) + :param params: The model parameters. There is none so far, the structure is used as a placeholder for possible extensions. -.. ocv:function:: CvNormalBayesClassifier::CvNormalBayesClassifier( const CvMat* trainData, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0 ) +Use ``StatModel::train`` to train the model, ``StatModel::train(traindata, params)`` to create and train the model, ``StatModel::load(filename)`` to load the pre-trained model. -.. ocv:pyfunction:: cv2.NormalBayesClassifier([trainData, responses[, varIdx[, sampleIdx]]]) -> - -The constructors follow conventions of :ocv:func:`CvStatModel::CvStatModel`. See :ocv:func:`CvStatModel::train` for parameters descriptions. - -CvNormalBayesClassifier::train ------------------------------- -Trains the model. - -.. ocv:function:: bool CvNormalBayesClassifier::train( const Mat& trainData, const Mat& responses, const Mat& varIdx = Mat(), const Mat& sampleIdx=Mat(), bool update=false ) - -.. ocv:function:: bool CvNormalBayesClassifier::train( const CvMat* trainData, const CvMat* responses, const CvMat* varIdx = 0, const CvMat* sampleIdx=0, bool update=false ) - -.. ocv:pyfunction:: cv2.NormalBayesClassifier.train(trainData, responses[, varIdx[, sampleIdx[, update]]]) -> retval - - :param update: Identifies whether the model should be trained from scratch (``update=false``) or should be updated using the new training data (``update=true``). - -The method trains the Normal Bayes classifier. It follows the conventions of the generic :ocv:func:`CvStatModel::train` approach with the following limitations: - -* Only ``CV_ROW_SAMPLE`` data layout is supported. -* Input variables are all ordered. -* Output variable is categorical , which means that elements of ``responses`` must be integer numbers, though the vector may have the ``CV_32FC1`` type. -* Missing measurements are not supported. - -CvNormalBayesClassifier::predict --------------------------------- +NormalBayesClassifier::predictProb +---------------------------------- Predicts the response for sample(s). -.. ocv:function:: float CvNormalBayesClassifier::predict( const Mat& samples, Mat* results=0, Mat* results_prob=0 ) const - -.. ocv:function:: float CvNormalBayesClassifier::predict( const CvMat* samples, CvMat* results=0, CvMat* results_prob=0 ) const - -.. ocv:pyfunction:: cv2.NormalBayesClassifier.predict(samples) -> retval, results - -The method estimates the most probable classes for input vectors. Input vectors (one or more) are stored as rows of the matrix ``samples``. In case of multiple input vectors, there should be one output vector ``results``. The predicted class for a single input vector is returned by the method. The vector ``results_prob`` contains the output probabilities coresponding to each element of ``result``. +.. ocv:function:: float NormalBayesClassifier::predictProb( InputArray inputs, OutputArray outputs, OutputArray outputProbs, int flags=0 ) const -The function is parallelized with the TBB library. +The method estimates the most probable classes for input vectors. Input vectors (one or more) are stored as rows of the matrix ``inputs``. In case of multiple input vectors, there should be one output vector ``outputs``. The predicted class for a single input vector is returned by the method. The vector ``outputProbs`` contains the output probabilities corresponding to each element of ``result``. diff --git a/modules/ml/doc/random_trees.rst b/modules/ml/doc/random_trees.rst index 8d7911d3687dfe9848881ca7dd2efe4638b4d3e5..602786d57e48e00018f5e67ac26c450babef486f 100644 --- a/modules/ml/doc/random_trees.rst +++ b/modules/ml/doc/random_trees.rst @@ -40,179 +40,64 @@ For the random trees usage example, please, see letter_recog.cpp sample in OpenC * And other articles from the web site http://www.stat.berkeley.edu/users/breiman/RandomForests/cc_home.htm -CvRTParams ----------- -.. ocv:struct:: CvRTParams : public CvDTreeParams +RTrees::Params +-------------- +.. ocv:struct:: RTrees::Params : public DTrees::Params Training parameters of random trees. The set of training parameters for the forest is a superset of the training parameters for a single tree. However, random trees do not need all the functionality/features of decision trees. Most noticeably, the trees are not pruned, so the cross-validation parameters are not used. -CvRTParams::CvRTParams: +RTrees::Params::Params ----------------------- -The constructors. +The constructors -.. ocv:function:: CvRTParams::CvRTParams() +.. ocv:function:: RTrees::Params::Params() -.. ocv:function:: CvRTParams::CvRTParams( int max_depth, int min_sample_count, float regression_accuracy, bool use_surrogates, int max_categories, const float* priors, bool calc_var_importance, int nactive_vars, int max_num_of_trees_in_the_forest, float forest_accuracy, int termcrit_type ) +.. ocv:function:: RTrees::Params::Params( int maxDepth, int minSampleCount, double regressionAccuracy, bool useSurrogates, int maxCategories, const Mat& priors, bool calcVarImportance, int nactiveVars, TermCriteria termCrit ) - :param max_depth: the depth of the tree. A low value will likely underfit and conversely a high value will likely overfit. The optimal value can be obtained using cross validation or other suitable methods. + :param maxDepth: the depth of the tree. A low value will likely underfit and conversely a high value will likely overfit. The optimal value can be obtained using cross validation or other suitable methods. - :param min_sample_count: minimum samples required at a leaf node for it to be split. A reasonable value is a small percentage of the total data e.g. 1%. + :param minSampleCount: minimum samples required at a leaf node for it to be split. A reasonable value is a small percentage of the total data e.g. 1%. - :param max_categories: Cluster possible values of a categorical variable into ``K`` :math:`\leq` ``max_categories`` clusters to find a suboptimal split. If a discrete variable, on which the training procedure tries to make a split, takes more than ``max_categories`` values, the precise best subset estimation may take a very long time because the algorithm is exponential. Instead, many decision trees engines (including ML) try to find sub-optimal split in this case by clustering all the samples into ``max_categories`` clusters that is some categories are merged together. The clustering is applied only in ``n``>2-class classification problems for categorical variables with ``N > max_categories`` possible values. In case of regression and 2-class classification the optimal split can be found efficiently without employing clustering, thus the parameter is not used in these cases. + :param maxCategories: Cluster possible values of a categorical variable into ``K <= maxCategories`` clusters to find a suboptimal split. If a discrete variable, on which the training procedure tries to make a split, takes more than ``max_categories`` values, the precise best subset estimation may take a very long time because the algorithm is exponential. Instead, many decision trees engines (including ML) try to find sub-optimal split in this case by clustering all the samples into ``maxCategories`` clusters that is some categories are merged together. The clustering is applied only in ``n``>2-class classification problems for categorical variables with ``N > max_categories`` possible values. In case of regression and 2-class classification the optimal split can be found efficiently without employing clustering, thus the parameter is not used in these cases. - :param calc_var_importance: If true then variable importance will be calculated and then it can be retrieved by :ocv:func:`CvRTrees::get_var_importance`. + :param calcVarImportance: If true then variable importance will be calculated and then it can be retrieved by ``RTrees::getVarImportance``. - :param nactive_vars: The size of the randomly selected subset of features at each tree node and that are used to find the best split(s). If you set it to 0 then the size will be set to the square root of the total number of features. + :param nactiveVars: The size of the randomly selected subset of features at each tree node and that are used to find the best split(s). If you set it to 0 then the size will be set to the square root of the total number of features. - :param max_num_of_trees_in_the_forest: The maximum number of trees in the forest (surprise, surprise). Typically the more trees you have the better the accuracy. However, the improvement in accuracy generally diminishes and asymptotes pass a certain number of trees. Also to keep in mind, the number of tree increases the prediction time linearly. + :param termCrit: The termination criteria that specifies when the training algorithm stops - either when the specified number of trees is trained and added to the ensemble or when sufficient accuracy (measured as OOB error) is achieved. Typically the more trees you have the better the accuracy. However, the improvement in accuracy generally diminishes and asymptotes pass a certain number of trees. Also to keep in mind, the number of tree increases the prediction time linearly. - :param forest_accuracy: Sufficient accuracy (OOB error). - - :param termcrit_type: The type of the termination criteria: - - * **CV_TERMCRIT_ITER** Terminate learning by the ``max_num_of_trees_in_the_forest``; - - * **CV_TERMCRIT_EPS** Terminate learning by the ``forest_accuracy``; - - * **CV_TERMCRIT_ITER | CV_TERMCRIT_EPS** Use both termination criteria. - -For meaning of other parameters see :ocv:func:`CvDTreeParams::CvDTreeParams`. - -The default constructor sets all parameters to default values which are different from default values of :ocv:class:`CvDTreeParams`: +The default constructor sets all parameters to default values which are different from default values of ``DTrees::Params``: :: - CvRTParams::CvRTParams() : CvDTreeParams( 5, 10, 0, false, 10, 0, false, false, 0 ), - calc_var_importance(false), nactive_vars(0) + RTrees::Params::Params() : DTrees::Params( 5, 10, 0, false, 10, 0, false, false, Mat() ), + calcVarImportance(false), nactiveVars(0) { - term_crit = cvTermCriteria( CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 50, 0.1 ); + termCrit = cvTermCriteria( TermCriteria::MAX_ITERS + TermCriteria::EPS, 50, 0.1 ); } -CvRTrees +RTrees -------- -.. ocv:class:: CvRTrees : public CvStatModel +.. ocv:class:: RTrees : public DTrees The class implements the random forest predictor as described in the beginning of this section. -CvRTrees::train +RTrees::create --------------- -Trains the Random Trees model. - -.. ocv:function:: bool CvRTrees::train( const Mat& trainData, int tflag, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), const Mat& varType=Mat(), const Mat& missingDataMask=Mat(), CvRTParams params=CvRTParams() ) - -.. ocv:function:: bool CvRTrees::train( const CvMat* trainData, int tflag, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, const CvMat* varType=0, const CvMat* missingDataMask=0, CvRTParams params=CvRTParams() ) - -.. ocv:function:: bool CvRTrees::train( CvMLData* data, CvRTParams params=CvRTParams() ) - -.. ocv:pyfunction:: cv2.RTrees.train(trainData, tflag, responses[, varIdx[, sampleIdx[, varType[, missingDataMask[, params]]]]]) -> retval - -The method :ocv:func:`CvRTrees::train` is very similar to the method :ocv:func:`CvDTree::train` and follows the generic method :ocv:func:`CvStatModel::train` conventions. All the parameters specific to the algorithm training are passed as a :ocv:class:`CvRTParams` instance. The estimate of the training error (``oob-error``) is stored in the protected class member ``oob_error``. - -The function is parallelized with the TBB library. - -CvRTrees::predict ------------------ -Predicts the output for an input sample. - -.. ocv:function:: float CvRTrees::predict( const Mat& sample, const Mat& missing=Mat() ) const +Creates the empty model -.. ocv:function:: float CvRTrees::predict( const CvMat* sample, const CvMat* missing = 0 ) const +.. ocv:function:: bool RTrees::create(const RTrees::Params& params=Params()) -.. ocv:pyfunction:: cv2.RTrees.predict(sample[, missing]) -> retval +Use ``StatModel::train`` to train the model, ``StatModel::train(traindata, params)`` to create and train the model, ``StatModel::load(filename)`` to load the pre-trained model. - :param sample: Sample for classification. - - :param missing: Optional missing measurement mask of the sample. - -The input parameters of the prediction method are the same as in :ocv:func:`CvDTree::predict` but the return value type is different. This method returns the cumulative result from all the trees in the forest (the class that receives the majority of voices, or the mean of the regression function estimates). - - -CvRTrees::predict_prob ----------------------- -Returns a fuzzy-predicted class label. - -.. ocv:function:: float CvRTrees::predict_prob( const cv::Mat& sample, const cv::Mat& missing = cv::Mat() ) const - -.. ocv:function:: float CvRTrees::predict_prob( const CvMat* sample, const CvMat* missing = 0 ) const - -.. ocv:pyfunction:: cv2.RTrees.predict_prob(sample[, missing]) -> retval - - :param sample: Sample for classification. - - :param missing: Optional missing measurement mask of the sample. - -The function works for binary classification problems only. It returns the number between 0 and 1. This number represents probability or confidence of the sample belonging to the second class. It is calculated as the proportion of decision trees that classified the sample to the second class. - - -CvRTrees::getVarImportance +RTrees::getVarImportance ---------------------------- Returns the variable importance array. -.. ocv:function:: Mat CvRTrees::getVarImportance() - -.. ocv:function:: const CvMat* CvRTrees::get_var_importance() - -.. ocv:pyfunction:: cv2.RTrees.getVarImportance() -> retval - -The method returns the variable importance vector, computed at the training stage when ``CvRTParams::calc_var_importance`` is set to true. If this flag was set to false, the ``NULL`` pointer is returned. This differs from the decision trees where variable importance can be computed anytime after the training. - - -CvRTrees::get_proximity ------------------------ -Retrieves the proximity measure between two training samples. - -.. ocv:function:: float CvRTrees::get_proximity( const CvMat* sample1, const CvMat* sample2, const CvMat* missing1 = 0, const CvMat* missing2 = 0 ) const - - :param sample1: The first sample. - - :param sample2: The second sample. - - :param missing1: Optional missing measurement mask of the first sample. - - :param missing2: Optional missing measurement mask of the second sample. - -The method returns proximity measure between any two samples. This is a ratio of those trees in the ensemble, in which the samples fall into the same leaf node, to the total number of the trees. - -CvRTrees::calc_error --------------------- -Returns error of the random forest. - -.. ocv:function:: float CvRTrees::calc_error( CvMLData* data, int type, std::vector* resp=0 ) - -The method is identical to :ocv:func:`CvDTree::calc_error` but uses the random forest as predictor. - - -CvRTrees::get_train_error -------------------------- -Returns the train error. - -.. ocv:function:: float CvRTrees::get_train_error() - -The method works for classification problems only. It returns the proportion of incorrectly classified train samples. - - -CvRTrees::get_rng ------------------ -Returns the state of the used random number generator. - -.. ocv:function:: CvRNG* CvRTrees::get_rng() - - -CvRTrees::get_tree_count ------------------------- -Returns the number of trees in the constructed random forest. - -.. ocv:function:: int CvRTrees::get_tree_count() const - - -CvRTrees::get_tree ------------------- -Returns the specific decision tree in the constructed random forest. - -.. ocv:function:: CvForestTree* CvRTrees::get_tree(int i) const +.. ocv:function:: Mat RTrees::getVarImportance() const - :param i: Index of the decision tree. +The method returns the variable importance vector, computed at the training stage when ``RTParams::calcVarImportance`` is set to true. If this flag was set to false, the empty matrix is returned. diff --git a/modules/ml/doc/statistical_models.rst b/modules/ml/doc/statistical_models.rst index af250b7864524399ba86e448f187126cea8cee8a..82cffbbfe3e70fce0e047ce4641f6865785896f1 100644 --- a/modules/ml/doc/statistical_models.rst +++ b/modules/ml/doc/statistical_models.rst @@ -3,161 +3,110 @@ Statistical Models .. highlight:: cpp -.. index:: CvStatModel +.. index:: StatModel -CvStatModel +StatModel ----------- -.. ocv:class:: CvStatModel +.. ocv:class:: StatModel -Base class for statistical models in ML. :: +Base class for statistical models in OpenCV ML. - class CvStatModel - { - public: - /* CvStatModel(); */ - /* CvStatModel( const Mat& train_data ... ); */ - virtual ~CvStatModel(); - - virtual void clear()=0; - - /* virtual bool train( const Mat& train_data, [int tflag,] ..., const - Mat& responses, ..., - [const Mat& var_idx,] ..., [const Mat& sample_idx,] ... - [const Mat& var_type,] ..., [const Mat& missing_mask,] - ... )=0; - */ - - /* virtual float predict( const Mat& sample ... ) const=0; */ - - virtual void save( const char* filename, const char* name=0 )=0; - virtual void load( const char* filename, const char* name=0 )=0; - - virtual void write( CvFileStorage* storage, const char* name )=0; - virtual void read( CvFileStorage* storage, CvFileNode* node )=0; - }; - - -In this declaration, some methods are commented off. These are methods for which there is no unified API (with the exception of the default constructor). However, there are many similarities in the syntax and semantics that are briefly described below in this section, as if they are part of the base class. - -CvStatModel::CvStatModel +StatModel::train ------------------------ -The default constructor. +Trains the statistical model -.. ocv:function:: CvStatModel::CvStatModel() +.. ocv:function:: bool StatModel::train( const Ptr& trainData, int flags=0 ) -Each statistical model class in ML has a default constructor without parameters. This constructor is useful for a two-stage model construction, when the default constructor is followed by :ocv:func:`CvStatModel::train` or :ocv:func:`CvStatModel::load`. +.. ocv:function:: bool StatModel::train( InputArray samples, int layout, InputArray responses ) -CvStatModel::CvStatModel(...) ------------------------------ -The training constructor. - -.. ocv:function:: CvStatModel::CvStatModel() +.. ocv:function:: Ptr<_Tp> StatModel::train(const Ptr& data, const _Tp::Params& p, int flags=0 ) -Most ML classes provide a single-step constructor and train constructors. This constructor is equivalent to the default constructor, followed by the :ocv:func:`CvStatModel::train` method with the parameters that are passed to the constructor. +.. ocv:function:: Ptr<_Tp> StatModel::train(InputArray samples, int layout, InputArray responses, const _Tp::Params& p, int flags=0 ) -CvStatModel::~CvStatModel -------------------------- -The virtual destructor. + :param trainData: training data that can be loaded from file using ``TrainData::loadFromCSV`` or created with ``TrainData::create``. -.. ocv:function:: CvStatModel::~CvStatModel() + :param samples: training samples -The destructor of the base class is declared as virtual. So, it is safe to write the following code: :: + :param layout: ``ROW_SAMPLE`` (training samples are the matrix rows) or ``COL_SAMPLE`` (training samples are the matrix columns) - CvStatModel* model; - if( use_svm ) - model = new CvSVM(... /* SVM params */); - else - model = new CvDTree(... /* Decision tree params */); - ... - delete model; + :param responses: vector of responses associated with the training samples. + :param p: the stat model parameters. -Normally, the destructor of each derived class does nothing. But in this instance, it calls the overridden method :ocv:func:`CvStatModel::clear` that deallocates all the memory. + :param flags: optional flags, depending on the model. Some of the models can be updated with the new training samples, not completely overwritten (such as ``NormalBayesClassifier`` or ``ANN_MLP``). -CvStatModel::clear ------------------- -Deallocates memory and resets the model state. +There are 2 instance methods and 2 static (class) template methods. The first two train the already created model (the very first method must be overwritten in the derived classes). And the latter two variants are convenience methods that construct empty model and then call its train method. -.. ocv:function:: void CvStatModel::clear() -The method ``clear`` does the same job as the destructor: it deallocates all the memory occupied by the class members. But the object itself is not destructed and can be reused further. This method is called from the destructor, from the :ocv:func:`CvStatModel::train` methods of the derived classes, from the methods :ocv:func:`CvStatModel::load`, :ocv:func:`CvStatModel::read()`, or even explicitly by the user. - -CvStatModel::save ------------------ -Saves the model to a file. +StatModel::isTrained +----------------------------- +Returns true if the model is trained -.. ocv:function:: void CvStatModel::save( const char* filename, const char* name=0 ) +.. ocv:function:: bool StatModel::isTrained() -.. ocv:pyfunction:: cv2.StatModel.save(filename[, name]) -> None +The method must be overwritten in the derived classes. -The method ``save`` saves the complete model state to the specified XML or YAML file with the specified name or default name (which depends on a particular class). *Data persistence* functionality from ``CxCore`` is used. +StatModel::isClassifier +----------------------------- +Returns true if the model is classifier -CvStatModel::load ------------------ -Loads the model from a file. +.. ocv:function:: bool StatModel::isClassifier() -.. ocv:function:: void CvStatModel::load( const char* filename, const char* name=0 ) +The method must be overwritten in the derived classes. -.. ocv:pyfunction:: cv2.StatModel.load(filename[, name]) -> None +StatModel::getVarCount +----------------------------- +Returns the number of variables in training samples -The method ``load`` loads the complete model state with the specified name (or default model-dependent name) from the specified XML or YAML file. The previous model state is cleared by :ocv:func:`CvStatModel::clear`. +.. ocv:function:: int StatModel::getVarCount() +The method must be overwritten in the derived classes. -CvStatModel::write +StatModel::predict ------------------ -Writes the model to the file storage. - -.. ocv:function:: void CvStatModel::write( CvFileStorage* storage, const char* name ) - -The method ``write`` stores the complete model state in the file storage with the specified name or default name (which depends on the particular class). The method is called by :ocv:func:`CvStatModel::save`. - +Predicts response(s) for the provided sample(s) -CvStatModel::read ------------------ -Reads the model from the file storage. - -.. ocv:function:: void CvStatModel::read( CvFileStorage* storage, CvFileNode* node ) +.. ocv:function:: float StatModel::predict( InputArray samples, OutputArray results=noArray(), int flags=0 ) const -The method ``read`` restores the complete model state from the specified node of the file storage. Use the function -:ocv:cfunc:`GetFileNodeByName` to locate the node. + :param samples: The input samples, floating-point matrix -The previous model state is cleared by :ocv:func:`CvStatModel::clear`. + :param results: The optional output matrix of results. -CvStatModel::train ------------------- -Trains the model. + :param flags: The optional flags, model-dependent. Some models, such as ``Boost``, ``SVM`` recognize ``StatModel::RAW_OUTPUT`` flag, which makes the method return the raw results (the sum), not the class label. -.. ocv:function:: bool CvStatModel::train( const Mat& train_data, [int tflag,] ..., const Mat& responses, ..., [const Mat& var_idx,] ..., [const Mat& sample_idx,] ... [const Mat& var_type,] ..., [const Mat& missing_mask,] ... ) -The method trains the statistical model using a set of input feature vectors and the corresponding output values (responses). Both input and output vectors/values are passed as matrices. By default, the input feature vectors are stored as ``train_data`` rows, that is, all the components (features) of a training vector are stored continuously. However, some algorithms can handle the transposed representation when all values of each particular feature (component/input variable) over the whole input set are stored continuously. If both layouts are supported, the method includes the ``tflag`` parameter that specifies the orientation as follows: +StatModel::calcError +------------------------- +Computes error on the training or test dataset -* ``tflag=CV_ROW_SAMPLE`` The feature vectors are stored as rows. +.. ocv:function:: float StatModel::calcError( const Ptr& data, bool test, OutputArray resp ) const -* ``tflag=CV_COL_SAMPLE`` The feature vectors are stored as columns. + :param data: the training data -The ``train_data`` must have the ``CV_32FC1`` (32-bit floating-point, single-channel) format. Responses are usually stored in a 1D vector (a row or a column) of ``CV_32SC1`` (only in the classification problem) or ``CV_32FC1`` format, one value per input vector. Although, some algorithms, like various flavors of neural nets, take vector responses. + :param test: if true, the error is computed over the test subset of the data, otherwise it's computed over the training subset of the data. Please note that if you loaded a completely different dataset to evaluate already trained classifier, you will probably want not to set the test subset at all with ``TrainData::setTrainTestSplitRatio`` and specify ``test=false``, so that the error is computed for the whole new set. Yes, this sounds a bit confusing. -For classification problems, the responses are discrete class labels. For regression problems, the responses are values of the function to be approximated. Some algorithms can deal only with classification problems, some - only with regression problems, and some can deal with both problems. In the latter case, the type of output variable is either passed as a separate parameter or as the last element of the ``var_type`` vector: + :param resp: the optional output responses. -* ``CV_VAR_CATEGORICAL`` The output values are discrete class labels. +The method uses ``StatModel::predict`` to compute the error. For regression models the error is computed as RMS, for classifiers - as a percent of missclassified samples (0%-100%). -* ``CV_VAR_ORDERED(=CV_VAR_NUMERICAL)`` The output values are ordered. This means that two different values can be compared as numbers, and this is a regression problem. -Types of input variables can be also specified using ``var_type``. Most algorithms can handle only ordered input variables. +StatModel::save +----------------- +Saves the model to a file. -Many ML models may be trained on a selected feature subset, and/or on a selected sample subset of the training set. To make it easier for you, the method ``train`` usually includes the ``var_idx`` and ``sample_idx`` parameters. The former parameter identifies variables (features) of interest, and the latter one identifies samples of interest. Both vectors are either integer (``CV_32SC1``) vectors (lists of 0-based indices) or 8-bit (``CV_8UC1``) masks of active variables/samples. You may pass ``NULL`` pointers instead of either of the arguments, meaning that all of the variables/samples are used for training. +.. ocv:function:: void StatModel::save( const String& filename ) -Additionally, some algorithms can handle missing measurements, that is, when certain features of certain training samples have unknown values (for example, they forgot to measure a temperature of patient A on Monday). The parameter ``missing_mask``, an 8-bit matrix of the same size as ``train_data``, is used to mark the missed values (non-zero elements of the mask). +In order to make this method work, the derived class must overwrite ``Algorithm::write(FileStorage& fs)``. -Usually, the previous model state is cleared by :ocv:func:`CvStatModel::clear` before running the training procedure. However, some algorithms may optionally update the model state with the new training data, instead of resetting it. +StatModel::load +----------------- +Loads model from the file -CvStatModel::predict --------------------- -Predicts the response for a sample. +.. ocv:function:: Ptr<_Tp> StatModel::load( const String& filename ) -.. ocv:function:: float CvStatModel::predict( const Mat& sample, ... ) const +This is static template method of StatModel. It's usage is following (in the case of SVM): :: -The method is used to predict the response for a new sample. In case of a classification, the method returns the class label. In case of a regression, the method returns the output function value. The input sample must have as many components as the ``train_data`` passed to ``train`` contains. If the ``var_idx`` parameter is passed to ``train``, it is remembered and then is used to extract only the necessary components from the input sample in the method ``predict``. + Ptr svm = StatModel::load("my_svm_model.xml"); -The suffix ``const`` means that prediction does not affect the internal model state, so the method can be safely called from within different threads. +In order to make this method work, the derived class must overwrite ``Algorithm::read(const FileNode& fn)``. diff --git a/modules/ml/doc/support_vector_machines.rst b/modules/ml/doc/support_vector_machines.rst index 9793bd6e3fdc4473ef8359537ce38b3036597f67..d514db28af739d8dd5a5f7042c63144a47dba3d3 100644 --- a/modules/ml/doc/support_vector_machines.rst +++ b/modules/ml/doc/support_vector_machines.rst @@ -14,21 +14,21 @@ SVM implementation in OpenCV is based on [LibSVM]_. .. [LibSVM] C.-C. Chang and C.-J. Lin. *LIBSVM: a library for support vector machines*, ACM Transactions on Intelligent Systems and Technology, 2:27:1--27:27, 2011. (http://www.csie.ntu.edu.tw/~cjlin/papers/libsvm.pdf) -CvParamGrid +ParamGrid ----------- -.. ocv:struct:: CvParamGrid +.. ocv:class:: ParamGrid The structure represents the logarithmic grid range of statmodel parameters. It is used for optimizing statmodel accuracy by varying model parameters, the accuracy estimate being computed by cross-validation. - .. ocv:member:: double CvParamGrid::min_val + .. ocv:member:: double ParamGrid::minVal Minimum value of the statmodel parameter. - .. ocv:member:: double CvParamGrid::max_val + .. ocv:member:: double ParamGrid::maxVal Maximum value of the statmodel parameter. - .. ocv:member:: double CvParamGrid::step + .. ocv:member:: double ParamGrid::logStep Logarithmic step for iterating the statmodel parameter. @@ -36,88 +36,78 @@ The grid determines the following iteration sequence of the statmodel parameter .. math:: - (min\_val, min\_val*step, min\_val*{step}^2, \dots, min\_val*{step}^n), + (minVal, minVal*step, minVal*{step}^2, \dots, minVal*{logStep}^n), where :math:`n` is the maximal index satisfying .. math:: - \texttt{min\_val} * \texttt{step} ^n < \texttt{max\_val} + \texttt{minVal} * \texttt{logStep} ^n < \texttt{maxVal} -The grid is logarithmic, so ``step`` must always be greater then 1. +The grid is logarithmic, so ``logStep`` must always be greater then 1. -CvParamGrid::CvParamGrid +ParamGrid::ParamGrid ------------------------ The constructors. -.. ocv:function:: CvParamGrid::CvParamGrid() +.. ocv:function:: ParamGrid::ParamGrid() -.. ocv:function:: CvParamGrid::CvParamGrid( double min_val, double max_val, double log_step ) +.. ocv:function:: ParamGrid::ParamGrid( double minVal, double maxVal, double logStep ) The full constructor initializes corresponding members. The default constructor creates a dummy grid: :: - CvParamGrid::CvParamGrid() + ParamGrid::ParamGrid() { - min_val = max_val = step = 0; + minVal = maxVal = 0; + logStep = 1; } -CvParamGrid::check ------------------- -Checks validness of the grid. -.. ocv:function:: bool CvParamGrid::check() - -Returns ``true`` if the grid is valid and ``false`` otherwise. The grid is valid if and only if: - -* Lower bound of the grid is less then the upper one. -* Lower bound of the grid is positive. -* Grid step is greater then 1. - -CvSVMParams +SVM::Params ----------- -.. ocv:struct:: CvSVMParams +.. ocv:class:: SVM::Params SVM training parameters. -The structure must be initialized and passed to the training method of :ocv:class:`CvSVM`. +The structure must be initialized and passed to the training method of :ocv:class:`SVM`. -CvSVMParams::CvSVMParams +SVM::Params::Params ------------------------ -The constructors. +The constructors -.. ocv:function:: CvSVMParams::CvSVMParams() +.. ocv:function:: SVM::Params::Params() -.. ocv:function:: CvSVMParams::CvSVMParams( int svm_type, int kernel_type, double degree, double gamma, double coef0, double Cvalue, double nu, double p, CvMat* class_weights, CvTermCriteria term_crit ) +.. ocv:function:: SVM::Params::Params( int svmType, int kernelType, double degree, double gamma, double coef0, double Cvalue, double nu, double p, const Mat& classWeights, TermCriteria termCrit ) - :param svm_type: Type of a SVM formulation. Possible values are: + :param svmType: Type of a SVM formulation. Possible values are: - * **CvSVM::C_SVC** C-Support Vector Classification. ``n``-class classification (``n`` :math:`\geq` 2), allows imperfect separation of classes with penalty multiplier ``C`` for outliers. + * **SVM::C_SVC** C-Support Vector Classification. ``n``-class classification (``n`` :math:`\geq` 2), allows imperfect separation of classes with penalty multiplier ``C`` for outliers. - * **CvSVM::NU_SVC** :math:`\nu`-Support Vector Classification. ``n``-class classification with possible imperfect separation. Parameter :math:`\nu` (in the range 0..1, the larger the value, the smoother the decision boundary) is used instead of ``C``. + * **SVM::NU_SVC** :math:`\nu`-Support Vector Classification. ``n``-class classification with possible imperfect separation. Parameter :math:`\nu` (in the range 0..1, the larger the value, the smoother the decision boundary) is used instead of ``C``. - * **CvSVM::ONE_CLASS** Distribution Estimation (One-class SVM). All the training data are from the same class, SVM builds a boundary that separates the class from the rest of the feature space. + * **SVM::ONE_CLASS** Distribution Estimation (One-class SVM). All the training data are from the same class, SVM builds a boundary that separates the class from the rest of the feature space. - * **CvSVM::EPS_SVR** :math:`\epsilon`-Support Vector Regression. The distance between feature vectors from the training set and the fitting hyper-plane must be less than ``p``. For outliers the penalty multiplier ``C`` is used. + * **SVM::EPS_SVR** :math:`\epsilon`-Support Vector Regression. The distance between feature vectors from the training set and the fitting hyper-plane must be less than ``p``. For outliers the penalty multiplier ``C`` is used. - * **CvSVM::NU_SVR** :math:`\nu`-Support Vector Regression. :math:`\nu` is used instead of ``p``. + * **SVM::NU_SVR** :math:`\nu`-Support Vector Regression. :math:`\nu` is used instead of ``p``. See [LibSVM]_ for details. - :param kernel_type: Type of a SVM kernel. Possible values are: + :param kernelType: Type of a SVM kernel. Possible values are: - * **CvSVM::LINEAR** Linear kernel. No mapping is done, linear discrimination (or regression) is done in the original feature space. It is the fastest option. :math:`K(x_i, x_j) = x_i^T x_j`. + * **SVM::LINEAR** Linear kernel. No mapping is done, linear discrimination (or regression) is done in the original feature space. It is the fastest option. :math:`K(x_i, x_j) = x_i^T x_j`. - * **CvSVM::POLY** Polynomial kernel: :math:`K(x_i, x_j) = (\gamma x_i^T x_j + coef0)^{degree}, \gamma > 0`. + * **SVM::POLY** Polynomial kernel: :math:`K(x_i, x_j) = (\gamma x_i^T x_j + coef0)^{degree}, \gamma > 0`. - * **CvSVM::RBF** Radial basis function (RBF), a good choice in most cases. :math:`K(x_i, x_j) = e^{-\gamma ||x_i - x_j||^2}, \gamma > 0`. + * **SVM::RBF** Radial basis function (RBF), a good choice in most cases. :math:`K(x_i, x_j) = e^{-\gamma ||x_i - x_j||^2}, \gamma > 0`. - * **CvSVM::SIGMOID** Sigmoid kernel: :math:`K(x_i, x_j) = \tanh(\gamma x_i^T x_j + coef0)`. + * **SVM::SIGMOID** Sigmoid kernel: :math:`K(x_i, x_j) = \tanh(\gamma x_i^T x_j + coef0)`. - * **CvSVM::CHI2** Exponential Chi2 kernel, similar to the RBF kernel: :math:`K(x_i, x_j) = e^{-\gamma \chi^2(x_i,x_j)}, \chi^2(x_i,x_j) = (x_i-x_j)^2/(x_i+x_j), \gamma > 0`. + * **SVM::CHI2** Exponential Chi2 kernel, similar to the RBF kernel: :math:`K(x_i, x_j) = e^{-\gamma \chi^2(x_i,x_j)}, \chi^2(x_i,x_j) = (x_i-x_j)^2/(x_i+x_j), \gamma > 0`. - * **CvSVM::INTER** Histogram intersection kernel. A fast kernel. :math:`K(x_i, x_j) = min(x_i,x_j)`. + * **SVM::INTER** Histogram intersection kernel. A fast kernel. :math:`K(x_i, x_j) = min(x_i,x_j)`. :param degree: Parameter ``degree`` of a kernel function (POLY). @@ -131,19 +121,19 @@ The constructors. :param p: Parameter :math:`\epsilon` of a SVM optimization problem (EPS_SVR). - :param class_weights: Optional weights in the C_SVC problem , assigned to particular classes. They are multiplied by ``C`` so the parameter ``C`` of class ``#i`` becomes :math:`class\_weights_i * C`. Thus these weights affect the misclassification penalty for different classes. The larger weight, the larger penalty on misclassification of data from the corresponding class. + :param classWeights: Optional weights in the C_SVC problem , assigned to particular classes. They are multiplied by ``C`` so the parameter ``C`` of class ``#i`` becomes ``classWeights(i) * C``. Thus these weights affect the misclassification penalty for different classes. The larger weight, the larger penalty on misclassification of data from the corresponding class. - :param term_crit: Termination criteria of the iterative SVM training procedure which solves a partial case of constrained quadratic optimization problem. You can specify tolerance and/or the maximum number of iterations. + :param termCrit: Termination criteria of the iterative SVM training procedure which solves a partial case of constrained quadratic optimization problem. You can specify tolerance and/or the maximum number of iterations. The default constructor initialize the structure with following values: :: - CvSVMParams::CvSVMParams() : - svm_type(CvSVM::C_SVC), kernel_type(CvSVM::RBF), degree(0), - gamma(1), coef0(0), C(1), nu(0), p(0), class_weights(0) + SVMParams::SVMParams() : + svmType(SVM::C_SVC), kernelType(SVM::RBF), degree(0), + gamma(1), coef0(0), C(1), nu(0), p(0), classWeights(0) { - term_crit = cvTermCriteria( CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 1000, FLT_EPSILON ); + termCrit = TermCriteria( TermCriteria::MAX_ITER+TermCriteria::EPS, 1000, FLT_EPSILON ); } A comparison of different kernels on the following 2D test case with four classes. Four C_SVC SVMs have been trained (one against rest) with auto_train. Evaluation on three different kernels (CHI2, INTER, RBF). The color depicts the class with max score. Bright means max-score > 0, dark means max-score < 0. @@ -151,10 +141,9 @@ A comparison of different kernels on the following 2D test case with four classe .. image:: pics/SVM_Comparison.png - -CvSVM +SVM ----- -.. ocv:class:: CvSVM : public CvStatModel +.. ocv:class:: SVM : public StatModel Support Vector Machines. @@ -164,55 +153,27 @@ Support Vector Machines. * (Python) An example of grid search digit recognition using SVM can be found at opencv_source/samples/python2/digits_adjust.py * (Python) An example of video digit recognition using SVM can be found at opencv_source/samples/python2/digits_video.py -CvSVM::CvSVM ------------- -Default and training constructors. - -.. ocv:function:: CvSVM::CvSVM() - -.. ocv:function:: CvSVM::CvSVM( const Mat& trainData, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), CvSVMParams params=CvSVMParams() ) - -.. ocv:function:: CvSVM::CvSVM( const CvMat* trainData, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, CvSVMParams params=CvSVMParams() ) - -.. ocv:pyfunction:: cv2.SVM([trainData, responses[, varIdx[, sampleIdx[, params]]]]) -> - -The constructors follow conventions of :ocv:func:`CvStatModel::CvStatModel`. See :ocv:func:`CvStatModel::train` for parameters descriptions. - -CvSVM::train +SVM::create ------------ -Trains an SVM. - -.. ocv:function:: bool CvSVM::train( const Mat& trainData, const Mat& responses, const Mat& varIdx=Mat(), const Mat& sampleIdx=Mat(), CvSVMParams params=CvSVMParams() ) - -.. ocv:function:: bool CvSVM::train( const CvMat* trainData, const CvMat* responses, const CvMat* varIdx=0, const CvMat* sampleIdx=0, CvSVMParams params=CvSVMParams() ) - -.. ocv:pyfunction:: cv2.SVM.train(trainData, responses[, varIdx[, sampleIdx[, params]]]) -> retval - -The method trains the SVM model. It follows the conventions of the generic :ocv:func:`CvStatModel::train` approach with the following limitations: - -* Only the ``CV_ROW_SAMPLE`` data layout is supported. - -* Input variables are all ordered. +Creates empty model -* Output variables can be either categorical (``params.svm_type=CvSVM::C_SVC`` or ``params.svm_type=CvSVM::NU_SVC``), or ordered (``params.svm_type=CvSVM::EPS_SVR`` or ``params.svm_type=CvSVM::NU_SVR``), or not required at all (``params.svm_type=CvSVM::ONE_CLASS``). +.. ocv:function:: Ptr SVM::create(const Params& p=Params(), const Ptr& customKernel=Ptr()) -* Missing measurements are not supported. + :param p: SVM parameters + :param customKernel: the optional custom kernel to use. It must implement ``SVM::Kernel`` interface. -All the other parameters are gathered in the -:ocv:class:`CvSVMParams` structure. +Use ``StatModel::train`` to train the model, ``StatModel::train(traindata, params)`` to create and train the model, ``StatModel::load(filename)`` to load the pre-trained model. Since SVM has several parameters, you may want to find the best parameters for your problem. It can be done with ``SVM::trainAuto``. -CvSVM::train_auto +SVM::trainAuto ----------------- Trains an SVM with optimal parameters. -.. ocv:function:: bool CvSVM::train_auto( const Mat& trainData, const Mat& responses, const Mat& varIdx, const Mat& sampleIdx, CvSVMParams params, int k_fold = 10, CvParamGrid Cgrid = CvSVM::get_default_grid(CvSVM::C), CvParamGrid gammaGrid = CvSVM::get_default_grid(CvSVM::GAMMA), CvParamGrid pGrid = CvSVM::get_default_grid(CvSVM::P), CvParamGrid nuGrid = CvSVM::get_default_grid(CvSVM::NU), CvParamGrid coeffGrid = CvSVM::get_default_grid(CvSVM::COEF), CvParamGrid degreeGrid = CvSVM::get_default_grid(CvSVM::DEGREE), bool balanced=false) +.. ocv:function:: bool SVM::trainAuto( const Ptr& data, int kFold = 10, ParamGrid Cgrid = SVM::getDefaultGrid(SVM::C), ParamGrid gammaGrid = SVM::getDefaultGrid(SVM::GAMMA), ParamGrid pGrid = SVM::getDefaultGrid(SVM::P), ParamGrid nuGrid = SVM::getDefaultGrid(SVM::NU), ParamGrid coeffGrid = SVM::getDefaultGrid(SVM::COEF), ParamGrid degreeGrid = SVM::getDefaultGrid(SVM::DEGREE), bool balanced=false) -.. ocv:function:: bool CvSVM::train_auto( const CvMat* trainData, const CvMat* responses, const CvMat* varIdx, const CvMat* sampleIdx, CvSVMParams params, int kfold = 10, CvParamGrid Cgrid = get_default_grid(CvSVM::C), CvParamGrid gammaGrid = get_default_grid(CvSVM::GAMMA), CvParamGrid pGrid = get_default_grid(CvSVM::P), CvParamGrid nuGrid = get_default_grid(CvSVM::NU), CvParamGrid coeffGrid = get_default_grid(CvSVM::COEF), CvParamGrid degreeGrid = get_default_grid(CvSVM::DEGREE), bool balanced=false ) + :param data: the training data that can be constructed using ``TrainData::create`` or ``TrainData::loadFromCSV``. -.. ocv:pyfunction:: cv2.SVM.train_auto(trainData, responses, varIdx, sampleIdx, params[, k_fold[, Cgrid[, gammaGrid[, pGrid[, nuGrid[, coeffGrid[, degreeGrid[, balanced]]]]]]]]) -> retval - - :param k_fold: Cross-validation parameter. The training set is divided into ``k_fold`` subsets. One subset is used to test the model, the others form the train set. So, the SVM algorithm is executed ``k_fold`` times. + :param kFold: Cross-validation parameter. The training set is divided into ``kFold`` subsets. One subset is used to test the model, the others form the train set. So, the SVM algorithm is executed ``kFold`` times. :param \*Grid: Iteration grid for the corresponding SVM parameter. @@ -220,97 +181,76 @@ Trains an SVM with optimal parameters. The method trains the SVM model automatically by choosing the optimal parameters ``C``, ``gamma``, ``p``, ``nu``, ``coef0``, ``degree`` from -:ocv:class:`CvSVMParams`. Parameters are considered optimal +``SVM::Params``. Parameters are considered optimal when the cross-validation estimate of the test set error is minimal. -If there is no need to optimize a parameter, the corresponding grid step should be set to any value less than or equal to 1. For example, to avoid optimization in ``gamma``, set ``gamma_grid.step = 0``, ``gamma_grid.min_val``, ``gamma_grid.max_val`` as arbitrary numbers. In this case, the value ``params.gamma`` is taken for ``gamma``. +If there is no need to optimize a parameter, the corresponding grid step should be set to any value less than or equal to 1. For example, to avoid optimization in ``gamma``, set ``gammaGrid.step = 0``, ``gammaGrid.minVal``, ``gamma_grid.maxVal`` as arbitrary numbers. In this case, the value ``params.gamma`` is taken for ``gamma``. And, finally, if the optimization in a parameter is required but -the corresponding grid is unknown, you may call the function :ocv:func:`CvSVM::get_default_grid`. To generate a grid, for example, for ``gamma``, call ``CvSVM::get_default_grid(CvSVM::GAMMA)``. +the corresponding grid is unknown, you may call the function :ocv:func:`SVM::getDefaulltGrid`. To generate a grid, for example, for ``gamma``, call ``SVM::getDefaulltGrid(SVM::GAMMA)``. This function works for the classification -(``params.svm_type=CvSVM::C_SVC`` or ``params.svm_type=CvSVM::NU_SVC``) +(``params.svmType=SVM::C_SVC`` or ``params.svmType=SVM::NU_SVC``) as well as for the regression -(``params.svm_type=CvSVM::EPS_SVR`` or ``params.svm_type=CvSVM::NU_SVR``). If ``params.svm_type=CvSVM::ONE_CLASS``, no optimization is made and the usual SVM with parameters specified in ``params`` is executed. - -CvSVM::predict --------------- -Predicts the response for input sample(s). - -.. ocv:function:: float CvSVM::predict( const Mat& sample, bool returnDFVal=false ) const - -.. ocv:function:: float CvSVM::predict( const CvMat* sample, bool returnDFVal=false ) const - -.. ocv:function:: float CvSVM::predict( const CvMat* samples, CvMat* results, bool returnDFVal=false ) const - -.. ocv:pyfunction:: cv2.SVM.predict(sample[, returnDFVal]) -> retval - -.. ocv:pyfunction:: cv2.SVM.predict_all(samples[, results]) -> results - - :param sample: Input sample for prediction. - - :param samples: Input samples for prediction. +(``params.svmType=SVM::EPS_SVR`` or ``params.svmType=SVM::NU_SVR``). If ``params.svmType=SVM::ONE_CLASS``, no optimization is made and the usual SVM with parameters specified in ``params`` is executed. - :param returnDFVal: Specifies a type of the return value. If ``true`` and the problem is 2-class classification then the method returns the decision function value that is signed distance to the margin, else the function returns a class label (classification) or estimated function value (regression). - :param results: Output prediction responses for corresponding samples. - -If you pass one sample then prediction result is returned. If you want to get responses for several samples then you should pass the ``results`` matrix where prediction results will be stored. - -The function is parallelized with the TBB library. - - -CvSVM::get_default_grid +SVM::getDefaulltGrid ----------------------- Generates a grid for SVM parameters. -.. ocv:function:: CvParamGrid CvSVM::get_default_grid( int param_id ) +.. ocv:function:: ParamGrid SVM::getDefaulltGrid( int param_id ) :param param_id: SVM parameters IDs that must be one of the following: - * **CvSVM::C** + * **SVM::C** - * **CvSVM::GAMMA** + * **SVM::GAMMA** - * **CvSVM::P** + * **SVM::P** - * **CvSVM::NU** + * **SVM::NU** - * **CvSVM::COEF** + * **SVM::COEF** - * **CvSVM::DEGREE** + * **SVM::DEGREE** The grid is generated for the parameter with this ID. -The function generates a grid for the specified parameter of the SVM algorithm. The grid may be passed to the function :ocv:func:`CvSVM::train_auto`. +The function generates a grid for the specified parameter of the SVM algorithm. The grid may be passed to the function :ocv:func:`SVM::trainAuto`. -CvSVM::get_params +SVM::getParams ----------------- Returns the current SVM parameters. -.. ocv:function:: CvSVMParams CvSVM::get_params() const +.. ocv:function:: SVM::Params SVM::getParams() const -This function may be used to get the optimal parameters obtained while automatically training :ocv:func:`CvSVM::train_auto`. +This function may be used to get the optimal parameters obtained while automatically training ``SVM::trainAuto``. -CvSVM::get_support_vector +SVM::getSupportVectors -------------------------- -Retrieves a number of support vectors and the particular vector. +Retrieves all the support vectors -.. ocv:function:: int CvSVM::get_support_vector_count() const +.. ocv:function:: Mat SVM::getSupportVectors() const -.. ocv:function:: const float* CvSVM::get_support_vector(int i) const +The method returns all the support vector as floating-point matrix, where support vectors are stored as matrix rows. -.. ocv:pyfunction:: cv2.SVM.get_support_vector_count() -> retval +SVM::getDecisionFunction +-------------------------- +Retrieves the decision function - :param i: Index of the particular support vector. +.. ocv:function:: double SVM::getDecisionFunction(int i, OutputArray alpha, OutputArray svidx) const -The methods can be used to retrieve a set of support vectors. + :param i: the index of the decision function. If the problem solved is regression, 1-class or 2-class classification, then there will be just one decision function and the index should always be 0. Otherwise, in the case of N-class classification, there will be N*(N-1)/2 decision functions. -CvSVM::get_var_count --------------------- -Returns the number of used features (variables count). + :param alpha: the optional output vector for weights, corresponding to different support vectors. In the case of linear SVM all the alpha's will be 1's. + + :param svidx: the optional output vector of indices of support vectors within the matrix of support vectors (which can be retrieved by ``SVM::getSupportVectors``). In the case of linear SVM each decision function consists of a single "compressed" support vector. -.. ocv:function:: int CvSVM::get_var_count() const +The method returns ``rho`` parameter of the decision function, a scalar subtracted from the weighted sum of kernel responses. + +Prediction with SVM +-------------------- -.. ocv:pyfunction:: cv2.SVM.get_var_count() -> retval +StatModel::predict(samples, results, flags) should be used. Pass ``flags=StatModel::RAW_OUTPUT`` to get the raw response from SVM (in the case of regression, 1-class or 2-class classification problem). diff --git a/modules/ml/include/opencv2/ml.hpp b/modules/ml/include/opencv2/ml.hpp index f13e192be8963906b037180a4e7942cca55dd42d..a5ce3010bfb349e3ac787499ca619a18a075fb4d 100644 --- a/modules/ml/include/opencv2/ml.hpp +++ b/modules/ml/include/opencv2/ml.hpp @@ -7,9 +7,11 @@ // copy or use the software. // // -// Intel License Agreement +// License Agreement +// For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -22,7 +24,7 @@ // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // -// * The name of Intel Corporation may not be used to endorse or promote products +// * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and @@ -45,111 +47,150 @@ # include "opencv2/core.hpp" #endif -#include "opencv2/core/core_c.h" -#include - #ifdef __cplusplus +#include #include #include -// Apple defines a check() macro somewhere in the debug headers -// that interferes with a method definiton in this header -#undef check - -/****************************************************************************************\ -* Main struct definitions * -\****************************************************************************************/ - -/* log(2*PI) */ -#define CV_LOG2PI (1.8378770664093454835606594728112) - -/* columns of matrix are training samples */ -#define CV_COL_SAMPLE 0 - -/* rows of matrix are training samples */ -#define CV_ROW_SAMPLE 1 +namespace cv +{ -#define CV_IS_ROW_SAMPLE(flags) ((flags) & CV_ROW_SAMPLE) +namespace ml +{ -struct CvVectors +/* Variable type */ +enum { - int type; - int dims, count; - CvVectors* next; - union - { - uchar** ptr; - float** fl; - double** db; - } data; + VAR_NUMERICAL =0, + VAR_ORDERED =0, + VAR_CATEGORICAL =1 }; -#if 0 -/* A structure, representing the lattice range of statmodel parameters. - It is used for optimizing statmodel parameters by cross-validation method. - The lattice is logarithmic, so must be greater then 1. */ -typedef struct CvParamLattice +enum { - double min_val; - double max_val; - double step; -} -CvParamLattice; + TEST_ERROR = 0, + TRAIN_ERROR = 1 +}; -CV_INLINE CvParamLattice cvParamLattice( double min_val, double max_val, - double log_step ) +enum { - CvParamLattice pl; - pl.min_val = MIN( min_val, max_val ); - pl.max_val = MAX( min_val, max_val ); - pl.step = MAX( log_step, 1. ); - return pl; -} + ROW_SAMPLE = 0, + COL_SAMPLE = 1 +}; -CV_INLINE CvParamLattice cvDefaultParamLattice( void ) +class CV_EXPORTS_W_MAP ParamGrid { - CvParamLattice pl = {0,0,0}; - return pl; -} -#endif +public: + ParamGrid(); + ParamGrid(double _minVal, double _maxVal, double _logStep); -/* Variable type */ -#define CV_VAR_NUMERICAL 0 -#define CV_VAR_ORDERED 0 -#define CV_VAR_CATEGORICAL 1 - -#define CV_TYPE_NAME_ML_SVM "opencv-ml-svm" -#define CV_TYPE_NAME_ML_KNN "opencv-ml-knn" -#define CV_TYPE_NAME_ML_NBAYES "opencv-ml-bayesian" -#define CV_TYPE_NAME_ML_EM "opencv-ml-em" -#define CV_TYPE_NAME_ML_BOOSTING "opencv-ml-boost-tree" -#define CV_TYPE_NAME_ML_TREE "opencv-ml-tree" -#define CV_TYPE_NAME_ML_ANN_MLP "opencv-ml-ann-mlp" -#define CV_TYPE_NAME_ML_CNN "opencv-ml-cnn" -#define CV_TYPE_NAME_ML_RTREES "opencv-ml-random-trees" -#define CV_TYPE_NAME_ML_ERTREES "opencv-ml-extremely-randomized-trees" -#define CV_TYPE_NAME_ML_GBT "opencv-ml-gradient-boosting-trees" - -#define CV_TRAIN_ERROR 0 -#define CV_TEST_ERROR 1 - -class CV_EXPORTS_W CvStatModel + CV_PROP_RW double minVal; + CV_PROP_RW double maxVal; + CV_PROP_RW double logStep; +}; + + +class CV_EXPORTS TrainData { public: - CvStatModel(); - virtual ~CvStatModel(); - + static inline float missingValue() { return FLT_MAX; } + virtual ~TrainData(); + + virtual int getLayout() const = 0; + virtual int getNTrainSamples() const = 0; + virtual int getNTestSamples() const = 0; + virtual int getNSamples() const = 0; + virtual int getNVars() const = 0; + virtual int getNAllVars() const = 0; + + virtual void getSample(InputArray varIdx, int sidx, float* buf) const = 0; + virtual Mat getSamples() const = 0; + virtual Mat getMissing() const = 0; + virtual Mat getTrainSamples(int layout=ROW_SAMPLE, + bool compressSamples=true, + bool compressVars=true) const = 0; + virtual Mat getTrainResponses() const = 0; + virtual Mat getTrainNormCatResponses() const = 0; + virtual Mat getTestResponses() const = 0; + virtual Mat getTestNormCatResponses() const = 0; + virtual Mat getResponses() const = 0; + virtual Mat getNormCatResponses() const = 0; + virtual Mat getSampleWeights() const = 0; + virtual Mat getTrainSampleWeights() const = 0; + virtual Mat getTestSampleWeights() const = 0; + virtual Mat getVarIdx() const = 0; + virtual Mat getVarType() const = 0; + virtual int getResponseType() const = 0; + virtual Mat getTrainSampleIdx() const = 0; + virtual Mat getTestSampleIdx() const = 0; + virtual void getValues(int vi, InputArray sidx, float* values) const = 0; + virtual void getNormCatValues(int vi, InputArray sidx, int* values) const = 0; + virtual Mat getDefaultSubstValues() const = 0; + + virtual int getCatCount(int vi) const = 0; + virtual Mat getClassLabels() const = 0; + + virtual Mat getCatOfs() const = 0; + virtual Mat getCatMap() const = 0; + + virtual void setTrainTestSplit(int count, bool shuffle=true) = 0; + virtual void setTrainTestSplitRatio(double ratio, bool shuffle=true) = 0; + virtual void shuffleTrainTest() = 0; + + static Mat getSubVector(const Mat& vec, const Mat& idx); + static Ptr loadFromCSV(const String& filename, + int headerLineCount, + int responseStartIdx=-1, + int responseEndIdx=-1, + const String& varTypeSpec=String(), + char delimiter=',', + char missch='?'); + static Ptr create(InputArray samples, int layout, InputArray responses, + InputArray varIdx=noArray(), InputArray sampleIdx=noArray(), + InputArray sampleWeights=noArray(), InputArray varType=noArray()); +}; + + +class CV_EXPORTS_W StatModel : public Algorithm +{ +public: + enum { UPDATE_MODEL = 1, RAW_OUTPUT=1, COMPRESSED_INPUT=2, PREPROCESSED_INPUT=4 }; virtual void clear(); - CV_WRAP virtual void save( const char* filename, const char* name=0 ) const; - CV_WRAP virtual void load( const char* filename, const char* name=0 ); + virtual int getVarCount() const = 0; + + virtual bool isTrained() const = 0; + virtual bool isClassifier() const = 0; + + virtual bool train( const Ptr& trainData, int flags=0 ); + virtual bool train( InputArray samples, int layout, InputArray responses ); + virtual float calcError( const Ptr& data, bool test, OutputArray resp ) const; + virtual float predict( InputArray samples, OutputArray results=noArray(), int flags=0 ) const = 0; + + template static Ptr<_Tp> load(const String& filename) + { + FileStorage fs(filename, FileStorage::READ); + Ptr<_Tp> model = _Tp::create(); + model->read(fs.getFirstTopLevelNode()); + return model->isTrained() ? model : Ptr<_Tp>(); + } + + template static Ptr<_Tp> train(const Ptr& data, const typename _Tp::Params& p, int flags=0) + { + Ptr<_Tp> model = _Tp::create(p); + return !model.empty() && model->train(data, flags) ? model : Ptr<_Tp>(); + } - virtual void write( CvFileStorage* storage, const char* name ) const; - virtual void read( CvFileStorage* storage, CvFileNode* node ); + template static Ptr<_Tp> train(InputArray samples, int layout, InputArray responses, + const typename _Tp::Params& p, int flags=0) + { + Ptr<_Tp> model = _Tp::create(p); + return !model.empty() && model->train(TrainData::create(samples, layout, responses), flags) ? model : Ptr<_Tp>(); + } -protected: - const char* default_model_name; + virtual void save(const String& filename) const; + virtual String getDefaultModelName() const = 0; }; /****************************************************************************************\ @@ -161,413 +202,115 @@ protected: the accuracy estimate being computed by cross-validation. The grid is logarithmic, so must be greater then 1. */ -class CvMLData; - -struct CV_EXPORTS_W_MAP CvParamGrid +class CV_EXPORTS_W NormalBayesClassifier : public StatModel { - // SVM params type - enum { SVM_C=0, SVM_GAMMA=1, SVM_P=2, SVM_NU=3, SVM_COEF=4, SVM_DEGREE=5 }; - - CvParamGrid() +public: + class CV_EXPORTS_W Params { - min_val = max_val = step = 0; - } - - CvParamGrid( double min_val, double max_val, double log_step ); - //CvParamGrid( int param_id ); - bool check() const; - - CV_PROP_RW double min_val; - CV_PROP_RW double max_val; - CV_PROP_RW double step; -}; - -inline CvParamGrid::CvParamGrid( double _min_val, double _max_val, double _log_step ) -{ - min_val = _min_val; - max_val = _max_val; - step = _log_step; -} + public: + Params(); + }; + virtual float predictProb( InputArray inputs, OutputArray outputs, + OutputArray outputProbs, int flags=0 ) const = 0; + virtual void setParams(const Params& params) = 0; + virtual Params getParams() const = 0; -class CV_EXPORTS_W CvNormalBayesClassifier : public CvStatModel -{ -public: - CV_WRAP CvNormalBayesClassifier(); - virtual ~CvNormalBayesClassifier(); - - CvNormalBayesClassifier( const CvMat* trainData, const CvMat* responses, - const CvMat* varIdx=0, const CvMat* sampleIdx=0 ); - - virtual bool train( const CvMat* trainData, const CvMat* responses, - const CvMat* varIdx = 0, const CvMat* sampleIdx=0, bool update=false ); - - virtual float predict( const CvMat* samples, CV_OUT CvMat* results=0, CV_OUT CvMat* results_prob=0 ) const; - CV_WRAP virtual void clear(); - - CV_WRAP CvNormalBayesClassifier( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& varIdx=cv::Mat(), const cv::Mat& sampleIdx=cv::Mat() ); - CV_WRAP virtual bool train( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& varIdx = cv::Mat(), const cv::Mat& sampleIdx=cv::Mat(), - bool update=false ); - CV_WRAP virtual float predict( const cv::Mat& samples, CV_OUT cv::Mat* results=0, CV_OUT cv::Mat* results_prob=0 ) const; - - virtual void write( CvFileStorage* storage, const char* name ) const; - virtual void read( CvFileStorage* storage, CvFileNode* node ); - -protected: - int var_count, var_all; - CvMat* var_idx; - CvMat* cls_labels; - CvMat** count; - CvMat** sum; - CvMat** productsum; - CvMat** avg; - CvMat** inv_eigen_values; - CvMat** cov_rotate_mats; - CvMat* c; + static Ptr create(const Params& params=Params()); }; - /****************************************************************************************\ * K-Nearest Neighbour Classifier * \****************************************************************************************/ // k Nearest Neighbors -class CV_EXPORTS_W CvKNearest : public CvStatModel +class CV_EXPORTS_W KNearest : public StatModel { public: + class CV_EXPORTS_W_MAP Params + { + public: + Params(int defaultK=10, bool isclassifier=true); - CV_WRAP CvKNearest(); - virtual ~CvKNearest(); - - CvKNearest( const CvMat* trainData, const CvMat* responses, - const CvMat* sampleIdx=0, bool isRegression=false, int max_k=32 ); - - virtual bool train( const CvMat* trainData, const CvMat* responses, - const CvMat* sampleIdx=0, bool is_regression=false, - int maxK=32, bool updateBase=false ); - - virtual float find_nearest( const CvMat* samples, int k, CV_OUT CvMat* results=0, - const float** neighbors=0, CV_OUT CvMat* neighborResponses=0, CV_OUT CvMat* dist=0 ) const; - - CV_WRAP CvKNearest( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& sampleIdx=cv::Mat(), bool isRegression=false, int max_k=32 ); - - CV_WRAP virtual bool train( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& sampleIdx=cv::Mat(), bool isRegression=false, - int maxK=32, bool updateBase=false ); - - virtual float find_nearest( const cv::Mat& samples, int k, cv::Mat* results=0, - const float** neighbors=0, cv::Mat* neighborResponses=0, - cv::Mat* dist=0 ) const; - CV_WRAP virtual float find_nearest( const cv::Mat& samples, int k, CV_OUT cv::Mat& results, - CV_OUT cv::Mat& neighborResponses, CV_OUT cv::Mat& dists) const; - - virtual void clear(); - int get_max_k() const; - int get_var_count() const; - int get_sample_count() const; - bool is_regression() const; - - virtual float write_results( int k, int k1, int start, int end, - const float* neighbor_responses, const float* dist, CvMat* _results, - CvMat* _neighbor_responses, CvMat* _dist, Cv32suf* sort_buf ) const; - - virtual void find_neighbors_direct( const CvMat* _samples, int k, int start, int end, - float* neighbor_responses, const float** neighbors, float* dist ) const; - -protected: - - int max_k, var_count; - int total; - bool regression; - CvVectors* samples; + CV_PROP_RW int defaultK; + CV_PROP_RW bool isclassifier; + }; + virtual void setParams(const Params& p) = 0; + virtual Params getParams() const = 0; + virtual float findNearest( InputArray samples, int k, + OutputArray results, + OutputArray neighborResponses=noArray(), + OutputArray dist=noArray() ) const = 0; + static Ptr create(const Params& params=Params()); }; /****************************************************************************************\ * Support Vector Machines * \****************************************************************************************/ -// SVM training parameters -struct CV_EXPORTS_W_MAP CvSVMParams -{ - CvSVMParams(); - CvSVMParams( int svm_type, int kernel_type, - double degree, double gamma, double coef0, - double Cvalue, double nu, double p, - CvMat* class_weights, CvTermCriteria term_crit ); - - CV_PROP_RW int svm_type; - CV_PROP_RW int kernel_type; - CV_PROP_RW double degree; // for poly - CV_PROP_RW double gamma; // for poly/rbf/sigmoid/chi2 - CV_PROP_RW double coef0; // for poly/sigmoid - - CV_PROP_RW double C; // for CV_SVM_C_SVC, CV_SVM_EPS_SVR and CV_SVM_NU_SVR - CV_PROP_RW double nu; // for CV_SVM_NU_SVC, CV_SVM_ONE_CLASS, and CV_SVM_NU_SVR - CV_PROP_RW double p; // for CV_SVM_EPS_SVR - CvMat* class_weights; // for CV_SVM_C_SVC - CV_PROP_RW CvTermCriteria term_crit; // termination criteria -}; - - -struct CV_EXPORTS CvSVMKernel -{ - typedef void (CvSVMKernel::*Calc)( int vec_count, int vec_size, const float** vecs, - const float* another, float* results ); - CvSVMKernel(); - CvSVMKernel( const CvSVMParams* params, Calc _calc_func ); - virtual bool create( const CvSVMParams* params, Calc _calc_func ); - virtual ~CvSVMKernel(); - - virtual void clear(); - virtual void calc( int vcount, int n, const float** vecs, const float* another, float* results ); - - const CvSVMParams* params; - Calc calc_func; - - virtual void calc_non_rbf_base( int vec_count, int vec_size, const float** vecs, - const float* another, float* results, - double alpha, double beta ); - virtual void calc_intersec( int vcount, int var_count, const float** vecs, - const float* another, float* results ); - virtual void calc_chi2( int vec_count, int vec_size, const float** vecs, - const float* another, float* results ); - virtual void calc_linear( int vec_count, int vec_size, const float** vecs, - const float* another, float* results ); - virtual void calc_rbf( int vec_count, int vec_size, const float** vecs, - const float* another, float* results ); - virtual void calc_poly( int vec_count, int vec_size, const float** vecs, - const float* another, float* results ); - virtual void calc_sigmoid( int vec_count, int vec_size, const float** vecs, - const float* another, float* results ); -}; - - -struct CvSVMKernelRow -{ - CvSVMKernelRow* prev; - CvSVMKernelRow* next; - float* data; -}; - - -struct CvSVMSolutionInfo -{ - double obj; - double rho; - double upper_bound_p; - double upper_bound_n; - double r; // for Solver_NU -}; - -class CV_EXPORTS CvSVMSolver +// SVM model +class CV_EXPORTS_W SVM : public StatModel { public: - typedef bool (CvSVMSolver::*SelectWorkingSet)( int& i, int& j ); - typedef float* (CvSVMSolver::*GetRow)( int i, float* row, float* dst, bool existed ); - typedef void (CvSVMSolver::*CalcRho)( double& rho, double& r ); - - CvSVMSolver(); - - CvSVMSolver( int count, int var_count, const float** samples, schar* y, - int alpha_count, double* alpha, double Cp, double Cn, - CvMemStorage* storage, CvSVMKernel* kernel, GetRow get_row, - SelectWorkingSet select_working_set, CalcRho calc_rho ); - virtual bool create( int count, int var_count, const float** samples, schar* y, - int alpha_count, double* alpha, double Cp, double Cn, - CvMemStorage* storage, CvSVMKernel* kernel, GetRow get_row, - SelectWorkingSet select_working_set, CalcRho calc_rho ); - virtual ~CvSVMSolver(); - - virtual void clear(); - virtual bool solve_generic( CvSVMSolutionInfo& si ); - - virtual bool solve_c_svc( int count, int var_count, const float** samples, schar* y, - double Cp, double Cn, CvMemStorage* storage, - CvSVMKernel* kernel, double* alpha, CvSVMSolutionInfo& si ); - virtual bool solve_nu_svc( int count, int var_count, const float** samples, schar* y, - CvMemStorage* storage, CvSVMKernel* kernel, - double* alpha, CvSVMSolutionInfo& si ); - virtual bool solve_one_class( int count, int var_count, const float** samples, - CvMemStorage* storage, CvSVMKernel* kernel, - double* alpha, CvSVMSolutionInfo& si ); - - virtual bool solve_eps_svr( int count, int var_count, const float** samples, const float* y, - CvMemStorage* storage, CvSVMKernel* kernel, - double* alpha, CvSVMSolutionInfo& si ); - - virtual bool solve_nu_svr( int count, int var_count, const float** samples, const float* y, - CvMemStorage* storage, CvSVMKernel* kernel, - double* alpha, CvSVMSolutionInfo& si ); - - virtual float* get_row_base( int i, bool* _existed ); - virtual float* get_row( int i, float* dst ); - - int sample_count; - int var_count; - int cache_size; - int cache_line_size; - const float** samples; - const CvSVMParams* params; - CvMemStorage* storage; - CvSVMKernelRow lru_list; - CvSVMKernelRow* rows; - - int alpha_count; - - double* G; - double* alpha; - - // -1 - lower bound, 0 - free, 1 - upper bound - schar* alpha_status; - - schar* y; - double* b; - float* buf[2]; - double eps; - int max_iter; - double C[2]; // C[0] == Cn, C[1] == Cp - CvSVMKernel* kernel; - - SelectWorkingSet select_working_set_func; - CalcRho calc_rho_func; - GetRow get_row_func; - - virtual bool select_working_set( int& i, int& j ); - virtual bool select_working_set_nu_svm( int& i, int& j ); - virtual void calc_rho( double& rho, double& r ); - virtual void calc_rho_nu_svm( double& rho, double& r ); - - virtual float* get_row_svc( int i, float* row, float* dst, bool existed ); - virtual float* get_row_one_class( int i, float* row, float* dst, bool existed ); - virtual float* get_row_svr( int i, float* row, float* dst, bool existed ); -}; - - -struct CvSVMDecisionFunc -{ - double rho; - int sv_count; - double* alpha; - int* sv_index; -}; + class CV_EXPORTS_W_MAP Params + { + public: + Params(); + Params( int svm_type, int kernel_type, + double degree, double gamma, double coef0, + double Cvalue, double nu, double p, + const Mat& classWeights, TermCriteria termCrit ); + + CV_PROP_RW int svmType; + CV_PROP_RW int kernelType; + CV_PROP_RW double gamma, coef0, degree; + + CV_PROP_RW double C; // for CV_SVM_C_SVC, CV_SVM_EPS_SVR and CV_SVM_NU_SVR + CV_PROP_RW double nu; // for CV_SVM_NU_SVC, CV_SVM_ONE_CLASS, and CV_SVM_NU_SVR + CV_PROP_RW double p; // for CV_SVM_EPS_SVR + CV_PROP_RW Mat classWeights; // for CV_SVM_C_SVC + CV_PROP_RW TermCriteria termCrit; // termination criteria + }; + class CV_EXPORTS Kernel : public Algorithm + { + public: + virtual int getType() const = 0; + virtual void calc( int vcount, int n, const float* vecs, const float* another, float* results ) = 0; + }; -// SVM model -class CV_EXPORTS_W CvSVM : public CvStatModel -{ -public: // SVM type enum { C_SVC=100, NU_SVC=101, ONE_CLASS=102, EPS_SVR=103, NU_SVR=104 }; // SVM kernel type - enum { LINEAR=0, POLY=1, RBF=2, SIGMOID=3, CHI2=4, INTER=5 }; + enum { CUSTOM=-1, LINEAR=0, POLY=1, RBF=2, SIGMOID=3, CHI2=4, INTER=5 }; // SVM params type enum { C=0, GAMMA=1, P=2, NU=3, COEF=4, DEGREE=5 }; - CV_WRAP CvSVM(); - virtual ~CvSVM(); - - CvSVM( const CvMat* trainData, const CvMat* responses, - const CvMat* varIdx=0, const CvMat* sampleIdx=0, - CvSVMParams params=CvSVMParams() ); - - virtual bool train( const CvMat* trainData, const CvMat* responses, - const CvMat* varIdx=0, const CvMat* sampleIdx=0, - CvSVMParams params=CvSVMParams() ); - - virtual bool train_auto( const CvMat* trainData, const CvMat* responses, - const CvMat* varIdx, const CvMat* sampleIdx, CvSVMParams params, - int kfold = 10, - CvParamGrid Cgrid = get_default_grid(CvSVM::C), - CvParamGrid gammaGrid = get_default_grid(CvSVM::GAMMA), - CvParamGrid pGrid = get_default_grid(CvSVM::P), - CvParamGrid nuGrid = get_default_grid(CvSVM::NU), - CvParamGrid coeffGrid = get_default_grid(CvSVM::COEF), - CvParamGrid degreeGrid = get_default_grid(CvSVM::DEGREE), - bool balanced=false ); - - virtual float predict( const CvMat* sample, bool returnDFVal=false ) const; - virtual float predict( const CvMat* samples, CV_OUT CvMat* results, bool returnDFVal=false ) const; - - CV_WRAP CvSVM( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& varIdx=cv::Mat(), const cv::Mat& sampleIdx=cv::Mat(), - CvSVMParams params=CvSVMParams() ); - - CV_WRAP virtual bool train( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& varIdx=cv::Mat(), const cv::Mat& sampleIdx=cv::Mat(), - CvSVMParams params=CvSVMParams() ); - - CV_WRAP virtual bool train_auto( const cv::Mat& trainData, const cv::Mat& responses, - const cv::Mat& varIdx, const cv::Mat& sampleIdx, CvSVMParams params, - int k_fold = 10, - CvParamGrid Cgrid = CvSVM::get_default_grid(CvSVM::C), - CvParamGrid gammaGrid = CvSVM::get_default_grid(CvSVM::GAMMA), - CvParamGrid pGrid = CvSVM::get_default_grid(CvSVM::P), - CvParamGrid nuGrid = CvSVM::get_default_grid(CvSVM::NU), - CvParamGrid coeffGrid = CvSVM::get_default_grid(CvSVM::COEF), - CvParamGrid degreeGrid = CvSVM::get_default_grid(CvSVM::DEGREE), - bool balanced=false); - CV_WRAP virtual float predict( const cv::Mat& sample, bool returnDFVal=false ) const; - CV_WRAP_AS(predict_all) virtual void predict( cv::InputArray samples, cv::OutputArray results ) const; - - CV_WRAP virtual int get_support_vector_count() const; - virtual const float* get_support_vector(int i) const; - virtual CvSVMParams get_params() const { return params; } - CV_WRAP virtual void clear(); - - virtual const CvSVMDecisionFunc* get_decision_function() const { return decision_func; } - - static CvParamGrid get_default_grid( int param_id ); - - virtual void write( CvFileStorage* storage, const char* name ) const; - virtual void read( CvFileStorage* storage, CvFileNode* node ); - CV_WRAP int get_var_count() const { return var_idx ? var_idx->cols : var_all; } - -protected: - - virtual bool set_params( const CvSVMParams& params ); - virtual bool train1( int sample_count, int var_count, const float** samples, - const void* responses, double Cp, double Cn, - CvMemStorage* _storage, double* alpha, double& rho ); - virtual bool do_train( int svm_type, int sample_count, int var_count, const float** samples, - const CvMat* responses, CvMemStorage* _storage, double* alpha ); - virtual void create_kernel(); - virtual void create_solver(); - - virtual float predict( const float* row_sample, int row_len, bool returnDFVal=false ) const; - - virtual void write_params( CvFileStorage* fs ) const; - virtual void read_params( CvFileStorage* fs, CvFileNode* node ); - - void optimize_linear_svm(); - - CvSVMParams params; - CvMat* class_labels; - int var_all; - float** sv; - int sv_total; - CvMat* var_idx; - CvMat* class_weights; - CvSVMDecisionFunc* decision_func; - CvMemStorage* storage; - - CvSVMSolver* solver; - CvSVMKernel* kernel; - -private: - CvSVM(const CvSVM&); - CvSVM& operator = (const CvSVM&); + virtual bool trainAuto( const Ptr& data, int kFold = 10, + ParamGrid Cgrid = SVM::getDefaultGrid(SVM::C), + ParamGrid gammaGrid = SVM::getDefaultGrid(SVM::GAMMA), + ParamGrid pGrid = SVM::getDefaultGrid(SVM::P), + ParamGrid nuGrid = SVM::getDefaultGrid(SVM::NU), + ParamGrid coeffGrid = SVM::getDefaultGrid(SVM::COEF), + ParamGrid degreeGrid = SVM::getDefaultGrid(SVM::DEGREE), + bool balanced=false) = 0; + + CV_WRAP virtual Mat getSupportVectors() const = 0; + + virtual void setParams(const Params& p, const Ptr& customKernel=Ptr()) = 0; + virtual Params getParams() const = 0; + virtual Ptr getKernel() const = 0; + virtual double getDecisionFunction(int i, OutputArray alpha, OutputArray svidx) const = 0; + + static ParamGrid getDefaultGrid( int param_id ); + static Ptr create(const Params& p=Params(), const Ptr& customKernel=Ptr()); }; /****************************************************************************************\ * Expectation - Maximization * \****************************************************************************************/ -namespace cv -{ -class CV_EXPORTS_W EM : public Algorithm +class CV_EXPORTS_W EM : public StatModel { public: // Type of covariation matrices @@ -579,1298 +322,204 @@ public: // The initial step enum {START_E_STEP=1, START_M_STEP=2, START_AUTO_STEP=0}; - CV_WRAP EM(int nclusters=EM::DEFAULT_NCLUSTERS, int covMatType=EM::COV_MAT_DIAGONAL, - const TermCriteria& termCrit=TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, - EM::DEFAULT_MAX_ITERS, FLT_EPSILON)); - - virtual ~EM(); - CV_WRAP virtual void clear(); - - CV_WRAP virtual bool train(InputArray samples, - OutputArray logLikelihoods=noArray(), - OutputArray labels=noArray(), - OutputArray probs=noArray()); - - CV_WRAP virtual bool trainE(InputArray samples, - InputArray means0, - InputArray covs0=noArray(), - InputArray weights0=noArray(), - OutputArray logLikelihoods=noArray(), - OutputArray labels=noArray(), - OutputArray probs=noArray()); - - CV_WRAP virtual bool trainM(InputArray samples, - InputArray probs0, - OutputArray logLikelihoods=noArray(), - OutputArray labels=noArray(), - OutputArray probs=noArray()); - - CV_WRAP Vec2d predict(InputArray sample, - OutputArray probs=noArray()) const; - - CV_WRAP bool isTrained() const; - - AlgorithmInfo* info() const; - virtual void read(const FileNode& fn); - -protected: - - virtual void setTrainData(int startStep, const Mat& samples, - const Mat* probs0, - const Mat* means0, - const std::vector* covs0, - const Mat* weights0); - - bool doTrain(int startStep, - OutputArray logLikelihoods, - OutputArray labels, - OutputArray probs); - virtual void eStep(); - virtual void mStep(); - - void clusterTrainSamples(); - void decomposeCovs(); - void computeLogWeightDivDet(); - - Vec2d computeProbabilities(const Mat& sample, Mat* probs) const; - - // all inner matrices have type CV_64FC1 - CV_PROP_RW int nclusters; - CV_PROP_RW int covMatType; - CV_PROP_RW int maxIters; - CV_PROP_RW double epsilon; - - Mat trainSamples; - Mat trainProbs; - Mat trainLogLikelihoods; - Mat trainLabels; - - CV_PROP Mat weights; - CV_PROP Mat means; - CV_PROP std::vector covs; - - std::vector covsEigenValues; - std::vector covsRotateMats; - std::vector invCovsEigenValues; - Mat logWeightDivDet; -}; -} // namespace cv - -/****************************************************************************************\ -* Decision Tree * -\****************************************************************************************/\ -struct CvPair16u32s -{ - unsigned short* u; - int* i; -}; - - -#define CV_DTREE_CAT_DIR(idx,subset) \ - (2*((subset[(idx)>>5]&(1 << ((idx) & 31)))==0)-1) - -struct CvDTreeSplit -{ - int var_idx; - int condensed_idx; - int inversed; - float quality; - CvDTreeSplit* next; - union + class CV_EXPORTS_W_MAP Params { - int subset[2]; - struct - { - float c; - int split_point; - } - ord; + public: + explicit Params(int nclusters=DEFAULT_NCLUSTERS, int covMatType=EM::COV_MAT_DIAGONAL, + const TermCriteria& termCrit=TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, + EM::DEFAULT_MAX_ITERS, 1e-6)); + CV_PROP_RW int nclusters; + CV_PROP_RW int covMatType; + CV_PROP_RW TermCriteria termCrit; }; -}; -struct CvDTreeNode -{ - int class_idx; - int Tn; - double value; - - CvDTreeNode* parent; - CvDTreeNode* left; - CvDTreeNode* right; - - CvDTreeSplit* split; - - int sample_count; - int depth; - int* num_valid; - int offset; - int buf_idx; - double maxlr; - - // global pruning data - int complexity; - double alpha; - double node_risk, tree_risk, tree_error; - - // cross-validation pruning data - int* cv_Tn; - double* cv_node_risk; - double* cv_node_error; - - int get_num_valid(int vi) { return num_valid ? num_valid[vi] : sample_count; } - void set_num_valid(int vi, int n) { if( num_valid ) num_valid[vi] = n; } -}; + virtual void setParams(const Params& p) = 0; + virtual Params getParams() const = 0; + virtual Mat getWeights() const = 0; + virtual Mat getMeans() const = 0; + virtual void getCovs(std::vector& covs) const = 0; + CV_WRAP virtual Vec2d predict2(InputArray sample, OutputArray probs) const = 0; -struct CV_EXPORTS_W_MAP CvDTreeParams -{ - CV_PROP_RW int max_categories; - CV_PROP_RW int max_depth; - CV_PROP_RW int min_sample_count; - CV_PROP_RW int cv_folds; - CV_PROP_RW bool use_surrogates; - CV_PROP_RW bool use_1se_rule; - CV_PROP_RW bool truncate_pruned_tree; - CV_PROP_RW float regression_accuracy; - const float* priors; - - CvDTreeParams(); - CvDTreeParams( int max_depth, int min_sample_count, - float regression_accuracy, bool use_surrogates, - int max_categories, int cv_folds, - bool use_1se_rule, bool truncate_pruned_tree, - const float* priors ); -}; + virtual bool train( const Ptr& trainData, int flags=0 ) = 0; + static Ptr train(InputArray samples, + OutputArray logLikelihoods=noArray(), + OutputArray labels=noArray(), + OutputArray probs=noArray(), + const Params& params=Params()); -struct CV_EXPORTS CvDTreeTrainData -{ - CvDTreeTrainData(); - CvDTreeTrainData( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - const CvDTreeParams& params=CvDTreeParams(), - bool _shared=false, bool _add_labels=false ); - virtual ~CvDTreeTrainData(); - - virtual void set_data( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - const CvDTreeParams& params=CvDTreeParams(), - bool _shared=false, bool _add_labels=false, - bool _update_data=false ); - virtual void do_responses_copy(); - - virtual void get_vectors( const CvMat* _subsample_idx, - float* values, uchar* missing, float* responses, bool get_class_idx=false ); - - virtual CvDTreeNode* subsample_data( const CvMat* _subsample_idx ); - - virtual void write_params( CvFileStorage* fs ) const; - virtual void read_params( CvFileStorage* fs, CvFileNode* node ); - - // release all the data - virtual void clear(); + static Ptr train_startWithE(InputArray samples, InputArray means0, + InputArray covs0=noArray(), + InputArray weights0=noArray(), + OutputArray logLikelihoods=noArray(), + OutputArray labels=noArray(), + OutputArray probs=noArray(), + const Params& params=Params()); - int get_num_classes() const; - int get_var_type(int vi) const; - int get_work_var_count() const {return work_var_count;} - - virtual const float* get_ord_responses( CvDTreeNode* n, float* values_buf, int* sample_indices_buf ); - virtual const int* get_class_labels( CvDTreeNode* n, int* labels_buf ); - virtual const int* get_cv_labels( CvDTreeNode* n, int* labels_buf ); - virtual const int* get_sample_indices( CvDTreeNode* n, int* indices_buf ); - virtual const int* get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf ); - virtual void get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* sorted_indices_buf, - const float** ord_values, const int** sorted_indices, int* sample_indices_buf ); - virtual int get_child_buf_idx( CvDTreeNode* n ); - - //////////////////////////////////// - - virtual bool set_params( const CvDTreeParams& params ); - virtual CvDTreeNode* new_node( CvDTreeNode* parent, int count, - int storage_idx, int offset ); - - virtual CvDTreeSplit* new_split_ord( int vi, float cmp_val, - int split_point, int inversed, float quality ); - virtual CvDTreeSplit* new_split_cat( int vi, float quality ); - virtual void free_node_data( CvDTreeNode* node ); - virtual void free_train_data(); - virtual void free_node( CvDTreeNode* node ); - - int sample_count, var_all, var_count, max_c_count; - int ord_var_count, cat_var_count, work_var_count; - bool have_labels, have_priors; - bool is_classifier; - int tflag; - - const CvMat* train_data; - const CvMat* responses; - CvMat* responses_copy; // used in Boosting - - int buf_count, buf_size; // buf_size is obsolete, please do not use it, use expression ((int64)buf->rows * (int64)buf->cols / buf_count) instead - bool shared; - int is_buf_16u; - - CvMat* cat_count; - CvMat* cat_ofs; - CvMat* cat_map; - - CvMat* counts; - CvMat* buf; - inline size_t get_length_subbuf() const - { - size_t res = (size_t)(work_var_count + 1) * (size_t)sample_count; - return res; - } - - CvMat* direction; - CvMat* split_buf; - - CvMat* var_idx; - CvMat* var_type; // i-th element = - // k<0 - ordered - // k>=0 - categorical, see k-th element of cat_* arrays - CvMat* priors; - CvMat* priors_mult; - - CvDTreeParams params; - - CvMemStorage* tree_storage; - CvMemStorage* temp_storage; - - CvDTreeNode* data_root; - - CvSet* node_heap; - CvSet* split_heap; - CvSet* cv_heap; - CvSet* nv_heap; - - cv::RNG* rng; -}; - -class CvDTree; -class CvForestTree; - -namespace cv -{ - struct DTreeBestSplitFinder; - struct ForestTreeBestSplitFinder; -} - -class CV_EXPORTS_W CvDTree : public CvStatModel -{ -public: - CV_WRAP CvDTree(); - virtual ~CvDTree(); - - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvDTreeParams params=CvDTreeParams() ); - - virtual bool train( CvMLData* trainData, CvDTreeParams params=CvDTreeParams() ); - - // type in {CV_TRAIN_ERROR, CV_TEST_ERROR} - virtual float calc_error( CvMLData* trainData, int type, std::vector *resp = 0 ); - - virtual bool train( CvDTreeTrainData* trainData, const CvMat* subsampleIdx ); - - virtual CvDTreeNode* predict( const CvMat* sample, const CvMat* missingDataMask=0, - bool preprocessedInput=false ) const; - - CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvDTreeParams params=CvDTreeParams() ); - - CV_WRAP virtual CvDTreeNode* predict( const cv::Mat& sample, const cv::Mat& missingDataMask=cv::Mat(), - bool preprocessedInput=false ) const; - CV_WRAP virtual cv::Mat getVarImportance(); - - virtual const CvMat* get_var_importance(); - CV_WRAP virtual void clear(); - - virtual void read( CvFileStorage* fs, CvFileNode* node ); - virtual void write( CvFileStorage* fs, const char* name ) const; - - // special read & write methods for trees in the tree ensembles - virtual void read( CvFileStorage* fs, CvFileNode* node, - CvDTreeTrainData* data ); - virtual void write( CvFileStorage* fs ) const; - - const CvDTreeNode* get_root() const; - int get_pruned_tree_idx() const; - CvDTreeTrainData* get_data(); - -protected: - friend struct cv::DTreeBestSplitFinder; - - virtual bool do_train( const CvMat* _subsample_idx ); - - virtual void try_split_node( CvDTreeNode* n ); - virtual void split_node_data( CvDTreeNode* n ); - virtual CvDTreeSplit* find_best_split( CvDTreeNode* n ); - virtual CvDTreeSplit* find_split_ord_class( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_cat_class( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_ord_reg( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_cat_reg( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_surrogate_split_ord( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_surrogate_split_cat( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); - virtual double calc_node_dir( CvDTreeNode* node ); - virtual void complete_node_dir( CvDTreeNode* node ); - virtual void cluster_categories( const int* vectors, int vector_count, - int var_count, int* sums, int k, int* cluster_labels ); - - virtual void calc_node_value( CvDTreeNode* node ); - - virtual void prune_cv(); - virtual double update_tree_rnc( int T, int fold ); - virtual int cut_tree( int T, int fold, double min_alpha ); - virtual void free_prune_data(bool cut_tree); - virtual void free_tree(); - - virtual void write_node( CvFileStorage* fs, CvDTreeNode* node ) const; - virtual void write_split( CvFileStorage* fs, CvDTreeSplit* split ) const; - virtual CvDTreeNode* read_node( CvFileStorage* fs, CvFileNode* node, CvDTreeNode* parent ); - virtual CvDTreeSplit* read_split( CvFileStorage* fs, CvFileNode* node ); - virtual void write_tree_nodes( CvFileStorage* fs ) const; - virtual void read_tree_nodes( CvFileStorage* fs, CvFileNode* node ); - - CvDTreeNode* root; - CvMat* var_importance; - CvDTreeTrainData* data; - CvMat train_data_hdr, responses_hdr; - cv::Mat train_data_mat, responses_mat; - -public: - int pruned_tree_idx; + static Ptr train_startWithM(InputArray samples, InputArray probs0, + OutputArray logLikelihoods=noArray(), + OutputArray labels=noArray(), + OutputArray probs=noArray(), + const Params& params=Params()); + static Ptr create(const Params& params=Params()); }; /****************************************************************************************\ -* Random Trees Classifier * +* Decision Tree * \****************************************************************************************/ -class CvRTrees; - -class CV_EXPORTS CvForestTree: public CvDTree +class CV_EXPORTS_W DTrees : public StatModel { public: - CvForestTree(); - virtual ~CvForestTree(); + enum { PREDICT_AUTO=0, PREDICT_SUM=(1<<8), PREDICT_MAX_VOTE=(2<<8), PREDICT_MASK=(3<<8) }; - virtual bool train( CvDTreeTrainData* trainData, const CvMat* _subsample_idx, CvRTrees* forest ); + class CV_EXPORTS_W_MAP Params + { + public: + Params(); + Params( int maxDepth, int minSampleCount, + double regressionAccuracy, bool useSurrogates, + int maxCategories, int CVFolds, + bool use1SERule, bool truncatePrunedTree, + const Mat& priors ); + + CV_PROP_RW int maxCategories; + CV_PROP_RW int maxDepth; + CV_PROP_RW int minSampleCount; + CV_PROP_RW int CVFolds; + CV_PROP_RW bool useSurrogates; + CV_PROP_RW bool use1SERule; + CV_PROP_RW bool truncatePrunedTree; + CV_PROP_RW float regressionAccuracy; + CV_PROP_RW Mat priors; + }; - virtual int get_var_count() const {return data ? data->var_count : 0;} - virtual void read( CvFileStorage* fs, CvFileNode* node, CvRTrees* forest, CvDTreeTrainData* _data ); + class CV_EXPORTS Node + { + public: + Node(); + double value; + int classIdx; - /* dummy methods to avoid warnings: BEGIN */ - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvDTreeParams params=CvDTreeParams() ); + int parent; + int left; + int right; + int defaultDir; - virtual bool train( CvDTreeTrainData* trainData, const CvMat* _subsample_idx ); - virtual void read( CvFileStorage* fs, CvFileNode* node ); - virtual void read( CvFileStorage* fs, CvFileNode* node, - CvDTreeTrainData* data ); - /* dummy methods to avoid warnings: END */ + int split; + }; -protected: - friend struct cv::ForestTreeBestSplitFinder; + class CV_EXPORTS Split + { + public: + Split(); + int varIdx; + bool inversed; + float quality; + int next; + float c; + int subsetOfs; + }; - virtual CvDTreeSplit* find_best_split( CvDTreeNode* n ); - CvRTrees* forest; -}; + virtual void setDParams(const Params& p); + virtual Params getDParams() const; + virtual const std::vector& getRoots() const = 0; + virtual const std::vector& getNodes() const = 0; + virtual const std::vector& getSplits() const = 0; + virtual const std::vector& getSubsets() const = 0; -struct CV_EXPORTS_W_MAP CvRTParams : public CvDTreeParams -{ - //Parameters for the forest - CV_PROP_RW bool calc_var_importance; // true <=> RF processes variable importance - CV_PROP_RW int nactive_vars; - CV_PROP_RW CvTermCriteria term_crit; - - CvRTParams(); - CvRTParams( int max_depth, int min_sample_count, - float regression_accuracy, bool use_surrogates, - int max_categories, const float* priors, bool calc_var_importance, - int nactive_vars, int max_num_of_trees_in_the_forest, - float forest_accuracy, int termcrit_type ); + static Ptr create(const Params& params=Params()); }; +/****************************************************************************************\ +* Random Trees Classifier * +\****************************************************************************************/ -class CV_EXPORTS_W CvRTrees : public CvStatModel +class CV_EXPORTS_W RTrees : public DTrees { public: - CV_WRAP CvRTrees(); - virtual ~CvRTrees(); - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvRTParams params=CvRTParams() ); - - virtual bool train( CvMLData* data, CvRTParams params=CvRTParams() ); - virtual float predict( const CvMat* sample, const CvMat* missing = 0 ) const; - virtual float predict_prob( const CvMat* sample, const CvMat* missing = 0 ) const; - - CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvRTParams params=CvRTParams() ); - CV_WRAP virtual float predict( const cv::Mat& sample, const cv::Mat& missing = cv::Mat() ) const; - CV_WRAP virtual float predict_prob( const cv::Mat& sample, const cv::Mat& missing = cv::Mat() ) const; - CV_WRAP virtual cv::Mat getVarImportance(); - - CV_WRAP virtual void clear(); - - virtual const CvMat* get_var_importance(); - virtual float get_proximity( const CvMat* sample1, const CvMat* sample2, - const CvMat* missing1 = 0, const CvMat* missing2 = 0 ) const; - - virtual float calc_error( CvMLData* data, int type , std::vector* resp = 0 ); // type in {CV_TRAIN_ERROR, CV_TEST_ERROR} - - virtual float get_train_error(); - - virtual void read( CvFileStorage* fs, CvFileNode* node ); - virtual void write( CvFileStorage* fs, const char* name ) const; - - CvMat* get_active_var_mask(); - CvRNG* get_rng(); - - int get_tree_count() const; - CvForestTree* get_tree(int i) const; - -protected: - virtual cv::String getName() const; - - virtual bool grow_forest( const CvTermCriteria term_crit ); - - // array of the trees of the forest - CvForestTree** trees; - CvDTreeTrainData* data; - CvMat train_data_hdr, responses_hdr; - cv::Mat train_data_mat, responses_mat; - int ntrees; - int nclasses; - double oob_error; - CvMat* var_importance; - int nsamples; - - cv::RNG* rng; - CvMat* active_var_mask; -}; + class CV_EXPORTS_W_MAP Params : public DTrees::Params + { + public: + Params(); + Params( int maxDepth, int minSampleCount, + double regressionAccuracy, bool useSurrogates, + int maxCategories, const Mat& priors, + bool calcVarImportance, int nactiveVars, + TermCriteria termCrit ); + + CV_PROP_RW bool calcVarImportance; // true <=> RF processes variable importance + CV_PROP_RW int nactiveVars; + CV_PROP_RW TermCriteria termCrit; + }; -/****************************************************************************************\ -* Extremely randomized trees Classifier * -\****************************************************************************************/ -struct CV_EXPORTS CvERTreeTrainData : public CvDTreeTrainData -{ - virtual void set_data( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - const CvDTreeParams& params=CvDTreeParams(), - bool _shared=false, bool _add_labels=false, - bool _update_data=false ); - virtual void get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* missing_buf, - const float** ord_values, const int** missing, int* sample_buf = 0 ); - virtual const int* get_sample_indices( CvDTreeNode* n, int* indices_buf ); - virtual const int* get_cv_labels( CvDTreeNode* n, int* labels_buf ); - virtual const int* get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf ); - virtual void get_vectors( const CvMat* _subsample_idx, float* values, uchar* missing, - float* responses, bool get_class_idx=false ); - virtual CvDTreeNode* subsample_data( const CvMat* _subsample_idx ); - const CvMat* missing_mask; -}; + virtual void setRParams(const Params& p) = 0; + virtual Params getRParams() const = 0; -class CV_EXPORTS CvForestERTree : public CvForestTree -{ -protected: - virtual double calc_node_dir( CvDTreeNode* node ); - virtual CvDTreeSplit* find_split_ord_class( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_cat_class( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_ord_reg( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_cat_reg( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual void split_node_data( CvDTreeNode* n ); -}; + virtual Mat getVarImportance() const = 0; -class CV_EXPORTS_W CvERTrees : public CvRTrees -{ -public: - CV_WRAP CvERTrees(); - virtual ~CvERTrees(); - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvRTParams params=CvRTParams()); - CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvRTParams params=CvRTParams()); - virtual bool train( CvMLData* data, CvRTParams params=CvRTParams() ); -protected: - virtual cv::String getName() const; - virtual bool grow_forest( const CvTermCriteria term_crit ); + static Ptr create(const Params& params=Params()); }; - /****************************************************************************************\ * Boosted tree classifier * \****************************************************************************************/ -struct CV_EXPORTS_W_MAP CvBoostParams : public CvDTreeParams -{ - CV_PROP_RW int boost_type; - CV_PROP_RW int weak_count; - CV_PROP_RW int split_criteria; - CV_PROP_RW double weight_trim_rate; - - CvBoostParams(); - CvBoostParams( int boost_type, int weak_count, double weight_trim_rate, - int max_depth, bool use_surrogates, const float* priors ); -}; - - -class CvBoost; - -class CV_EXPORTS CvBoostTree: public CvDTree +class CV_EXPORTS_W Boost : public DTrees { public: - CvBoostTree(); - virtual ~CvBoostTree(); - - virtual bool train( CvDTreeTrainData* trainData, - const CvMat* subsample_idx, CvBoost* ensemble ); - - virtual void scale( double s ); - virtual void read( CvFileStorage* fs, CvFileNode* node, - CvBoost* ensemble, CvDTreeTrainData* _data ); - virtual void clear(); - - /* dummy methods to avoid warnings: BEGIN */ - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvDTreeParams params=CvDTreeParams() ); - virtual bool train( CvDTreeTrainData* trainData, const CvMat* _subsample_idx ); - - virtual void read( CvFileStorage* fs, CvFileNode* node ); - virtual void read( CvFileStorage* fs, CvFileNode* node, - CvDTreeTrainData* data ); - /* dummy methods to avoid warnings: END */ - -protected: - - virtual void try_split_node( CvDTreeNode* n ); - virtual CvDTreeSplit* find_surrogate_split_ord( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_surrogate_split_cat( CvDTreeNode* n, int vi, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_ord_class( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_cat_class( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_ord_reg( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual CvDTreeSplit* find_split_cat_reg( CvDTreeNode* n, int vi, - float init_quality = 0, CvDTreeSplit* _split = 0, uchar* ext_buf = 0 ); - virtual void calc_node_value( CvDTreeNode* n ); - virtual double calc_node_dir( CvDTreeNode* n ); - - CvBoost* ensemble; -}; - + class CV_EXPORTS_W_MAP Params : public DTrees::Params + { + public: + CV_PROP_RW int boostType; + CV_PROP_RW int weakCount; + CV_PROP_RW double weightTrimRate; + + Params(); + Params( int boostType, int weakCount, double weightTrimRate, + int maxDepth, bool useSurrogates, const Mat& priors ); + }; -class CV_EXPORTS_W CvBoost : public CvStatModel -{ -public: // Boosting type enum { DISCRETE=0, REAL=1, LOGIT=2, GENTLE=3 }; - // Splitting criteria - enum { DEFAULT=0, GINI=1, MISCLASS=3, SQERR=4 }; - - CV_WRAP CvBoost(); - virtual ~CvBoost(); - - CvBoost( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvBoostParams params=CvBoostParams() ); - - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvBoostParams params=CvBoostParams(), - bool update=false ); - - virtual bool train( CvMLData* data, - CvBoostParams params=CvBoostParams(), - bool update=false ); - - virtual float predict( const CvMat* sample, const CvMat* missing=0, - CvMat* weak_responses=0, CvSlice slice=CV_WHOLE_SEQ, - bool raw_mode=false, bool return_sum=false ) const; - - CV_WRAP CvBoost( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvBoostParams params=CvBoostParams() ); - - CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvBoostParams params=CvBoostParams(), - bool update=false ); - - CV_WRAP virtual float predict( const cv::Mat& sample, const cv::Mat& missing=cv::Mat(), - const cv::Range& slice=cv::Range::all(), bool rawMode=false, - bool returnSum=false ) const; - - virtual float calc_error( CvMLData* _data, int type , std::vector *resp = 0 ); // type in {CV_TRAIN_ERROR, CV_TEST_ERROR} - - CV_WRAP virtual void prune( CvSlice slice ); - - CV_WRAP virtual void clear(); - - virtual void write( CvFileStorage* storage, const char* name ) const; - virtual void read( CvFileStorage* storage, CvFileNode* node ); - virtual const CvMat* get_active_vars(bool absolute_idx=true); - - CvSeq* get_weak_predictors(); - - CvMat* get_weights(); - CvMat* get_subtree_weights(); - CvMat* get_weak_response(); - const CvBoostParams& get_params() const; - const CvDTreeTrainData* get_data() const; - -protected: - - virtual bool set_params( const CvBoostParams& params ); - virtual void update_weights( CvBoostTree* tree ); - virtual void trim_weights(); - virtual void write_params( CvFileStorage* fs ) const; - virtual void read_params( CvFileStorage* fs, CvFileNode* node ); - - virtual void initialize_weights(double (&p)[2]); - - CvDTreeTrainData* data; - CvMat train_data_hdr, responses_hdr; - cv::Mat train_data_mat, responses_mat; - CvBoostParams params; - CvSeq* weak; - - CvMat* active_vars; - CvMat* active_vars_abs; - bool have_active_cat_vars; - - CvMat* orig_response; - CvMat* sum_response; - CvMat* weak_eval; - CvMat* subsample_mask; - CvMat* weights; - CvMat* subtree_weights; - bool have_subsample; -}; + virtual Params getBParams() const = 0; + virtual void setBParams(const Params& p) = 0; + static Ptr create(const Params& params=Params()); +}; /****************************************************************************************\ * Gradient Boosted Trees * \****************************************************************************************/ -// DataType: STRUCT CvGBTreesParams -// Parameters of GBT (Gradient Boosted trees model), including single -// tree settings and ensemble parameters. -// -// weak_count - count of trees in the ensemble -// loss_function_type - loss function used for ensemble training -// subsample_portion - portion of whole training set used for -// every single tree training. -// subsample_portion value is in (0.0, 1.0]. -// subsample_portion == 1.0 when whole dataset is -// used on each step. Count of sample used on each -// step is computed as -// int(total_samples_count * subsample_portion). -// shrinkage - regularization parameter. -// Each tree prediction is multiplied on shrinkage value. - - -struct CV_EXPORTS_W_MAP CvGBTreesParams : public CvDTreeParams -{ - CV_PROP_RW int weak_count; - CV_PROP_RW int loss_function_type; - CV_PROP_RW float subsample_portion; - CV_PROP_RW float shrinkage; - - CvGBTreesParams(); - CvGBTreesParams( int loss_function_type, int weak_count, float shrinkage, - float subsample_portion, int max_depth, bool use_surrogates ); -}; - -// DataType: CLASS CvGBTrees -// Gradient Boosting Trees (GBT) algorithm implementation. -// -// data - training dataset -// params - parameters of the CvGBTrees -// weak - array[0..(class_count-1)] of CvSeq -// for storing tree ensembles -// orig_response - original responses of the training set samples -// sum_response - predicitons of the current model on the training dataset. -// this matrix is updated on every iteration. -// sum_response_tmp - predicitons of the model on the training set on the next -// step. On every iteration values of sum_responses_tmp are -// computed via sum_responses values. When the current -// step is complete sum_response values become equal to -// sum_responses_tmp. -// sampleIdx - indices of samples used for training the ensemble. -// CvGBTrees training procedure takes a set of samples -// (train_data) and a set of responses (responses). -// Only pairs (train_data[i], responses[i]), where i is -// in sample_idx are used for training the ensemble. -// subsample_train - indices of samples used for training a single decision -// tree on the current step. This indices are countered -// relatively to the sample_idx, so that pairs -// (train_data[sample_idx[i]], responses[sample_idx[i]]) -// are used for training a decision tree. -// Training set is randomly splited -// in two parts (subsample_train and subsample_test) -// on every iteration accordingly to the portion parameter. -// subsample_test - relative indices of samples from the training set, -// which are not used for training a tree on the current -// step. -// missing - mask of the missing values in the training set. This -// matrix has the same size as train_data. 1 - missing -// value, 0 - not a missing value. -// class_labels - output class labels map. -// rng - random number generator. Used for spliting the -// training set. -// class_count - count of output classes. -// class_count == 1 in the case of regression, -// and > 1 in the case of classification. -// delta - Huber loss function parameter. -// base_value - start point of the gradient descent procedure. -// model prediction is -// f(x) = f_0 + sum_{i=1..weak_count-1}(f_i(x)), where -// f_0 is the base value. - - - -class CV_EXPORTS_W CvGBTrees : public CvStatModel +/*class CV_EXPORTS_W GBTrees : public DTrees { public: + struct CV_EXPORTS_W_MAP Params : public DTrees::Params + { + CV_PROP_RW int weakCount; + CV_PROP_RW int lossFunctionType; + CV_PROP_RW float subsamplePortion; + CV_PROP_RW float shrinkage; + + Params(); + Params( int lossFunctionType, int weakCount, float shrinkage, + float subsamplePortion, int maxDepth, bool useSurrogates ); + }; - /* - // DataType: ENUM - // Loss functions implemented in CvGBTrees. - // - // SQUARED_LOSS - // problem: regression - // loss = (x - x')^2 - // - // ABSOLUTE_LOSS - // problem: regression - // loss = abs(x - x') - // - // HUBER_LOSS - // problem: regression - // loss = delta*( abs(x - x') - delta/2), if abs(x - x') > delta - // 1/2*(x - x')^2, if abs(x - x') <= delta, - // where delta is the alpha-quantile of pseudo responses from - // the training set. - // - // DEVIANCE_LOSS - // problem: classification - // - */ enum {SQUARED_LOSS=0, ABSOLUTE_LOSS, HUBER_LOSS=3, DEVIANCE_LOSS}; + virtual void setK(int k) = 0; - /* - // Default constructor. Creates a model only (without training). - // Should be followed by one form of the train(...) function. - // - // API - // CvGBTrees(); - - // INPUT - // OUTPUT - // RESULT - */ - CV_WRAP CvGBTrees(); - - - /* - // Full form constructor. Creates a gradient boosting model and does the - // train. - // - // API - // CvGBTrees( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvGBTreesParams params=CvGBTreesParams() ); - - // INPUT - // trainData - a set of input feature vectors. - // size of matrix is - // x - // or x - // depending on the tflag parameter. - // matrix values are float. - // tflag - a flag showing how do samples stored in the - // trainData matrix row by row (tflag=CV_ROW_SAMPLE) - // or column by column (tflag=CV_COL_SAMPLE). - // responses - a vector of responses corresponding to the samples - // in trainData. - // varIdx - indices of used variables. zero value means that all - // variables are active. - // sampleIdx - indices of used samples. zero value means that all - // samples from trainData are in the training set. - // varType - vector of length. gives every - // variable type CV_VAR_CATEGORICAL or CV_VAR_ORDERED. - // varType = 0 means all variables are numerical. - // missingDataMask - a mask of misiing values in trainData. - // missingDataMask = 0 means that there are no missing - // values. - // params - parameters of GTB algorithm. - // OUTPUT - // RESULT - */ - CvGBTrees( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvGBTreesParams params=CvGBTreesParams() ); - - - /* - // Destructor. - */ - virtual ~CvGBTrees(); - - - /* - // Gradient tree boosting model training - // - // API - // virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvGBTreesParams params=CvGBTreesParams(), - bool update=false ); - - // INPUT - // trainData - a set of input feature vectors. - // size of matrix is - // x - // or x - // depending on the tflag parameter. - // matrix values are float. - // tflag - a flag showing how do samples stored in the - // trainData matrix row by row (tflag=CV_ROW_SAMPLE) - // or column by column (tflag=CV_COL_SAMPLE). - // responses - a vector of responses corresponding to the samples - // in trainData. - // varIdx - indices of used variables. zero value means that all - // variables are active. - // sampleIdx - indices of used samples. zero value means that all - // samples from trainData are in the training set. - // varType - vector of length. gives every - // variable type CV_VAR_CATEGORICAL or CV_VAR_ORDERED. - // varType = 0 means all variables are numerical. - // missingDataMask - a mask of misiing values in trainData. - // missingDataMask = 0 means that there are no missing - // values. - // params - parameters of GTB algorithm. - // update - is not supported now. (!) - // OUTPUT - // RESULT - // Error state. - */ - virtual bool train( const CvMat* trainData, int tflag, - const CvMat* responses, const CvMat* varIdx=0, - const CvMat* sampleIdx=0, const CvMat* varType=0, - const CvMat* missingDataMask=0, - CvGBTreesParams params=CvGBTreesParams(), - bool update=false ); - - - /* - // Gradient tree boosting model training - // - // API - // virtual bool train( CvMLData* data, - CvGBTreesParams params=CvGBTreesParams(), - bool update=false ) {return false;} - - // INPUT - // data - training set. - // params - parameters of GTB algorithm. - // update - is not supported now. (!) - // OUTPUT - // RESULT - // Error state. - */ - virtual bool train( CvMLData* data, - CvGBTreesParams params=CvGBTreesParams(), - bool update=false ); - - - /* - // Response value prediction - // - // API - // virtual float predict_serial( const CvMat* sample, const CvMat* missing=0, - CvMat* weak_responses=0, CvSlice slice = CV_WHOLE_SEQ, - int k=-1 ) const; - - // INPUT - // sample - input sample of the same type as in the training set. - // missing - missing values mask. missing=0 if there are no - // missing values in sample vector. - // weak_responses - predictions of all of the trees. - // not implemented (!) - // slice - part of the ensemble used for prediction. - // slice = CV_WHOLE_SEQ when all trees are used. - // k - number of ensemble used. - // k is in {-1,0,1,..,}. - // in the case of classification problem - // ensembles are built. - // If k = -1 ordinary prediction is the result, - // otherwise function gives the prediction of the - // k-th ensemble only. - // OUTPUT - // RESULT - // Predicted value. - */ - virtual float predict_serial( const CvMat* sample, const CvMat* missing=0, - CvMat* weakResponses=0, CvSlice slice = CV_WHOLE_SEQ, - int k=-1 ) const; - - /* - // Response value prediction. - // Parallel version (in the case of TBB existence) - // - // API - // virtual float predict( const CvMat* sample, const CvMat* missing=0, - CvMat* weak_responses=0, CvSlice slice = CV_WHOLE_SEQ, - int k=-1 ) const; - - // INPUT - // sample - input sample of the same type as in the training set. - // missing - missing values mask. missing=0 if there are no - // missing values in sample vector. - // weak_responses - predictions of all of the trees. - // not implemented (!) - // slice - part of the ensemble used for prediction. - // slice = CV_WHOLE_SEQ when all trees are used. - // k - number of ensemble used. - // k is in {-1,0,1,..,}. - // in the case of classification problem - // ensembles are built. - // If k = -1 ordinary prediction is the result, - // otherwise function gives the prediction of the - // k-th ensemble only. - // OUTPUT - // RESULT - // Predicted value. - */ - virtual float predict( const CvMat* sample, const CvMat* missing=0, - CvMat* weakResponses=0, CvSlice slice = CV_WHOLE_SEQ, - int k=-1 ) const; - - /* - // Deletes all the data. - // - // API - // virtual void clear(); - - // INPUT - // OUTPUT - // delete data, weak, orig_response, sum_response, - // weak_eval, subsample_train, subsample_test, - // sample_idx, missing, lass_labels - // delta = 0.0 - // RESULT - */ - CV_WRAP virtual void clear(); - - /* - // Compute error on the train/test set. - // - // API - // virtual float calc_error( CvMLData* _data, int type, - // std::vector *resp = 0 ); - // - // INPUT - // data - dataset - // type - defines which error is to compute: train (CV_TRAIN_ERROR) or - // test (CV_TEST_ERROR). - // OUTPUT - // resp - vector of predicitons - // RESULT - // Error value. - */ - virtual float calc_error( CvMLData* _data, int type, - std::vector *resp = 0 ); - - /* - // - // Write parameters of the gtb model and data. Write learned model. - // - // API - // virtual void write( CvFileStorage* fs, const char* name ) const; - // - // INPUT - // fs - file storage to read parameters from. - // name - model name. - // OUTPUT - // RESULT - */ - virtual void write( CvFileStorage* fs, const char* name ) const; - - - /* - // - // Read parameters of the gtb model and data. Read learned model. - // - // API - // virtual void read( CvFileStorage* fs, CvFileNode* node ); - // - // INPUT - // fs - file storage to read parameters from. - // node - file node. - // OUTPUT - // RESULT - */ - virtual void read( CvFileStorage* fs, CvFileNode* node ); - - - // new-style C++ interface - CV_WRAP CvGBTrees( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvGBTreesParams params=CvGBTreesParams() ); - - CV_WRAP virtual bool train( const cv::Mat& trainData, int tflag, - const cv::Mat& responses, const cv::Mat& varIdx=cv::Mat(), - const cv::Mat& sampleIdx=cv::Mat(), const cv::Mat& varType=cv::Mat(), - const cv::Mat& missingDataMask=cv::Mat(), - CvGBTreesParams params=CvGBTreesParams(), - bool update=false ); - - CV_WRAP virtual float predict( const cv::Mat& sample, const cv::Mat& missing=cv::Mat(), - const cv::Range& slice = cv::Range::all(), - int k=-1 ) const; - -protected: - - /* - // Compute the gradient vector components. - // - // API - // virtual void find_gradient( const int k = 0); - - // INPUT - // k - used for classification problem, determining current - // tree ensemble. - // OUTPUT - // changes components of data->responses - // which correspond to samples used for training - // on the current step. - // RESULT - */ - virtual void find_gradient( const int k = 0); - - - /* - // - // Change values in tree leaves according to the used loss function. - // - // API - // virtual void change_values(CvDTree* tree, const int k = 0); - // - // INPUT - // tree - decision tree to change. - // k - used for classification problem, determining current - // tree ensemble. - // OUTPUT - // changes 'value' fields of the trees' leaves. - // changes sum_response_tmp. - // RESULT - */ - virtual void change_values(CvDTree* tree, const int k = 0); - - - /* - // - // Find optimal constant prediction value according to the used loss - // function. - // The goal is to find a constant which gives the minimal summary loss - // on the _Idx samples. - // - // API - // virtual float find_optimal_value( const CvMat* _Idx ); - // - // INPUT - // _Idx - indices of the samples from the training set. - // OUTPUT - // RESULT - // optimal constant value. - */ - virtual float find_optimal_value( const CvMat* _Idx ); - - - /* - // - // Randomly split the whole training set in two parts according - // to params.portion. - // - // API - // virtual void do_subsample(); - // - // INPUT - // OUTPUT - // subsample_train - indices of samples used for training - // subsample_test - indices of samples used for test - // RESULT - */ - virtual void do_subsample(); - - - /* - // - // Internal recursive function giving an array of subtree tree leaves. - // - // API - // void leaves_get( CvDTreeNode** leaves, int& count, CvDTreeNode* node ); - // - // INPUT - // node - current leaf. - // OUTPUT - // count - count of leaves in the subtree. - // leaves - array of pointers to leaves. - // RESULT - */ - void leaves_get( CvDTreeNode** leaves, int& count, CvDTreeNode* node ); - - - /* - // - // Get leaves of the tree. - // - // API - // CvDTreeNode** GetLeaves( const CvDTree* dtree, int& len ); - // - // INPUT - // dtree - decision tree. - // OUTPUT - // len - count of the leaves. - // RESULT - // CvDTreeNode** - array of pointers to leaves. - */ - CvDTreeNode** GetLeaves( const CvDTree* dtree, int& len ); - - - /* - // - // Is it a regression or a classification. - // - // API - // bool problem_type(); - // - // INPUT - // OUTPUT - // RESULT - // false if it is a classification problem, - // true - if regression. - */ - virtual bool problem_type() const; - - - /* - // - // Write parameters of the gtb model. - // - // API - // virtual void write_params( CvFileStorage* fs ) const; - // - // INPUT - // fs - file storage to write parameters to. - // OUTPUT - // RESULT - */ - virtual void write_params( CvFileStorage* fs ) const; - - - /* - // - // Read parameters of the gtb model and data. - // - // API - // virtual void read_params( CvFileStorage* fs ); - // - // INPUT - // fs - file storage to read parameters from. - // OUTPUT - // params - parameters of the gtb model. - // data - contains information about the structure - // of the data set (count of variables, - // their types, etc.). - // class_labels - output class labels map. - // RESULT - */ - virtual void read_params( CvFileStorage* fs, CvFileNode* fnode ); - int get_len(const CvMat* mat) const; - - - CvDTreeTrainData* data; - CvGBTreesParams params; - - CvSeq** weak; - CvMat* orig_response; - CvMat* sum_response; - CvMat* sum_response_tmp; - CvMat* sample_idx; - CvMat* subsample_train; - CvMat* subsample_test; - CvMat* missing; - CvMat* class_labels; - - cv::RNG* rng; - - int class_count; - float delta; - float base_value; - -}; - + virtual float predictSerial( InputArray samples, + OutputArray weakResponses, int flags) const = 0; + static Ptr create(const Params& p); +};*/ /****************************************************************************************\ * Artificial Neural Networks (ANN) * @@ -1878,62 +527,31 @@ protected: /////////////////////////////////// Multi-Layer Perceptrons ////////////////////////////// -struct CV_EXPORTS_W_MAP CvANN_MLP_TrainParams -{ - CvANN_MLP_TrainParams(); - CvANN_MLP_TrainParams( CvTermCriteria term_crit, int train_method, - double param1, double param2=0 ); - ~CvANN_MLP_TrainParams(); - - enum { BACKPROP=0, RPROP=1 }; - - CV_PROP_RW CvTermCriteria term_crit; - CV_PROP_RW int train_method; - - // backpropagation parameters - CV_PROP_RW double bp_dw_scale, bp_moment_scale; - - // rprop parameters - CV_PROP_RW double rp_dw0, rp_dw_plus, rp_dw_minus, rp_dw_min, rp_dw_max; -}; - - -class CV_EXPORTS_W CvANN_MLP : public CvStatModel +class CV_EXPORTS_W ANN_MLP : public StatModel { public: - CV_WRAP CvANN_MLP(); - CvANN_MLP( const CvMat* layerSizes, - int activateFunc=CvANN_MLP::SIGMOID_SYM, - double fparam1=0, double fparam2=0 ); - - virtual ~CvANN_MLP(); - - virtual void create( const CvMat* layerSizes, - int activateFunc=CvANN_MLP::SIGMOID_SYM, - double fparam1=0, double fparam2=0 ); - - virtual int train( const CvMat* inputs, const CvMat* outputs, - const CvMat* sampleWeights, const CvMat* sampleIdx=0, - CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(), - int flags=0 ); - virtual float predict( const CvMat* inputs, CV_OUT CvMat* outputs ) const; + struct CV_EXPORTS_W_MAP Params + { + Params(); + Params( const Mat& layerSizes, int activateFunc, double fparam1, double fparam2, + TermCriteria termCrit, int trainMethod, double param1, double param2=0 ); - CV_WRAP CvANN_MLP( const cv::Mat& layerSizes, - int activateFunc=CvANN_MLP::SIGMOID_SYM, - double fparam1=0, double fparam2=0 ); + enum { BACKPROP=0, RPROP=1 }; - CV_WRAP virtual void create( const cv::Mat& layerSizes, - int activateFunc=CvANN_MLP::SIGMOID_SYM, - double fparam1=0, double fparam2=0 ); + CV_PROP_RW Mat layerSizes; + CV_PROP_RW int activateFunc; + CV_PROP_RW double fparam1; + CV_PROP_RW double fparam2; - CV_WRAP virtual int train( const cv::Mat& inputs, const cv::Mat& outputs, - const cv::Mat& sampleWeights, const cv::Mat& sampleIdx=cv::Mat(), - CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(), - int flags=0 ); + CV_PROP_RW TermCriteria termCrit; + CV_PROP_RW int trainMethod; - CV_WRAP virtual float predict( const cv::Mat& inputs, CV_OUT cv::Mat& outputs ) const; + // backpropagation parameters + CV_PROP_RW double bpDWScale, bpMomentScale; - CV_WRAP virtual void clear(); + // rprop parameters + CV_PROP_RW double rpDW0, rpDWPlus, rpDWMinus, rpDWMin, rpDWMax; + }; // possible activation functions enum { IDENTITY = 0, SIGMOID_SYM = 1, GAUSSIAN = 2 }; @@ -1941,53 +559,11 @@ public: // available training flags enum { UPDATE_WEIGHTS = 1, NO_INPUT_SCALE = 2, NO_OUTPUT_SCALE = 4 }; - virtual void read( CvFileStorage* fs, CvFileNode* node ); - virtual void write( CvFileStorage* storage, const char* name ) const; + virtual Mat getWeights(int layerIdx) const = 0; + virtual void setParams(const Params& p) = 0; + virtual Params getParams() const = 0; - int get_layer_count() { return layer_sizes ? layer_sizes->cols : 0; } - const CvMat* get_layer_sizes() { return layer_sizes; } - double* get_weights(int layer) - { - return layer_sizes && weights && - (unsigned)layer <= (unsigned)layer_sizes->cols ? weights[layer] : 0; - } - - virtual void calc_activ_func_deriv( CvMat* xf, CvMat* deriv, const double* bias ) const; - -protected: - - virtual bool prepare_to_train( const CvMat* _inputs, const CvMat* _outputs, - const CvMat* _sample_weights, const CvMat* sampleIdx, - CvVectors* _ivecs, CvVectors* _ovecs, double** _sw, int _flags ); - - // sequential random backpropagation - virtual int train_backprop( CvVectors _ivecs, CvVectors _ovecs, const double* _sw ); - - // RPROP algorithm - virtual int train_rprop( CvVectors _ivecs, CvVectors _ovecs, const double* _sw ); - - virtual void calc_activ_func( CvMat* xf, const double* bias ) const; - virtual void set_activ_func( int _activ_func=SIGMOID_SYM, - double _f_param1=0, double _f_param2=0 ); - virtual void init_weights(); - virtual void scale_input( const CvMat* _src, CvMat* _dst ) const; - virtual void scale_output( const CvMat* _src, CvMat* _dst ) const; - virtual void calc_input_scale( const CvVectors* vecs, int flags ); - virtual void calc_output_scale( const CvVectors* vecs, int flags ); - - virtual void write_params( CvFileStorage* fs ) const; - virtual void read_params( CvFileStorage* fs, CvFileNode* node ); - - CvMat* layer_sizes; - CvMat* wbuf; - CvMat* sample_weights; - double** weights; - double f_param1, f_param2; - double min_val, max_val, min_val1, max_val1; - int activ_func; - int max_count, max_buf_sz; - CvANN_MLP_TrainParams params; - cv::RNG* rng; + static Ptr create(const Params& params=Params()); }; /****************************************************************************************\ @@ -1996,167 +572,17 @@ protected: /* Generates from multivariate normal distribution, where - is an average row vector, - symmetric covariation matrix */ -CVAPI(void) cvRandMVNormal( CvMat* mean, CvMat* cov, CvMat* sample, - CvRNG* rng CV_DEFAULT(0) ); +CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, OutputArray samples); /* Generates sample from gaussian mixture distribution */ -CVAPI(void) cvRandGaussMixture( CvMat* means[], - CvMat* covs[], - float weights[], - int clsnum, - CvMat* sample, - CvMat* sampClasses CV_DEFAULT(0) ); - -#define CV_TS_CONCENTRIC_SPHERES 0 +CV_EXPORTS void randGaussMixture( InputArray means, InputArray covs, InputArray weights, + int nsamples, OutputArray samples, OutputArray sampClasses ); /* creates test set */ -CVAPI(void) cvCreateTestSet( int type, CvMat** samples, - int num_samples, - int num_features, - CvMat** responses, - int num_classes, ... ); +CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses, + OutputArray samples, OutputArray responses); -/****************************************************************************************\ -* Data * -\****************************************************************************************/ - -#define CV_COUNT 0 -#define CV_PORTION 1 - -struct CV_EXPORTS CvTrainTestSplit -{ - CvTrainTestSplit(); - CvTrainTestSplit( int train_sample_count, bool mix = true); - CvTrainTestSplit( float train_sample_portion, bool mix = true); - - union - { - int count; - float portion; - } train_sample_part; - int train_sample_part_mode; - - bool mix; -}; - -class CV_EXPORTS CvMLData -{ -public: - CvMLData(); - virtual ~CvMLData(); - - // returns: - // 0 - OK - // -1 - file can not be opened or is not correct - int read_csv( const char* filename ); - - const CvMat* get_values() const; - const CvMat* get_responses(); - const CvMat* get_missing() const; - - void set_header_lines_number( int n ); - int get_header_lines_number() const; - - void set_response_idx( int idx ); // old response become predictors, new response_idx = idx - // if idx < 0 there will be no response - int get_response_idx() const; - - void set_train_test_split( const CvTrainTestSplit * spl ); - const CvMat* get_train_sample_idx() const; - const CvMat* get_test_sample_idx() const; - void mix_train_and_test_idx(); - - const CvMat* get_var_idx(); - void chahge_var_idx( int vi, bool state ); // misspelled (saved for back compitability), - // use change_var_idx - void change_var_idx( int vi, bool state ); // state == true to set vi-variable as predictor - - const CvMat* get_var_types(); - int get_var_type( int var_idx ) const; - // following 2 methods enable to change vars type - // use these methods to assign CV_VAR_CATEGORICAL type for categorical variable - // with numerical labels; in the other cases var types are correctly determined automatically - void set_var_types( const char* str ); // str examples: - // "ord[0-17],cat[18]", "ord[0,2,4,10-12], cat[1,3,5-9,13,14]", - // "cat", "ord" (all vars are categorical/ordered) - void change_var_type( int var_idx, int type); // type in { CV_VAR_ORDERED, CV_VAR_CATEGORICAL } - - void set_delimiter( char ch ); - char get_delimiter() const; - - void set_miss_ch( char ch ); - char get_miss_ch() const; - - const std::map& get_class_labels_map() const; - -protected: - virtual void clear(); - - void str_to_flt_elem( const char* token, float& flt_elem, int& type); - void free_train_test_idx(); - - char delimiter; - char miss_ch; - //char flt_separator; - - CvMat* values; - CvMat* missing; - CvMat* var_types; - CvMat* var_idx_mask; - - CvMat* response_out; // header - CvMat* var_idx_out; // mat - CvMat* var_types_out; // mat - - int header_lines_number; - - int response_idx; - - int train_sample_count; - bool mix; - - int total_class_count; - std::map class_map; - - CvMat* train_sample_idx; - CvMat* test_sample_idx; - int* sample_idx; // data of train_sample_idx and test_sample_idx - - cv::RNG* rng; -}; - - -namespace cv -{ - -typedef CvStatModel StatModel; -typedef CvParamGrid ParamGrid; -typedef CvNormalBayesClassifier NormalBayesClassifier; -typedef CvKNearest KNearest; -typedef CvSVMParams SVMParams; -typedef CvSVMKernel SVMKernel; -typedef CvSVMSolver SVMSolver; -typedef CvSVM SVM; -typedef CvDTreeParams DTreeParams; -typedef CvMLData TrainData; -typedef CvDTree DecisionTree; -typedef CvForestTree ForestTree; -typedef CvRTParams RandomTreeParams; -typedef CvRTrees RandomTrees; -typedef CvERTreeTrainData ERTreeTRainData; -typedef CvForestERTree ERTree; -typedef CvERTrees ERTrees; -typedef CvBoostParams BoostParams; -typedef CvBoostTree BoostTree; -typedef CvBoost Boost; -typedef CvANN_MLP_TrainParams ANN_MLP_TrainParams; -typedef CvANN_MLP NeuralNet_MLP; -typedef CvGBTreesParams GradientBoostingTreeParams; -typedef CvGBTrees GradientBoostingTrees; - -template<> CV_EXPORTS void DefaultDeleter::operator ()(CvDTreeSplit* obj) const; - -CV_EXPORTS bool initModule_ml(void); +} } #endif // __cplusplus diff --git a/modules/ml/src/ann_mlp.cpp b/modules/ml/src/ann_mlp.cpp index 7323ab57a708ec79f17cb1ab13254ca47091ce6c..73af1ae94d0f45d65a3997dc16472120b60ff454 100644 --- a/modules/ml/src/ann_mlp.cpp +++ b/modules/ml/src/ann_mlp.cpp @@ -40,1579 +40,1281 @@ #include "precomp.hpp" -CvANN_MLP_TrainParams::CvANN_MLP_TrainParams() +namespace cv { namespace ml { + +ANN_MLP::Params::Params() { - term_crit = cvTermCriteria( CV_TERMCRIT_ITER + CV_TERMCRIT_EPS, 1000, 0.01 ); - train_method = RPROP; - bp_dw_scale = bp_moment_scale = 0.1; - rp_dw0 = 0.1; rp_dw_plus = 1.2; rp_dw_minus = 0.5; - rp_dw_min = FLT_EPSILON; rp_dw_max = 50.; + layerSizes = Mat(); + activateFunc = SIGMOID_SYM; + fparam1 = fparam2 = 0; + termCrit = TermCriteria( TermCriteria::COUNT + TermCriteria::EPS, 1000, 0.01 ); + trainMethod = RPROP; + bpDWScale = bpMomentScale = 0.1; + rpDW0 = 0.1; rpDWPlus = 1.2; rpDWMinus = 0.5; + rpDWMin = FLT_EPSILON; rpDWMax = 50.; } -CvANN_MLP_TrainParams::CvANN_MLP_TrainParams( CvTermCriteria _term_crit, - int _train_method, - double _param1, double _param2 ) +ANN_MLP::Params::Params( const Mat& _layerSizes, int _activateFunc, double _fparam1, double _fparam2, + TermCriteria _termCrit, int _trainMethod, double _param1, double _param2 ) { - term_crit = _term_crit; - train_method = _train_method; - bp_dw_scale = bp_moment_scale = 0.1; - rp_dw0 = 1.; rp_dw_plus = 1.2; rp_dw_minus = 0.5; - rp_dw_min = FLT_EPSILON; rp_dw_max = 50.; - - if( train_method == RPROP ) + layerSizes = _layerSizes; + activateFunc = _activateFunc; + fparam1 = _fparam1; + fparam2 = _fparam2; + termCrit = _termCrit; + trainMethod = _trainMethod; + bpDWScale = bpMomentScale = 0.1; + rpDW0 = 1.; rpDWPlus = 1.2; rpDWMinus = 0.5; + rpDWMin = FLT_EPSILON; rpDWMax = 50.; + + if( trainMethod == RPROP ) { - rp_dw0 = _param1; - if( rp_dw0 < FLT_EPSILON ) - rp_dw0 = 1.; - rp_dw_min = _param2; - rp_dw_min = MAX( rp_dw_min, 0 ); + rpDW0 = _param1; + if( rpDW0 < FLT_EPSILON ) + rpDW0 = 1.; + rpDWMin = _param2; + rpDWMin = std::max( rpDWMin, 0. ); } - else if( train_method == BACKPROP ) + else if( trainMethod == BACKPROP ) { - bp_dw_scale = _param1; - if( bp_dw_scale <= 0 ) - bp_dw_scale = 0.1; - bp_dw_scale = MAX( bp_dw_scale, 1e-3 ); - bp_dw_scale = MIN( bp_dw_scale, 1 ); - bp_moment_scale = _param2; - if( bp_moment_scale < 0 ) - bp_moment_scale = 0.1; - bp_moment_scale = MIN( bp_moment_scale, 1 ); + bpDWScale = _param1; + if( bpDWScale <= 0 ) + bpDWScale = 0.1; + bpDWScale = std::max( bpDWScale, 1e-3 ); + bpDWScale = std::min( bpDWScale, 1. ); + bpMomentScale = _param2; + if( bpMomentScale < 0 ) + bpMomentScale = 0.1; + bpMomentScale = std::min( bpMomentScale, 1. ); } else - train_method = RPROP; -} - - -CvANN_MLP_TrainParams::~CvANN_MLP_TrainParams() -{ -} - - -CvANN_MLP::CvANN_MLP() -{ - layer_sizes = wbuf = 0; - min_val = max_val = min_val1 = max_val1 = 0.; - weights = 0; - rng = &cv::theRNG(); - default_model_name = "my_nn"; - clear(); -} - - -CvANN_MLP::CvANN_MLP( const CvMat* _layer_sizes, - int _activ_func, - double _f_param1, double _f_param2 ) -{ - layer_sizes = wbuf = 0; - min_val = max_val = min_val1 = max_val1 = 0.; - weights = 0; - rng = &cv::theRNG(); - default_model_name = "my_nn"; - create( _layer_sizes, _activ_func, _f_param1, _f_param2 ); -} - - -CvANN_MLP::~CvANN_MLP() -{ - clear(); -} - - -void CvANN_MLP::clear() -{ - cvReleaseMat( &layer_sizes ); - cvReleaseMat( &wbuf ); - cvFree( &weights ); - activ_func = SIGMOID_SYM; - f_param1 = f_param2 = 1; - max_buf_sz = 1 << 12; + trainMethod = RPROP; } -void CvANN_MLP::set_activ_func( int _activ_func, double _f_param1, double _f_param2 ) +class ANN_MLPImpl : public ANN_MLP { - CV_FUNCNAME( "CvANN_MLP::set_activ_func" ); - - __BEGIN__; - - if( _activ_func < 0 || _activ_func > GAUSSIAN ) - CV_ERROR( CV_StsOutOfRange, "Unknown activation function" ); - - activ_func = _activ_func; - - switch( activ_func ) +public: + ANN_MLPImpl() { - case SIGMOID_SYM: - max_val = 0.95; min_val = -max_val; - max_val1 = 0.98; min_val1 = -max_val1; - if( fabs(_f_param1) < FLT_EPSILON ) - _f_param1 = 2./3; - if( fabs(_f_param2) < FLT_EPSILON ) - _f_param2 = 1.7159; - break; - case GAUSSIAN: - max_val = 1.; min_val = 0.05; - max_val1 = 1.; min_val1 = 0.02; - if( fabs(_f_param1) < FLT_EPSILON ) - _f_param1 = 1.; - if( fabs(_f_param2) < FLT_EPSILON ) - _f_param2 = 1.; - break; - default: - min_val = max_val = min_val1 = max_val1 = 0.; - _f_param1 = 1.; - _f_param2 = 0.; + clear(); } - f_param1 = _f_param1; - f_param2 = _f_param2; - - __END__; -} - - -void CvANN_MLP::init_weights() -{ - int i, j, k; - - for( i = 1; i < layer_sizes->cols; i++ ) + ANN_MLPImpl( const Params& p ) { - int n1 = layer_sizes->data.i[i-1]; - int n2 = layer_sizes->data.i[i]; - double val = 0, G = n2 > 2 ? 0.7*pow((double)n1,1./(n2-1)) : 1.; - double* w = weights[i]; - - // initialize weights using Nguyen-Widrow algorithm - for( j = 0; j < n2; j++ ) - { - double s = 0; - for( k = 0; k <= n1; k++ ) - { - val = rng->uniform(0., 1.)*2-1.; - w[k*n2 + j] = val; - s += fabs(val); - } - - if( i < layer_sizes->cols - 1 ) - { - s = 1./(s - fabs(val)); - for( k = 0; k <= n1; k++ ) - w[k*n2 + j] *= s; - w[n1*n2 + j] *= G*(-1+j*2./n2); - } - } + setParams(p); } -} - -void CvANN_MLP::create( const CvMat* _layer_sizes, int _activ_func, - double _f_param1, double _f_param2 ) -{ - CV_FUNCNAME( "CvANN_MLP::create" ); - - __BEGIN__; - - int i, l_step, l_count, buf_sz = 0; - int *l_src, *l_dst; - - clear(); + virtual ~ANN_MLPImpl() {} - if( !CV_IS_MAT(_layer_sizes) || - (_layer_sizes->cols != 1 && _layer_sizes->rows != 1) || - CV_MAT_TYPE(_layer_sizes->type) != CV_32SC1 ) - CV_ERROR( CV_StsBadArg, - "The array of layer neuron counters must be an integer vector" ); - - CV_CALL( set_activ_func( _activ_func, _f_param1, _f_param2 )); - - l_count = _layer_sizes->rows + _layer_sizes->cols - 1; - l_src = _layer_sizes->data.i; - l_step = CV_IS_MAT_CONT(_layer_sizes->type) ? 1 : - _layer_sizes->step / sizeof(l_src[0]); - CV_CALL( layer_sizes = cvCreateMat( 1, l_count, CV_32SC1 )); - l_dst = layer_sizes->data.i; - - max_count = 0; - for( i = 0; i < l_count; i++ ) + void setParams(const Params& p) { - int n = l_src[i*l_step]; - if( n < 1 + (0 < i && i < l_count-1)) - CV_ERROR( CV_StsOutOfRange, - "there should be at least one input and one output " - "and every hidden layer must have more than 1 neuron" ); - l_dst[i] = n; - max_count = MAX( max_count, n ); - if( i > 0 ) - buf_sz += (l_dst[i-1]+1)*n; + params = p; + create( params.layerSizes ); + set_activ_func( params.activateFunc, params.fparam1, params.fparam2 ); } - buf_sz += (l_dst[0] + l_dst[l_count-1]*2)*2; - - CV_CALL( wbuf = cvCreateMat( 1, buf_sz, CV_64F )); - CV_CALL( weights = (double**)cvAlloc( (l_count+2)*sizeof(weights[0]) )); - - weights[0] = wbuf->data.db; - weights[1] = weights[0] + l_dst[0]*2; - for( i = 1; i < l_count; i++ ) - weights[i+1] = weights[i] + (l_dst[i-1] + 1)*l_dst[i]; - weights[l_count+1] = weights[l_count] + l_dst[l_count-1]*2; - - __END__; -} - + Params getParams() const + { + return params; + } -float CvANN_MLP::predict( const CvMat* _inputs, CvMat* _outputs ) const -{ - int i, j, n, dn = 0, l_count, dn0, buf_sz, min_buf_sz; - - if( !layer_sizes ) - CV_Error( CV_StsError, "The network has not been initialized" ); - - if( !CV_IS_MAT(_inputs) || !CV_IS_MAT(_outputs) || - !CV_ARE_TYPES_EQ(_inputs,_outputs) || - (CV_MAT_TYPE(_inputs->type) != CV_32FC1 && - CV_MAT_TYPE(_inputs->type) != CV_64FC1) || - _inputs->rows != _outputs->rows ) - CV_Error( CV_StsBadArg, "Both input and output must be floating-point matrices " - "of the same type and have the same number of rows" ); - - if( _inputs->cols != layer_sizes->data.i[0] ) - CV_Error( CV_StsBadSize, "input matrix must have the same number of columns as " - "the number of neurons in the input layer" ); - - if( _outputs->cols != layer_sizes->data.i[layer_sizes->cols - 1] ) - CV_Error( CV_StsBadSize, "output matrix must have the same number of columns as " - "the number of neurons in the output layer" ); - n = dn0 = _inputs->rows; - min_buf_sz = 2*max_count; - buf_sz = n*min_buf_sz; - - if( buf_sz > max_buf_sz ) + void clear() { - dn0 = max_buf_sz/min_buf_sz; - dn0 = MAX( dn0, 1 ); - buf_sz = dn0*min_buf_sz; + min_val = max_val = min_val1 = max_val1 = 0.; + rng = RNG((uint64)-1); + weights.clear(); + trained = false; } - cv::AutoBuffer buf(buf_sz); - l_count = layer_sizes->cols; + int layer_count() const { return (int)layer_sizes.size(); } - for( i = 0; i < n; i += dn ) + void set_activ_func( int _activ_func, double _f_param1, double _f_param2 ) { - CvMat hdr[2], _w, *layer_in = &hdr[0], *layer_out = &hdr[1], *temp; - dn = MIN( dn0, n - i ); - - cvGetRows( _inputs, layer_in, i, i + dn ); - cvInitMatHeader( layer_out, dn, layer_in->cols, CV_64F, &buf[0] ); + if( _activ_func < 0 || _activ_func > GAUSSIAN ) + CV_Error( CV_StsOutOfRange, "Unknown activation function" ); - scale_input( layer_in, layer_out ); - CV_SWAP( layer_in, layer_out, temp ); + activ_func = _activ_func; - for( j = 1; j < l_count; j++ ) + switch( activ_func ) { - double* data = buf + (j&1 ? max_count*dn0 : 0); - int cols = layer_sizes->data.i[j]; - - cvInitMatHeader( layer_out, dn, cols, CV_64F, data ); - cvInitMatHeader( &_w, layer_in->cols, layer_out->cols, CV_64F, weights[j] ); - cvGEMM( layer_in, &_w, 1, 0, 0, layer_out ); - calc_activ_func( layer_out, _w.data.db + _w.rows*_w.cols ); - - CV_SWAP( layer_in, layer_out, temp ); + case SIGMOID_SYM: + max_val = 0.95; min_val = -max_val; + max_val1 = 0.98; min_val1 = -max_val1; + if( fabs(_f_param1) < FLT_EPSILON ) + _f_param1 = 2./3; + if( fabs(_f_param2) < FLT_EPSILON ) + _f_param2 = 1.7159; + break; + case GAUSSIAN: + max_val = 1.; min_val = 0.05; + max_val1 = 1.; min_val1 = 0.02; + if( fabs(_f_param1) < FLT_EPSILON ) + _f_param1 = 1.; + if( fabs(_f_param2) < FLT_EPSILON ) + _f_param2 = 1.; + break; + default: + min_val = max_val = min_val1 = max_val1 = 0.; + _f_param1 = 1.; + _f_param2 = 0.; } - cvGetRows( _outputs, layer_out, i, i + dn ); - scale_output( layer_in, layer_out ); + f_param1 = _f_param1; + f_param2 = _f_param2; } - return 0.f; -} - -void CvANN_MLP::scale_input( const CvMat* _src, CvMat* _dst ) const -{ - int i, j, cols = _src->cols; - double* dst = _dst->data.db; - const double* w = weights[0]; - int step = _src->step; - - if( CV_MAT_TYPE( _src->type ) == CV_32F ) - { - const float* src = _src->data.fl; - step /= sizeof(src[0]); - - for( i = 0; i < _src->rows; i++, src += step, dst += cols ) - for( j = 0; j < cols; j++ ) - dst[j] = src[j]*w[j*2] + w[j*2+1]; - } - else + void init_weights() { - const double* src = _src->data.db; - step /= sizeof(src[0]); + int i, j, k, l_count = layer_count(); - for( i = 0; i < _src->rows; i++, src += step, dst += cols ) - for( j = 0; j < cols; j++ ) - dst[j] = src[j]*w[j*2] + w[j*2+1]; - } -} - - -void CvANN_MLP::scale_output( const CvMat* _src, CvMat* _dst ) const -{ - int i, j, cols = _src->cols; - const double* src = _src->data.db; - const double* w = weights[layer_sizes->cols]; - int step = _dst->step; + for( i = 1; i < l_count; i++ ) + { + int n1 = layer_sizes[i-1]; + int n2 = layer_sizes[i]; + double val = 0, G = n2 > 2 ? 0.7*pow((double)n1,1./(n2-1)) : 1.; + double* w = weights[i].ptr(); - if( CV_MAT_TYPE( _dst->type ) == CV_32F ) - { - float* dst = _dst->data.fl; - step /= sizeof(dst[0]); + // initialize weights using Nguyen-Widrow algorithm + for( j = 0; j < n2; j++ ) + { + double s = 0; + for( k = 0; k <= n1; k++ ) + { + val = rng.uniform(0., 1.)*2-1.; + w[k*n2 + j] = val; + s += fabs(val); + } - for( i = 0; i < _src->rows; i++, src += cols, dst += step ) - for( j = 0; j < cols; j++ ) - dst[j] = (float)(src[j]*w[j*2] + w[j*2+1]); + if( i < l_count - 1 ) + { + s = 1./(s - fabs(val)); + for( k = 0; k <= n1; k++ ) + w[k*n2 + j] *= s; + w[n1*n2 + j] *= G*(-1+j*2./n2); + } + } + } } - else + + void create( InputArray _layer_sizes ) { - double* dst = _dst->data.db; - step /= sizeof(dst[0]); + clear(); - for( i = 0; i < _src->rows; i++, src += cols, dst += step ) - for( j = 0; j < cols; j++ ) - dst[j] = src[j]*w[j*2] + w[j*2+1]; - } -} + _layer_sizes.copyTo(layer_sizes); + int l_count = layer_count(); + weights.resize(l_count + 2); + max_lsize = 0; -void CvANN_MLP::calc_activ_func( CvMat* sums, const double* bias ) const -{ - int i, j, n = sums->rows, cols = sums->cols; - double* data = sums->data.db; - double scale = 0, scale2 = f_param2; + if( l_count > 0 ) + { + for( int i = 0; i < l_count; i++ ) + { + int n = layer_sizes[i]; + if( n < 1 + (0 < i && i < l_count-1)) + CV_Error( CV_StsOutOfRange, + "there should be at least one input and one output " + "and every hidden layer must have more than 1 neuron" ); + max_lsize = std::max( max_lsize, n ); + if( i > 0 ) + weights[i].create(layer_sizes[i-1]+1, n, CV_64F); + } - switch( activ_func ) - { - case IDENTITY: - scale = 1.; - break; - case SIGMOID_SYM: - scale = -f_param1; - break; - case GAUSSIAN: - scale = -f_param1*f_param1; - break; - default: - ; + int ninputs = layer_sizes.front(); + int noutputs = layer_sizes.back(); + weights[0].create(1, ninputs*2, CV_64F); + weights[l_count].create(1, noutputs*2, CV_64F); + weights[l_count+1].create(1, noutputs*2, CV_64F); + } } - assert( CV_IS_MAT_CONT(sums->type) ); - - if( activ_func != GAUSSIAN ) + float predict( InputArray _inputs, OutputArray _outputs, int ) const { - for( i = 0; i < n; i++, data += cols ) - for( j = 0; j < cols; j++ ) - data[j] = (data[j] + bias[j])*scale; + if( !trained ) + CV_Error( CV_StsError, "The network has not been trained or loaded" ); - if( activ_func == IDENTITY ) - return; - } - else - { - for( i = 0; i < n; i++, data += cols ) - for( j = 0; j < cols; j++ ) - { - double t = data[j] + bias[j]; - data[j] = t*t*scale; - } - } + Mat inputs = _inputs.getMat(); + int type = inputs.type(), l_count = layer_count(); + int n = inputs.rows, dn0 = n; - cvExp( sums, sums ); + CV_Assert( (type == CV_32F || type == CV_64F) && inputs.cols == layer_sizes[0] ); + int noutputs = layer_sizes[l_count-1]; + Mat outputs; - n *= cols; - data -= n; + int min_buf_sz = 2*max_lsize; + int buf_sz = n*min_buf_sz; - switch( activ_func ) - { - case SIGMOID_SYM: - for( i = 0; i <= n - 4; i += 4 ) + if( buf_sz > max_buf_sz ) { - double x0 = 1.+data[i], x1 = 1.+data[i+1], x2 = 1.+data[i+2], x3 = 1.+data[i+3]; - double a = x0*x1, b = x2*x3, d = scale2/(a*b), t0, t1; - a *= d; b *= d; - t0 = (2 - x0)*b*x1; t1 = (2 - x1)*b*x0; - data[i] = t0; data[i+1] = t1; - t0 = (2 - x2)*a*x3; t1 = (2 - x3)*a*x2; - data[i+2] = t0; data[i+3] = t1; + dn0 = max_buf_sz/min_buf_sz; + dn0 = std::max( dn0, 1 ); + buf_sz = dn0*min_buf_sz; } - for( ; i < n; i++ ) + cv::AutoBuffer _buf(buf_sz+noutputs); + double* buf = _buf; + + if( !_outputs.needed() ) { - double t = scale2*(1. - data[i])/(1. + data[i]); - data[i] = t; + CV_Assert( n == 1 ); + outputs = Mat(n, noutputs, type, buf + buf_sz); + } + else + { + _outputs.create(n, noutputs, type); + outputs = _outputs.getMat(); } - break; - - case GAUSSIAN: - for( i = 0; i < n; i++ ) - data[i] = scale2*data[i]; - break; - default: - ; - } -} + int dn = 0; + for( int i = 0; i < n; i += dn ) + { + dn = std::min( dn0, n - i ); + Mat layer_in = inputs.rowRange(i, i + dn); + Mat layer_out( dn, layer_in.cols, CV_64F, buf); -void CvANN_MLP::calc_activ_func_deriv( CvMat* _xf, CvMat* _df, - const double* bias ) const -{ - int i, j, n = _xf->rows, cols = _xf->cols; - double* xf = _xf->data.db; - double* df = _df->data.db; - double scale, scale2 = f_param2; - assert( CV_IS_MAT_CONT( _xf->type & _df->type ) ); + scale_input( layer_in, layer_out ); + layer_in = layer_out; - if( activ_func == IDENTITY ) - { - for( i = 0; i < n; i++, xf += cols, df += cols ) - for( j = 0; j < cols; j++ ) - { - xf[j] += bias[j]; - df[j] = 1; - } - return; - } - else if( activ_func == GAUSSIAN ) - { - scale = -f_param1*f_param1; - scale2 *= scale; - for( i = 0; i < n; i++, xf += cols, df += cols ) - for( j = 0; j < cols; j++ ) + for( int j = 1; j < l_count; j++ ) { - double t = xf[j] + bias[j]; - df[j] = t*2*scale2; - xf[j] = t*t*scale; - } - cvExp( _xf, _xf ); + double* data = buf + ((j&1) ? max_lsize*dn0 : 0); + int cols = layer_sizes[j]; - n *= cols; - xf -= n; df -= n; + layer_out = Mat(dn, cols, CV_64F, data); + Mat w = weights[j].rowRange(0, layer_in.cols); + gemm(layer_in, w, 1, noArray(), 0, layer_out); + calc_activ_func( layer_out, weights[j] ); - for( i = 0; i < n; i++ ) - df[i] *= xf[i]; - } - else - { - scale = f_param1; - for( i = 0; i < n; i++, xf += cols, df += cols ) - for( j = 0; j < cols; j++ ) - { - xf[j] = (xf[j] + bias[j])*scale; - df[j] = -fabs(xf[j]); + layer_in = layer_out; } - cvExp( _df, _df ); - - n *= cols; - xf -= n; df -= n; + layer_out = outputs.rowRange(i, i + dn); + scale_output( layer_in, layer_out ); + } - // ((1+exp(-ax))^-1)'=a*((1+exp(-ax))^-2)*exp(-ax); - // ((1-exp(-ax))/(1+exp(-ax)))'=(a*exp(-ax)*(1+exp(-ax)) + a*exp(-ax)*(1-exp(-ax)))/(1+exp(-ax))^2= - // 2*a*exp(-ax)/(1+exp(-ax))^2 - scale *= 2*f_param2; - for( i = 0; i < n; i++ ) + if( n == 1 ) { - int s0 = xf[i] > 0 ? 1 : -1; - double t0 = 1./(1. + df[i]); - double t1 = scale*df[i]*t0*t0; - t0 *= scale2*(1. - df[i])*s0; - df[i] = t1; - xf[i] = t0; + int maxIdx[] = {0, 0}; + minMaxIdx(outputs, 0, 0, 0, maxIdx); + return (float)(maxIdx[0] + maxIdx[1]); } - } -} + return 0.f; + } -void CvANN_MLP::calc_input_scale( const CvVectors* vecs, int flags ) -{ - bool reset_weights = (flags & UPDATE_WEIGHTS) == 0; - bool no_scale = (flags & NO_INPUT_SCALE) != 0; - double* scale = weights[0]; - int count = vecs->count; - - if( reset_weights ) + void scale_input( const Mat& _src, Mat& _dst ) const { - int i, j, vcount = layer_sizes->data.i[0]; - int type = vecs->type; - double a = no_scale ? 1. : 0.; - - for( j = 0; j < vcount; j++ ) - scale[2*j] = a, scale[j*2+1] = 0.; + int cols = _src.cols; + const double* w = weights[0].ptr(); - if( no_scale ) - return; - - for( i = 0; i < count; i++ ) + if( _src.type() == CV_32F ) { - const float* f = vecs->data.fl[i]; - const double* d = vecs->data.db[i]; - for( j = 0; j < vcount; j++ ) + for( int i = 0; i < _src.rows; i++ ) { - double t = type == CV_32F ? (double)f[j] : d[j]; - scale[j*2] += t; - scale[j*2+1] += t*t; + const float* src = _src.ptr(i); + double* dst = _dst.ptr(i); + for( int j = 0; j < cols; j++ ) + dst[j] = src[j]*w[j*2] + w[j*2+1]; } } - - for( j = 0; j < vcount; j++ ) + else { - double s = scale[j*2], s2 = scale[j*2+1]; - double m = s/count, sigma2 = s2/count - m*m; - scale[j*2] = sigma2 < DBL_EPSILON ? 1 : 1./sqrt(sigma2); - scale[j*2+1] = -m*scale[j*2]; + for( int i = 0; i < _src.rows; i++ ) + { + const float* src = _src.ptr(i); + double* dst = _dst.ptr(i); + for( int j = 0; j < cols; j++ ) + dst[j] = src[j]*w[j*2] + w[j*2+1]; + } } } -} - -void CvANN_MLP::calc_output_scale( const CvVectors* vecs, int flags ) -{ - int i, j, vcount = layer_sizes->data.i[layer_sizes->cols-1]; - int type = vecs->type; - double m = min_val, M = max_val, m1 = min_val1, M1 = max_val1; - bool reset_weights = (flags & UPDATE_WEIGHTS) == 0; - bool no_scale = (flags & NO_OUTPUT_SCALE) != 0; - int l_count = layer_sizes->cols; - double* scale = weights[l_count]; - double* inv_scale = weights[l_count+1]; - int count = vecs->count; - - CV_FUNCNAME( "CvANN_MLP::calc_output_scale" ); - - __BEGIN__; - - if( reset_weights ) + void scale_output( const Mat& _src, Mat& _dst ) const { - double a0 = no_scale ? 1 : DBL_MAX, b0 = no_scale ? 0 : -DBL_MAX; + int cols = _src.cols; + const double* w = weights[layer_count()].ptr(); - for( j = 0; j < vcount; j++ ) + if( _dst.type() == CV_32F ) { - scale[2*j] = inv_scale[2*j] = a0; - scale[j*2+1] = inv_scale[2*j+1] = b0; + for( int i = 0; i < _src.rows; i++ ) + { + const double* src = _src.ptr(i); + float* dst = _dst.ptr(i); + for( int j = 0; j < cols; j++ ) + dst[j] = (float)(src[j]*w[j*2] + w[j*2+1]); + } + } + else + { + for( int i = 0; i < _src.rows; i++ ) + { + const double* src = _src.ptr(i); + double* dst = _dst.ptr(i); + for( int j = 0; j < cols; j++ ) + dst[j] = src[j]*w[j*2] + w[j*2+1]; + } } - - if( no_scale ) - EXIT; } - for( i = 0; i < count; i++ ) + void calc_activ_func( Mat& sums, const Mat& w ) const { - const float* f = vecs->data.fl[i]; - const double* d = vecs->data.db[i]; + const double* bias = w.ptr(w.rows-1); + int i, j, n = sums.rows, cols = sums.cols; + double scale = 0, scale2 = f_param2; - for( j = 0; j < vcount; j++ ) + switch( activ_func ) { - double t = type == CV_32F ? (double)f[j] : d[j]; + case IDENTITY: + scale = 1.; + break; + case SIGMOID_SYM: + scale = -f_param1; + break; + case GAUSSIAN: + scale = -f_param1*f_param1; + break; + default: + ; + } - if( reset_weights ) - { - double mj = scale[j*2], Mj = scale[j*2+1]; - if( mj > t ) mj = t; - if( Mj < t ) Mj = t; + CV_Assert( sums.isContinuous() ); - scale[j*2] = mj; - scale[j*2+1] = Mj; + if( activ_func != GAUSSIAN ) + { + for( i = 0; i < n; i++ ) + { + double* data = sums.ptr(i); + for( j = 0; j < cols; j++ ) + data[j] = (data[j] + bias[j])*scale; } - else + + if( activ_func == IDENTITY ) + return; + } + else + { + for( i = 0; i < n; i++ ) { - t = t*inv_scale[j*2] + inv_scale[2*j+1]; - if( t < m1 || t > M1 ) - CV_ERROR( CV_StsOutOfRange, - "Some of new output training vector components run exceed the original range too much" ); + double* data = sums.ptr(i); + for( j = 0; j < cols; j++ ) + { + double t = data[j] + bias[j]; + data[j] = t*t*scale; + } } } - } - if( reset_weights ) - for( j = 0; j < vcount; j++ ) + exp( sums, sums ); + + if( sums.isContinuous() ) { - // map mj..Mj to m..M - double mj = scale[j*2], Mj = scale[j*2+1]; - double a, b; - double delta = Mj - mj; - if( delta < DBL_EPSILON ) - a = 1, b = (M + m - Mj - mj)*0.5; - else - a = (M - m)/delta, b = m - mj*a; - inv_scale[j*2] = a; inv_scale[j*2+1] = b; - a = 1./a; b = -b*a; - scale[j*2] = a; scale[j*2+1] = b; + cols *= n; + n = 1; } - __END__; -} + switch( activ_func ) + { + case SIGMOID_SYM: + for( i = 0; i < n; i++ ) + { + double* data = sums.ptr(i); + for( j = 0; j < cols; j++ ) + { + double t = scale2*(1. - data[j])/(1. + data[j]); + data[j] = t; + } + } + break; + case GAUSSIAN: + for( i = 0; i < n; j++ ) + { + double* data = sums.ptr(i); + for( j = 0; j < cols; j++ ) + data[j] = scale2*data[j]; + } + break; -bool CvANN_MLP::prepare_to_train( const CvMat* _inputs, const CvMat* _outputs, - const CvMat* _sample_weights, const CvMat* _sample_idx, - CvVectors* _ivecs, CvVectors* _ovecs, double** _sw, int _flags ) -{ - bool ok = false; - CvMat* sample_idx = 0; - CvVectors ivecs, ovecs; - double* sw = 0; - int count = 0; - - CV_FUNCNAME( "CvANN_MLP::prepare_to_train" ); - - ivecs.data.ptr = ovecs.data.ptr = 0; - assert( _ivecs && _ovecs ); - - __BEGIN__; - - const int* sidx = 0; - int i, sw_type = 0, sw_count = 0; - int sw_step = 0; - double sw_sum = 0; - - if( !layer_sizes ) - CV_ERROR( CV_StsError, - "The network has not been created. Use method create or the appropriate constructor" ); - - if( !CV_IS_MAT(_inputs) || (CV_MAT_TYPE(_inputs->type) != CV_32FC1 && - CV_MAT_TYPE(_inputs->type) != CV_64FC1) || _inputs->cols != layer_sizes->data.i[0] ) - CV_ERROR( CV_StsBadArg, - "input training data should be a floating-point matrix with" - "the number of rows equal to the number of training samples and " - "the number of columns equal to the size of 0-th (input) layer" ); - - if( !CV_IS_MAT(_outputs) || (CV_MAT_TYPE(_outputs->type) != CV_32FC1 && - CV_MAT_TYPE(_outputs->type) != CV_64FC1) || - _outputs->cols != layer_sizes->data.i[layer_sizes->cols - 1] ) - CV_ERROR( CV_StsBadArg, - "output training data should be a floating-point matrix with" - "the number of rows equal to the number of training samples and " - "the number of columns equal to the size of last (output) layer" ); - - if( _inputs->rows != _outputs->rows ) - CV_ERROR( CV_StsUnmatchedSizes, "The numbers of input and output samples do not match" ); - - if( _sample_idx ) - { - CV_CALL( sample_idx = cvPreprocessIndexArray( _sample_idx, _inputs->rows )); - sidx = sample_idx->data.i; - count = sample_idx->cols + sample_idx->rows - 1; + default: + ; + } } - else - count = _inputs->rows; - if( _sample_weights ) + void calc_activ_func_deriv( Mat& _xf, Mat& _df, const Mat& w ) const { - if( !CV_IS_MAT(_sample_weights) ) - CV_ERROR( CV_StsBadArg, "sample_weights (if passed) must be a valid matrix" ); - - sw_type = CV_MAT_TYPE(_sample_weights->type); - sw_count = _sample_weights->cols + _sample_weights->rows - 1; + const double* bias = w.ptr(w.rows-1); + int i, j, n = _xf.rows, cols = _xf.cols; - if( (sw_type != CV_32FC1 && sw_type != CV_64FC1) || - (_sample_weights->cols != 1 && _sample_weights->rows != 1) || - (sw_count != count && sw_count != _inputs->rows) ) - CV_ERROR( CV_StsBadArg, - "sample_weights must be 1d floating-point vector containing weights " - "of all or selected training samples" ); - - sw_step = CV_IS_MAT_CONT(_sample_weights->type) ? 1 : - _sample_weights->step/CV_ELEM_SIZE(sw_type); + if( activ_func == IDENTITY ) + { + for( i = 0; i < n; i++ ) + { + double* xf = _xf.ptr(i); + double* df = _df.ptr(i); - CV_CALL( sw = (double*)cvAlloc( count*sizeof(sw[0]) )); - } + for( j = 0; j < cols; j++ ) + { + xf[j] += bias[j]; + df[j] = 1; + } + } + } + else if( activ_func == GAUSSIAN ) + { + double scale = -f_param1*f_param1; + double scale2 = scale*f_param2; + for( i = 0; i < n; i++ ) + { + double* xf = _xf.ptr(i); + double* df = _df.ptr(i); - CV_CALL( ivecs.data.ptr = (uchar**)cvAlloc( count*sizeof(ivecs.data.ptr[0]) )); - CV_CALL( ovecs.data.ptr = (uchar**)cvAlloc( count*sizeof(ovecs.data.ptr[0]) )); + for( j = 0; j < cols; j++ ) + { + double t = xf[j] + bias[j]; + df[j] = t*2*scale2; + xf[j] = t*t*scale; + } + } + exp( _xf, _xf ); - ivecs.type = CV_MAT_TYPE(_inputs->type); - ovecs.type = CV_MAT_TYPE(_outputs->type); - ivecs.count = ovecs.count = count; + for( i = 0; i < n; i++ ) + { + double* xf = _xf.ptr(i); + double* df = _df.ptr(i); - for( i = 0; i < count; i++ ) - { - int idx = sidx ? sidx[i] : i; - ivecs.data.ptr[i] = _inputs->data.ptr + idx*_inputs->step; - ovecs.data.ptr[i] = _outputs->data.ptr + idx*_outputs->step; - if( sw ) - { - int si = sw_count == count ? i : idx; - double w = sw_type == CV_32FC1 ? - (double)_sample_weights->data.fl[si*sw_step] : - _sample_weights->data.db[si*sw_step]; - sw[i] = w; - if( w < 0 ) - CV_ERROR( CV_StsOutOfRange, "some of sample weights are negative" ); - sw_sum += w; + for( j = 0; j < cols; j++ ) + df[j] *= xf[j]; + } } - } + else + { + double scale = f_param1; + double scale2 = f_param2; - // normalize weights - if( sw ) - { - sw_sum = sw_sum > DBL_EPSILON ? 1./sw_sum : 0; - for( i = 0; i < count; i++ ) - sw[i] *= sw_sum; - } + for( i = 0; i < n; i++ ) + { + double* xf = _xf.ptr(i); + double* df = _df.ptr(i); - calc_input_scale( &ivecs, _flags ); - CV_CALL( calc_output_scale( &ovecs, _flags )); + for( j = 0; j < cols; j++ ) + { + xf[j] = (xf[j] + bias[j])*scale; + df[j] = -fabs(xf[j]); + } + } - ok = true; + exp( _df, _df ); - __END__; + // ((1+exp(-ax))^-1)'=a*((1+exp(-ax))^-2)*exp(-ax); + // ((1-exp(-ax))/(1+exp(-ax)))'=(a*exp(-ax)*(1+exp(-ax)) + a*exp(-ax)*(1-exp(-ax)))/(1+exp(-ax))^2= + // 2*a*exp(-ax)/(1+exp(-ax))^2 + scale *= 2*f_param2; + for( i = 0; i < n; i++ ) + { + double* xf = _xf.ptr(i); + double* df = _df.ptr(i); - if( !ok ) - { - cvFree( &ivecs.data.ptr ); - cvFree( &ovecs.data.ptr ); - cvFree( &sw ); + for( j = 0; j < cols; j++ ) + { + int s0 = xf[j] > 0 ? 1 : -1; + double t0 = 1./(1. + df[j]); + double t1 = scale*df[j]*t0*t0; + t0 *= scale2*(1. - df[j])*s0; + df[j] = t1; + xf[j] = t0; + } + } + } } - cvReleaseMat( &sample_idx ); - *_ivecs = ivecs; - *_ovecs = ovecs; - *_sw = sw; - - return ok; -} + void calc_input_scale( const Mat& inputs, int flags ) + { + bool reset_weights = (flags & UPDATE_WEIGHTS) == 0; + bool no_scale = (flags & NO_INPUT_SCALE) != 0; + double* scale = weights[0].ptr(); + int count = inputs.rows; + if( reset_weights ) + { + int i, j, vcount = layer_sizes[0]; + int type = inputs.type(); + double a = no_scale ? 1. : 0.; -int CvANN_MLP::train( const CvMat* _inputs, const CvMat* _outputs, - const CvMat* _sample_weights, const CvMat* _sample_idx, - CvANN_MLP_TrainParams _params, int flags ) -{ - const int MAX_ITER = 1000; - const double DEFAULT_EPSILON = FLT_EPSILON; + for( j = 0; j < vcount; j++ ) + scale[2*j] = a, scale[j*2+1] = 0.; - double* sw = 0; - CvVectors x0, u; - int iter = -1; + if( no_scale ) + return; - x0.data.ptr = u.data.ptr = 0; + for( i = 0; i < count; i++ ) + { + const uchar* p = inputs.ptr(i); + const float* f = (const float*)p; + const double* d = (const double*)p; + for( j = 0; j < vcount; j++ ) + { + double t = type == CV_32F ? (double)f[j] : d[j]; + scale[j*2] += t; + scale[j*2+1] += t*t; + } + } - CV_FUNCNAME( "CvANN_MLP::train" ); + for( j = 0; j < vcount; j++ ) + { + double s = scale[j*2], s2 = scale[j*2+1]; + double m = s/count, sigma2 = s2/count - m*m; + scale[j*2] = sigma2 < DBL_EPSILON ? 1 : 1./sqrt(sigma2); + scale[j*2+1] = -m*scale[j*2]; + } + } + } - __BEGIN__; + void calc_output_scale( const Mat& outputs, int flags ) + { + int i, j, vcount = layer_sizes.back(); + int type = outputs.type(); + double m = min_val, M = max_val, m1 = min_val1, M1 = max_val1; + bool reset_weights = (flags & UPDATE_WEIGHTS) == 0; + bool no_scale = (flags & NO_OUTPUT_SCALE) != 0; + int l_count = layer_count(); + double* scale = weights[l_count].ptr(); + double* inv_scale = weights[l_count+1].ptr(); + int count = outputs.rows; + + if( reset_weights ) + { + double a0 = no_scale ? 1 : DBL_MAX, b0 = no_scale ? 0 : -DBL_MAX; - int max_iter; - double epsilon; + for( j = 0; j < vcount; j++ ) + { + scale[2*j] = inv_scale[2*j] = a0; + scale[j*2+1] = inv_scale[2*j+1] = b0; + } - params = _params; + if( no_scale ) + return; + } - // initialize training data - CV_CALL( prepare_to_train( _inputs, _outputs, _sample_weights, - _sample_idx, &x0, &u, &sw, flags )); + for( i = 0; i < count; i++ ) + { + const uchar* p = outputs.ptr(i); + const float* f = (const float*)p; + const double* d = (const double*)p; - // ... and link weights - if( !(flags & UPDATE_WEIGHTS) ) - init_weights(); + for( j = 0; j < vcount; j++ ) + { + double t = type == CV_32F ? (double)f[j] : d[j]; - max_iter = params.term_crit.type & CV_TERMCRIT_ITER ? params.term_crit.max_iter : MAX_ITER; - max_iter = MAX( max_iter, 1 ); + if( reset_weights ) + { + double mj = scale[j*2], Mj = scale[j*2+1]; + if( mj > t ) mj = t; + if( Mj < t ) Mj = t; - epsilon = params.term_crit.type & CV_TERMCRIT_EPS ? params.term_crit.epsilon : DEFAULT_EPSILON; - epsilon = MAX(epsilon, DBL_EPSILON); + scale[j*2] = mj; + scale[j*2+1] = Mj; + } + else + { + t = t*inv_scale[j*2] + inv_scale[2*j+1]; + if( t < m1 || t > M1 ) + CV_Error( CV_StsOutOfRange, + "Some of new output training vector components run exceed the original range too much" ); + } + } + } - params.term_crit.type = CV_TERMCRIT_ITER + CV_TERMCRIT_EPS; - params.term_crit.max_iter = max_iter; - params.term_crit.epsilon = epsilon; + if( reset_weights ) + for( j = 0; j < vcount; j++ ) + { + // map mj..Mj to m..M + double mj = scale[j*2], Mj = scale[j*2+1]; + double a, b; + double delta = Mj - mj; + if( delta < DBL_EPSILON ) + a = 1, b = (M + m - Mj - mj)*0.5; + else + a = (M - m)/delta, b = m - mj*a; + inv_scale[j*2] = a; inv_scale[j*2+1] = b; + a = 1./a; b = -b*a; + scale[j*2] = a; scale[j*2+1] = b; + } + } - if( params.train_method == CvANN_MLP_TrainParams::BACKPROP ) + void prepare_to_train( const Mat& inputs, const Mat& outputs, + Mat& sample_weights, int flags ) { - CV_CALL( iter = train_backprop( x0, u, sw )); + if( layer_sizes.empty() ) + CV_Error( CV_StsError, + "The network has not been created. Use method create or the appropriate constructor" ); + + if( (inputs.type() != CV_32F && inputs.type() != CV_64F) || + inputs.cols != layer_sizes[0] ) + CV_Error( CV_StsBadArg, + "input training data should be a floating-point matrix with " + "the number of rows equal to the number of training samples and " + "the number of columns equal to the size of 0-th (input) layer" ); + + if( (outputs.type() != CV_32F && outputs.type() != CV_64F) || + outputs.cols != layer_sizes.back() ) + CV_Error( CV_StsBadArg, + "output training data should be a floating-point matrix with " + "the number of rows equal to the number of training samples and " + "the number of columns equal to the size of last (output) layer" ); + + if( inputs.rows != outputs.rows ) + CV_Error( CV_StsUnmatchedSizes, "The numbers of input and output samples do not match" ); + + Mat temp; + double s = sum(sample_weights)[0]; + sample_weights.convertTo(temp, CV_64F, 1./s); + sample_weights = temp; + + calc_input_scale( inputs, flags ); + calc_output_scale( outputs, flags ); } - else + + bool train( const Ptr& trainData, int flags ) { - CV_CALL( iter = train_rprop( x0, u, sw )); + const int MAX_ITER = 1000; + const double DEFAULT_EPSILON = FLT_EPSILON; + + // initialize training data + Mat inputs = trainData->getTrainSamples(); + Mat outputs = trainData->getTrainResponses(); + Mat sw = trainData->getTrainSampleWeights(); + prepare_to_train( inputs, outputs, sw, flags ); + + // ... and link weights + if( !(flags & UPDATE_WEIGHTS) ) + init_weights(); + + TermCriteria termcrit; + termcrit.type = TermCriteria::COUNT + TermCriteria::EPS; + termcrit.maxCount = std::max((params.termCrit.type & CV_TERMCRIT_ITER ? params.termCrit.maxCount : MAX_ITER), 1); + termcrit.epsilon = std::max((params.termCrit.type & CV_TERMCRIT_EPS ? params.termCrit.epsilon : DEFAULT_EPSILON), DBL_EPSILON); + + int iter = params.trainMethod == Params::BACKPROP ? + train_backprop( inputs, outputs, sw, termcrit ) : + train_rprop( inputs, outputs, sw, termcrit ); + + trained = iter > 0; + return trained; } - __END__; - - cvFree( &x0.data.ptr ); - cvFree( &u.data.ptr ); - cvFree( &sw ); - - return iter; -} - - -int CvANN_MLP::train_backprop( CvVectors x0, CvVectors u, const double* sw ) -{ - CvMat* dw = 0; - CvMat* buf = 0; - double **x = 0, **df = 0; - CvMat* _idx = 0; - int iter = -1, count = x0.count; - - CV_FUNCNAME( "CvANN_MLP::train_backprop" ); - - __BEGIN__; - - int i, j, k, ivcount, ovcount, l_count, total = 0, max_iter; - double *buf_ptr; - double prev_E = DBL_MAX*0.5, E = 0, epsilon; - - max_iter = params.term_crit.max_iter*count; - epsilon = params.term_crit.epsilon*count; + int train_backprop( const Mat& inputs, const Mat& outputs, const Mat& _sw, TermCriteria termCrit ) + { + int i, j, k; + double prev_E = DBL_MAX*0.5, E = 0; + int itype = inputs.type(), otype = outputs.type(); - l_count = layer_sizes->cols; - ivcount = layer_sizes->data.i[0]; - ovcount = layer_sizes->data.i[l_count-1]; + int count = inputs.rows; - // allocate buffers - for( i = 0; i < l_count; i++ ) - total += layer_sizes->data.i[i] + 1; + int iter = -1, max_iter = termCrit.maxCount*count; + double epsilon = termCrit.epsilon*count; - CV_CALL( dw = cvCreateMat( wbuf->rows, wbuf->cols, wbuf->type )); - cvZero( dw ); - CV_CALL( buf = cvCreateMat( 1, (total + max_count)*2, CV_64F )); - CV_CALL( _idx = cvCreateMat( 1, count, CV_32SC1 )); - for( i = 0; i < count; i++ ) - _idx->data.i[i] = i; + int l_count = layer_count(); + int ivcount = layer_sizes[0]; + int ovcount = layer_sizes.back(); - CV_CALL( x = (double**)cvAlloc( total*2*sizeof(x[0]) )); - df = x + total; - buf_ptr = buf->data.db; + // allocate buffers + vector > x(l_count); + vector > df(l_count); + vector dw(l_count); - for( j = 0; j < l_count; j++ ) - { - x[j] = buf_ptr; - df[j] = x[j] + layer_sizes->data.i[j]; - buf_ptr += (df[j] - x[j])*2; - } - - // run back-propagation loop - /* - y_i = w_i*x_{i-1} - x_i = f(y_i) - E = 1/2*||u - x_N||^2 - grad_N = (x_N - u)*f'(y_i) - dw_i(t) = momentum*dw_i(t-1) + dw_scale*x_{i-1}*grad_i - w_i(t+1) = w_i(t) + dw_i(t) - grad_{i-1} = w_i^t*grad_i - */ - for( iter = 0; iter < max_iter; iter++ ) - { - int idx = iter % count; - double* w = weights[0]; - double sweight = sw ? count*sw[idx] : 1.; - CvMat _w, _dw, hdr1, hdr2, ghdr1, ghdr2, _df; - CvMat *x1 = &hdr1, *x2 = &hdr2, *grad1 = &ghdr1, *grad2 = &ghdr2, *temp; + for( i = 0; i < l_count; i++ ) + { + int n = layer_sizes[i]; + x[i].resize(n); + df[i].resize(n); + dw[i].create(weights[i].size(), CV_64F); + } - if( idx == 0 ) + Mat _idx_m(1, count, CV_32S); + int* _idx = _idx_m.ptr(); + for( i = 0; i < count; i++ ) + _idx[i] = i; + + AutoBuffer _buf(max_lsize*2); + double* buf[] = { _buf, (double*)_buf + max_lsize }; + + const double* sw = _sw.empty() ? 0 : _sw.ptr(); + + // run back-propagation loop + /* + y_i = w_i*x_{i-1} + x_i = f(y_i) + E = 1/2*||u - x_N||^2 + grad_N = (x_N - u)*f'(y_i) + dw_i(t) = momentum*dw_i(t-1) + dw_scale*x_{i-1}*grad_i + w_i(t+1) = w_i(t) + dw_i(t) + grad_{i-1} = w_i^t*grad_i + */ + for( iter = 0; iter < max_iter; iter++ ) { - //printf("%d. E = %g\n", iter/count, E); - if( fabs(prev_E - E) < epsilon ) - break; - prev_E = E; - E = 0; + int idx = iter % count; + double sweight = sw ? count*sw[idx] : 1.; - // shuffle indices - for( i = 0; i < count; i++ ) + if( idx == 0 ) { - int tt; - j = (*rng)(count); - k = (*rng)(count); - CV_SWAP( _idx->data.i[j], _idx->data.i[k], tt ); + //printf("%d. E = %g\n", iter/count, E); + if( fabs(prev_E - E) < epsilon ) + break; + prev_E = E; + E = 0; + + // shuffle indices + for( i = 0; i < count; i++ ) + { + j = rng.uniform(0, count); + k = rng.uniform(0, count); + std::swap(_idx[j], _idx[k]); + } } - } - idx = _idx->data.i[idx]; + idx = _idx[idx]; - if( x0.type == CV_32F ) - { - const float* x0data = x0.data.fl[idx]; - for( j = 0; j < ivcount; j++ ) - x[0][j] = x0data[j]*w[j*2] + w[j*2 + 1]; - } - else - { - const double* x0data = x0.data.db[idx]; + const uchar* x0data_p = inputs.ptr(idx); + const float* x0data_f = (const float*)x0data_p; + const double* x0data_d = (const double*)x0data_p; + + double* w = weights[0].ptr(); for( j = 0; j < ivcount; j++ ) - x[0][j] = x0data[j]*w[j*2] + w[j*2 + 1]; - } + x[0][j] = (itype == CV_32F ? (double)x0data_f[j] : x0data_d[j])*w[j*2] + w[j*2 + 1]; - cvInitMatHeader( x1, 1, ivcount, CV_64F, x[0] ); + Mat x1( 1, ivcount, CV_64F, &x[0][0] ); - // forward pass, compute y[i]=w*x[i-1], x[i]=f(y[i]), df[i]=f'(y[i]) - for( i = 1; i < l_count; i++ ) - { - cvInitMatHeader( x2, 1, layer_sizes->data.i[i], CV_64F, x[i] ); - cvInitMatHeader( &_w, x1->cols, x2->cols, CV_64F, weights[i] ); - cvGEMM( x1, &_w, 1, 0, 0, x2 ); - _df = *x2; - _df.data.db = df[i]; - calc_activ_func_deriv( x2, &_df, _w.data.db + _w.rows*_w.cols ); - CV_SWAP( x1, x2, temp ); - } + // forward pass, compute y[i]=w*x[i-1], x[i]=f(y[i]), df[i]=f'(y[i]) + for( i = 1; i < l_count; i++ ) + { + int n = layer_sizes[i]; + Mat x2(1, n, CV_64F, &x[i][0] ); + Mat _w = weights[i].rowRange(0, x1.cols); + gemm(x1, _w, 1, noArray(), 0, x2); + Mat _df(1, n, CV_64F, &df[i][0] ); + calc_activ_func_deriv( x2, _df, weights[i] ); + x1 = x2; + } - cvInitMatHeader( grad1, 1, ovcount, CV_64F, buf_ptr ); - *grad2 = *grad1; - grad2->data.db = buf_ptr + max_count; + Mat grad1( 1, ovcount, CV_64F, buf[l_count&1] ); + w = weights[l_count+1].ptr(); - w = weights[l_count+1]; + // calculate error + const uchar* udata_p = outputs.ptr(idx); + const float* udata_f = (const float*)udata_p; + const double* udata_d = (const double*)udata_p; - // calculate error - if( u.type == CV_32F ) - { - const float* udata = u.data.fl[idx]; - for( k = 0; k < ovcount; k++ ) - { - double t = udata[k]*w[k*2] + w[k*2+1] - x[l_count-1][k]; - grad1->data.db[k] = t*sweight; - E += t*t; - } - } - else - { - const double* udata = u.data.db[idx]; + double* gdata = grad1.ptr(); for( k = 0; k < ovcount; k++ ) { - double t = udata[k]*w[k*2] + w[k*2+1] - x[l_count-1][k]; - grad1->data.db[k] = t*sweight; + double t = (otype == CV_32F ? (double)udata_f[k] : udata_d[k])*w[k*2] + w[k*2+1] - x[l_count-1][k]; + gdata[k] = t*sweight; E += t*t; } - } - E *= sweight; + E *= sweight; - // backward pass, update weights - for( i = l_count-1; i > 0; i-- ) - { - int n1 = layer_sizes->data.i[i-1], n2 = layer_sizes->data.i[i]; - cvInitMatHeader( &_df, 1, n2, CV_64F, df[i] ); - cvMul( grad1, &_df, grad1 ); - cvInitMatHeader( &_w, n1+1, n2, CV_64F, weights[i] ); - cvInitMatHeader( &_dw, n1+1, n2, CV_64F, dw->data.db + (weights[i] - weights[0]) ); - cvInitMatHeader( x1, n1+1, 1, CV_64F, x[i-1] ); - x[i-1][n1] = 1.; - cvGEMM( x1, grad1, params.bp_dw_scale, &_dw, params.bp_moment_scale, &_dw ); - cvAdd( &_w, &_dw, &_w ); - if( i > 1 ) + // backward pass, update weights + for( i = l_count-1; i > 0; i-- ) { - grad2->cols = n1; - _w.rows = n1; - cvGEMM( grad1, &_w, 1, 0, 0, grad2, CV_GEMM_B_T ); + int n1 = layer_sizes[i-1], n2 = layer_sizes[i]; + Mat _df(1, n2, CV_64F, &df[i][0]); + multiply( grad1, _df, grad1 ); + Mat _x(n1+1, 1, CV_64F, &x[i-1][0]); + x[i-1][n1] = 1.; + gemm( _x, grad1, params.bpDWScale, dw[i], params.bpMomentScale, dw[i] ); + add( weights[i], dw[i], weights[i] ); + if( i > 1 ) + { + Mat grad2(1, n1, CV_64F, buf[i&1]); + Mat _w = weights[i].rowRange(0, n1); + gemm( grad1, _w, 1, noArray(), 0, grad2, GEMM_2_T ); + grad1 = grad2; + } } - CV_SWAP( grad1, grad2, temp ); } - } - - iter /= count; - - __END__; - - cvReleaseMat( &dw ); - cvReleaseMat( &buf ); - cvReleaseMat( &_idx ); - cvFree( &x ); - return iter; -} - -struct rprop_loop : cv::ParallelLoopBody { - rprop_loop(const CvANN_MLP* _point, double**& _weights, int& _count, int& _ivcount, CvVectors* _x0, - int& _l_count, CvMat*& _layer_sizes, int& _ovcount, int& _max_count, - CvVectors* _u, const double*& _sw, double& _inv_count, CvMat*& _dEdw, int& _dcount0, double* _E, int _buf_sz) - { - point = _point; - weights = _weights; - count = _count; - ivcount = _ivcount; - x0 = _x0; - l_count = _l_count; - layer_sizes = _layer_sizes; - ovcount = _ovcount; - max_count = _max_count; - u = _u; - sw = _sw; - inv_count = _inv_count; - dEdw = _dEdw; - dcount0 = _dcount0; - E = _E; - buf_sz = _buf_sz; - } - - const CvANN_MLP* point; - double** weights; - int count; - int ivcount; - CvVectors* x0; - int l_count; - CvMat* layer_sizes; - int ovcount; - int max_count; - CvVectors* u; - const double* sw; - double inv_count; - CvMat* dEdw; - int dcount0; - double* E; - int buf_sz; - - - void operator()( const cv::Range& range ) const - { - double* buf_ptr; - double** x = 0; - double **df = 0; - int total = 0; - - for(int i = 0; i < l_count; i++ ) - total += layer_sizes->data.i[i]; - CvMat* buf; - buf = cvCreateMat( 1, buf_sz, CV_64F ); - x = (double**)cvAlloc( total*2*sizeof(x[0]) ); - df = x + total; - buf_ptr = buf->data.db; - for(int i = 0; i < l_count; i++ ) - { - x[i] = buf_ptr; - df[i] = x[i] + layer_sizes->data.i[i]*dcount0; - buf_ptr += (df[i] - x[i])*2; + iter /= count; + return iter; } - for(int si = range.start; si < range.end; si++ ) - { - if (si % dcount0 != 0) continue; - int n1, n2, k; - double* w; - CvMat _w, _dEdw, hdr1, hdr2, ghdr1, ghdr2, _df; - CvMat *x1, *x2, *grad1, *grad2, *temp; - int dcount = 0; - - dcount = MIN(count - si , dcount0 ); - w = weights[0]; - grad1 = &ghdr1; grad2 = &ghdr2; - x1 = &hdr1; x2 = &hdr2; - - // grab and preprocess input data - if( x0->type == CV_32F ) + struct RPropLoop : public ParallelLoopBody { - for(int i = 0; i < dcount; i++ ) - { - const float* x0data = x0->data.fl[si+i]; - double* xdata = x[0]+i*ivcount; - for(int j = 0; j < ivcount; j++ ) - xdata[j] = x0data[j]*w[j*2] + w[j*2+1]; - } - } - else - for(int i = 0; i < dcount; i++ ) - { - const double* x0data = x0->data.db[si+i]; - double* xdata = x[0]+i*ivcount; - for(int j = 0; j < ivcount; j++ ) - xdata[j] = x0data[j]*w[j*2] + w[j*2+1]; - } - cvInitMatHeader( x1, dcount, ivcount, CV_64F, x[0] ); - - // forward pass, compute y[i]=w*x[i-1], x[i]=f(y[i]), df[i]=f'(y[i]) - for(int i = 1; i < l_count; i++ ) + RPropLoop(ANN_MLPImpl* _ann, + const Mat& _inputs, const Mat& _outputs, const Mat& _sw, + int _dcount0, vector& _dEdw, double* _E) { - cvInitMatHeader( x2, dcount, layer_sizes->data.i[i], CV_64F, x[i] ); - cvInitMatHeader( &_w, x1->cols, x2->cols, CV_64F, weights[i] ); - cvGEMM( x1, &_w, 1, 0, 0, x2 ); - _df = *x2; - _df.data.db = df[i]; - point->calc_activ_func_deriv( x2, &_df, _w.data.db + _w.rows*_w.cols ); - CV_SWAP( x1, x2, temp ); + ann = _ann; + inputs = _inputs; + outputs = _outputs; + sw = _sw.ptr(); + dcount0 = _dcount0; + dEdw = &_dEdw; + pE = _E; } - cvInitMatHeader( grad1, dcount, ovcount, CV_64F, buf_ptr ); - w = weights[l_count+1]; - grad2->data.db = buf_ptr + max_count*dcount; + ANN_MLPImpl* ann; + vector* dEdw; + Mat inputs, outputs; + const double* sw; + int dcount0; + double* pE; - // calculate error - if( u->type == CV_32F ) - for(int i = 0; i < dcount; i++ ) + void operator()( const Range& range ) const + { + double inv_count = 1./inputs.rows; + int ivcount = ann->layer_sizes.front(); + int ovcount = ann->layer_sizes.back(); + int itype = inputs.type(), otype = outputs.type(); + int count = inputs.rows; + int i, j, k, l_count = ann->layer_count(); + vector > x(l_count); + vector > df(l_count); + vector _buf(ann->max_lsize*dcount0*2); + double* buf[] = { &_buf[0], &_buf[ann->max_lsize*dcount0] }; + double E = 0; + + for( i = 0; i < l_count; i++ ) { - const float* udata = u->data.fl[si+i]; - const double* xdata = x[l_count-1] + i*ovcount; - double* gdata = grad1->data.db + i*ovcount; - double sweight = sw ? sw[si+i] : inv_count, E1 = 0; - - for(int j = 0; j < ovcount; j++ ) - { - double t = udata[j]*w[j*2] + w[j*2+1] - xdata[j]; - gdata[j] = t*sweight; - E1 += t*t; - } - *E += sweight*E1; + x[i].resize(ann->layer_sizes[i]*dcount0); + df[i].resize(ann->layer_sizes[i]*dcount0); } - else - for(int i = 0; i < dcount; i++ ) + + for( int si = range.start; si < range.end; si++ ) { - const double* udata = u->data.db[si+i]; - const double* xdata = x[l_count-1] + i*ovcount; - double* gdata = grad1->data.db + i*ovcount; - double sweight = sw ? sw[si+i] : inv_count, E1 = 0; + int i0 = si*dcount0, i1 = std::min((si + 1)*dcount0, count); + int dcount = i1 - i0; + const double* w = ann->weights[0].ptr(); - for(int j = 0; j < ovcount; j++ ) + // grab and preprocess input data + for( i = 0; i < dcount; i++ ) { - double t = udata[j]*w[j*2] + w[j*2+1] - xdata[j]; - gdata[j] = t*sweight; - E1 += t*t; - } - *E += sweight*E1; - } - - // backward pass, update dEdw - static cv::Mutex mutex; - - for(int i = l_count-1; i > 0; i-- ) - { - n1 = layer_sizes->data.i[i-1]; n2 = layer_sizes->data.i[i]; - cvInitMatHeader( &_df, dcount, n2, CV_64F, df[i] ); - cvMul( grad1, &_df, grad1 ); + const uchar* x0data_p = inputs.ptr(i0 + i); + const float* x0data_f = (const float*)x0data_p; + const double* x0data_d = (const double*)x0data_p; - { - cv::AutoLock lock(mutex); - cvInitMatHeader( &_dEdw, n1, n2, CV_64F, dEdw->data.db+(weights[i]-weights[0]) ); - cvInitMatHeader( x1, dcount, n1, CV_64F, x[i-1] ); - cvGEMM( x1, grad1, 1, &_dEdw, 1, &_dEdw, CV_GEMM_A_T ); + double* xdata = &x[0][i*ivcount]; + for( j = 0; j < ivcount; j++ ) + xdata[j] = (itype == CV_32F ? (double)x0data_f[j] : x0data_d[j])*w[j*2] + w[j*2+1]; + } + Mat x1(dcount, ivcount, CV_64F, &x[0][0]); - // update bias part of dEdw - for( k = 0; k < dcount; k++ ) + // forward pass, compute y[i]=w*x[i-1], x[i]=f(y[i]), df[i]=f'(y[i]) + for( i = 1; i < l_count; i++ ) { - double* dst = _dEdw.data.db + n1*n2; - const double* src = grad1->data.db + k*n2; - for(int j = 0; j < n2; j++ ) - dst[j] += src[j]; + Mat x2( dcount, ann->layer_sizes[i], CV_64F, &x[i][0] ); + Mat _w = ann->weights[i].rowRange(0, x1.cols); + gemm( x1, _w, 1, noArray(), 0, x2 ); + Mat _df( x2.size(), CV_64F, &df[i][0] ); + ann->calc_activ_func_deriv( x2, _df, ann->weights[i] ); + x1 = x2; } - if (i > 1) - cvInitMatHeader( &_w, n1, n2, CV_64F, weights[i] ); - } - - cvInitMatHeader( grad2, dcount, n1, CV_64F, grad2->data.db ); - if( i > 1 ) - cvGEMM( grad1, &_w, 1, 0, 0, grad2, CV_GEMM_B_T ); - CV_SWAP( grad1, grad2, temp ); - } - } - cvFree(&x); - cvReleaseMat( &buf ); -} - -}; - - -int CvANN_MLP::train_rprop( CvVectors x0, CvVectors u, const double* sw ) -{ - const int max_buf_size = 1 << 16; - CvMat* dw = 0; - CvMat* dEdw = 0; - CvMat* prev_dEdw_sign = 0; - CvMat* buf = 0; - double **x = 0, **df = 0; - int iter = -1, count = x0.count; - - CV_FUNCNAME( "CvANN_MLP::train" ); - - __BEGIN__; - - int i, ivcount, ovcount, l_count, total = 0, max_iter, buf_sz, dcount0; - double *buf_ptr; - double prev_E = DBL_MAX*0.5, epsilon; - double dw_plus, dw_minus, dw_min, dw_max; - double inv_count; - - max_iter = params.term_crit.max_iter; - epsilon = params.term_crit.epsilon; - dw_plus = params.rp_dw_plus; - dw_minus = params.rp_dw_minus; - dw_min = params.rp_dw_min; - dw_max = params.rp_dw_max; - - l_count = layer_sizes->cols; - ivcount = layer_sizes->data.i[0]; - ovcount = layer_sizes->data.i[l_count-1]; - - // allocate buffers - for( i = 0; i < l_count; i++ ) - total += layer_sizes->data.i[i]; - - CV_CALL( dw = cvCreateMat( wbuf->rows, wbuf->cols, wbuf->type )); - cvSet( dw, cvScalarAll(params.rp_dw0) ); - CV_CALL( dEdw = cvCreateMat( wbuf->rows, wbuf->cols, wbuf->type )); - cvZero( dEdw ); - CV_CALL( prev_dEdw_sign = cvCreateMat( wbuf->rows, wbuf->cols, CV_8SC1 )); - cvZero( prev_dEdw_sign ); - - inv_count = 1./count; - dcount0 = max_buf_size/(2*total); - dcount0 = MAX( dcount0, 1 ); - dcount0 = MIN( dcount0, count ); - buf_sz = dcount0*(total + max_count)*2; - - CV_CALL( buf = cvCreateMat( 1, buf_sz, CV_64F )); - - CV_CALL( x = (double**)cvAlloc( total*2*sizeof(x[0]) )); - df = x + total; - buf_ptr = buf->data.db; - - for( i = 0; i < l_count; i++ ) - { - x[i] = buf_ptr; - df[i] = x[i] + layer_sizes->data.i[i]*dcount0; - buf_ptr += (df[i] - x[i])*2; - } - - // run rprop loop - /* - y_i(t) = w_i(t)*x_{i-1}(t) - x_i(t) = f(y_i(t)) - E = sum_over_all_samples(1/2*||u - x_N||^2) - grad_N = (x_N - u)*f'(y_i) + Mat grad1(dcount, ovcount, CV_64F, buf[l_count & 1]); - MIN(dw_i{jk}(t)*dw_plus, dw_max), if dE/dw_i{jk}(t)*dE/dw_i{jk}(t-1) > 0 - dw_i{jk}(t) = MAX(dw_i{jk}(t)*dw_minus, dw_min), if dE/dw_i{jk}(t)*dE/dw_i{jk}(t-1) < 0 - dw_i{jk}(t-1) else + w = ann->weights[l_count+1].ptr(); - if (dE/dw_i{jk}(t)*dE/dw_i{jk}(t-1) < 0) - dE/dw_i{jk}(t)<-0 - else - w_i{jk}(t+1) = w_i{jk}(t) + dw_i{jk}(t) - grad_{i-1}(t) = w_i^t(t)*grad_i(t) - */ - for( iter = 0; iter < max_iter; iter++ ) - { - int n1, n2, j, k; - double E = 0; - - // first, iterate through all the samples and compute dEdw - cv::parallel_for_(cv::Range(0, count), - rprop_loop(this, weights, count, ivcount, &x0, l_count, layer_sizes, - ovcount, max_count, &u, sw, inv_count, dEdw, dcount0, &E, buf_sz) - ); + // calculate error + for( i = 0; i < dcount; i++ ) + { + const uchar* udata_p = outputs.ptr(i0+i); + const float* udata_f = (const float*)udata_p; + const double* udata_d = (const double*)udata_p; - // now update weights - for( i = 1; i < l_count; i++ ) - { - n1 = layer_sizes->data.i[i-1]; n2 = layer_sizes->data.i[i]; - for( k = 0; k <= n1; k++ ) - { - double* wk = weights[i]+k*n2; - size_t delta = wk - weights[0]; - double* dwk = dw->data.db + delta; - double* dEdwk = dEdw->data.db + delta; - char* prevEk = (char*)(prev_dEdw_sign->data.ptr + delta); + const double* xdata = &x[l_count-1][i*ovcount]; + double* gdata = grad1.ptr(i); + double sweight = sw ? sw[si+i] : inv_count, E1 = 0; - for( j = 0; j < n2; j++ ) - { - double Eval = dEdwk[j]; - double dval = dwk[j]; - double wval = wk[j]; - int s = CV_SIGN(Eval); - int ss = prevEk[j]*s; - if( ss > 0 ) + for( j = 0; j < ovcount; j++ ) { - dval *= dw_plus; - dval = MIN( dval, dw_max ); - dwk[j] = dval; - wk[j] = wval + dval*s; + double t = (otype == CV_32F ? (double)udata_f[j] : udata_d[j])*w[j*2] + w[j*2+1] - xdata[j]; + gdata[j] = t*sweight; + E1 += t*t; } - else if( ss < 0 ) + E += sweight*E1; + } + + for( i = l_count-1; i > 0; i-- ) + { + int n1 = ann->layer_sizes[i-1], n2 = ann->layer_sizes[i]; + Mat _df(dcount, n2, CV_64F, &df[i][0]); + multiply(grad1, _df, grad1); + { - dval *= dw_minus; - dval = MAX( dval, dw_min ); - prevEk[j] = 0; - dwk[j] = dval; - wk[j] = wval + dval*s; + AutoLock lock(ann->mtx); + Mat _dEdw = dEdw->at(i).rowRange(0, n1); + x1 = Mat(dcount, n1, CV_64F, &x[i-1][0]); + gemm(x1, grad1, 1, _dEdw, 1, _dEdw, GEMM_1_T); + + // update bias part of dEdw + double* dst = dEdw->at(i).ptr(n1); + for( k = 0; k < dcount; k++ ) + { + const double* src = grad1.ptr(k); + for( j = 0; j < n2; j++ ) + dst[j] += src[j]; + } } - else + + Mat grad2( dcount, n1, CV_64F, buf[i&1] ); + if( i > 1 ) { - prevEk[j] = (char)s; - wk[j] = wval + dval*s; + Mat _w = ann->weights[i].rowRange(0, n1); + gemm(grad1, _w, 1, noArray(), 0, grad2, GEMM_2_T); } - dEdwk[j] = 0.; + grad1 = grad2; } } + { + AutoLock lock(ann->mtx); + *pE += E; + } } + }; - //printf("%d. E = %g\n", iter, E); - if( fabs(prev_E - E) < epsilon ) - break; - prev_E = E; - E = 0; - } - - __END__; - - cvReleaseMat( &dw ); - cvReleaseMat( &dEdw ); - cvReleaseMat( &prev_dEdw_sign ); - cvReleaseMat( &buf ); - cvFree( &x ); - - return iter; -} - - -void CvANN_MLP::write_params( CvFileStorage* fs ) const -{ - //CV_FUNCNAME( "CvANN_MLP::write_params" ); - - __BEGIN__; - - const char* activ_func_name = activ_func == IDENTITY ? "IDENTITY" : - activ_func == SIGMOID_SYM ? "SIGMOID_SYM" : - activ_func == GAUSSIAN ? "GAUSSIAN" : 0; - - if( activ_func_name ) - cvWriteString( fs, "activation_function", activ_func_name ); - else - cvWriteInt( fs, "activation_function", activ_func ); - - if( activ_func != IDENTITY ) + int train_rprop( const Mat& inputs, const Mat& outputs, const Mat& _sw, TermCriteria termCrit ) { - cvWriteReal( fs, "f_param1", f_param1 ); - cvWriteReal( fs, "f_param2", f_param2 ); - } + const int max_buf_size = 1 << 16; + int i, iter = -1, count = inputs.rows; - cvWriteReal( fs, "min_val", min_val ); - cvWriteReal( fs, "max_val", max_val ); - cvWriteReal( fs, "min_val1", min_val1 ); - cvWriteReal( fs, "max_val1", max_val1 ); + double prev_E = DBL_MAX*0.5; - cvStartWriteStruct( fs, "training_params", CV_NODE_MAP ); - if( params.train_method == CvANN_MLP_TrainParams::BACKPROP ) - { - cvWriteString( fs, "train_method", "BACKPROP" ); - cvWriteReal( fs, "dw_scale", params.bp_dw_scale ); - cvWriteReal( fs, "moment_scale", params.bp_moment_scale ); - } - else if( params.train_method == CvANN_MLP_TrainParams::RPROP ) - { - cvWriteString( fs, "train_method", "RPROP" ); - cvWriteReal( fs, "dw0", params.rp_dw0 ); - cvWriteReal( fs, "dw_plus", params.rp_dw_plus ); - cvWriteReal( fs, "dw_minus", params.rp_dw_minus ); - cvWriteReal( fs, "dw_min", params.rp_dw_min ); - cvWriteReal( fs, "dw_max", params.rp_dw_max ); - } + int max_iter = termCrit.maxCount; + double epsilon = termCrit.epsilon; + double dw_plus = params.rpDWPlus; + double dw_minus = params.rpDWMinus; + double dw_min = params.rpDWMin; + double dw_max = params.rpDWMax; - cvStartWriteStruct( fs, "term_criteria", CV_NODE_MAP + CV_NODE_FLOW ); - if( params.term_crit.type & CV_TERMCRIT_EPS ) - cvWriteReal( fs, "epsilon", params.term_crit.epsilon ); - if( params.term_crit.type & CV_TERMCRIT_ITER ) - cvWriteInt( fs, "iterations", params.term_crit.max_iter ); - cvEndWriteStruct( fs ); + int l_count = layer_count(); - cvEndWriteStruct( fs ); + // allocate buffers + vector dw(l_count), dEdw(l_count), prev_dEdw_sign(l_count); - __END__; -} - - -void CvANN_MLP::write( CvFileStorage* fs, const char* name ) const -{ - CV_FUNCNAME( "CvANN_MLP::write" ); - - __BEGIN__; - - int i, l_count = layer_sizes->cols; - - if( !layer_sizes ) - CV_ERROR( CV_StsError, "The network has not been initialized" ); + int total = 0; + for( i = 0; i < l_count; i++ ) + { + total += layer_sizes[i]; + dw[i].create(weights[i].size(), CV_64F); + dw[i].setTo(Scalar::all(params.rpDW0)); + prev_dEdw_sign[i] = Mat::zeros(weights[i].size(), CV_8S); + dEdw[i] = Mat::zeros(weights[i].size(), CV_64F); + } - cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_ANN_MLP ); + int dcount0 = max_buf_size/(2*total); + dcount0 = std::max( dcount0, 1 ); + dcount0 = std::min( dcount0, count ); + int chunk_count = (count + dcount0 - 1)/dcount0; + + // run rprop loop + /* + y_i(t) = w_i(t)*x_{i-1}(t) + x_i(t) = f(y_i(t)) + E = sum_over_all_samples(1/2*||u - x_N||^2) + grad_N = (x_N - u)*f'(y_i) + + std::min(dw_i{jk}(t)*dw_plus, dw_max), if dE/dw_i{jk}(t)*dE/dw_i{jk}(t-1) > 0 + dw_i{jk}(t) = std::max(dw_i{jk}(t)*dw_minus, dw_min), if dE/dw_i{jk}(t)*dE/dw_i{jk}(t-1) < 0 + dw_i{jk}(t-1) else + + if (dE/dw_i{jk}(t)*dE/dw_i{jk}(t-1) < 0) + dE/dw_i{jk}(t)<-0 + else + w_i{jk}(t+1) = w_i{jk}(t) + dw_i{jk}(t) + grad_{i-1}(t) = w_i^t(t)*grad_i(t) + */ + for( iter = 0; iter < max_iter; iter++ ) + { + double E = 0; - cvWrite( fs, "layer_sizes", layer_sizes ); + for( i = 0; i < l_count; i++ ) + dEdw[i].setTo(Scalar::all(0)); - write_params( fs ); + // first, iterate through all the samples and compute dEdw + RPropLoop invoker(this, inputs, outputs, _sw, dcount0, dEdw, &E); + parallel_for_(Range(0, chunk_count), invoker); + //invoker(Range(0, chunk_count)); - cvStartWriteStruct( fs, "input_scale", CV_NODE_SEQ + CV_NODE_FLOW ); - cvWriteRawData( fs, weights[0], layer_sizes->data.i[0]*2, "d" ); - cvEndWriteStruct( fs ); + // now update weights + for( i = 1; i < l_count; i++ ) + { + int n1 = layer_sizes[i-1], n2 = layer_sizes[i]; + for( int k = 0; k <= n1; k++ ) + { + CV_Assert(weights[i].size() == Size(n2, n1+1)); + double* wk = weights[i].ptr(k); + double* dwk = dw[i].ptr(k); + double* dEdwk = dEdw[i].ptr(k); + schar* prevEk = prev_dEdw_sign[i].ptr(k); - cvStartWriteStruct( fs, "output_scale", CV_NODE_SEQ + CV_NODE_FLOW ); - cvWriteRawData( fs, weights[l_count], layer_sizes->data.i[l_count-1]*2, "d" ); - cvEndWriteStruct( fs ); + for( int j = 0; j < n2; j++ ) + { + double Eval = dEdwk[j]; + double dval = dwk[j]; + double wval = wk[j]; + int s = CV_SIGN(Eval); + int ss = prevEk[j]*s; + if( ss > 0 ) + { + dval *= dw_plus; + dval = std::min( dval, dw_max ); + dwk[j] = dval; + wk[j] = wval + dval*s; + } + else if( ss < 0 ) + { + dval *= dw_minus; + dval = std::max( dval, dw_min ); + prevEk[j] = 0; + dwk[j] = dval; + wk[j] = wval + dval*s; + } + else + { + prevEk[j] = (schar)s; + wk[j] = wval + dval*s; + } + dEdwk[j] = 0.; + } + } + } - cvStartWriteStruct( fs, "inv_output_scale", CV_NODE_SEQ + CV_NODE_FLOW ); - cvWriteRawData( fs, weights[l_count+1], layer_sizes->data.i[l_count-1]*2, "d" ); - cvEndWriteStruct( fs ); + //printf("%d. E = %g\n", iter, E); + if( fabs(prev_E - E) < epsilon ) + break; + prev_E = E; + } - cvStartWriteStruct( fs, "weights", CV_NODE_SEQ ); - for( i = 1; i < l_count; i++ ) - { - cvStartWriteStruct( fs, 0, CV_NODE_SEQ + CV_NODE_FLOW ); - cvWriteRawData( fs, weights[i], (layer_sizes->data.i[i-1]+1)*layer_sizes->data.i[i], "d" ); - cvEndWriteStruct( fs ); + return iter; } - cvEndWriteStruct( fs ); + void write_params( FileStorage& fs ) const + { + const char* activ_func_name = activ_func == IDENTITY ? "IDENTITY" : + activ_func == SIGMOID_SYM ? "SIGMOID_SYM" : + activ_func == GAUSSIAN ? "GAUSSIAN" : 0; - __END__; -} + if( activ_func_name ) + fs << "activation_function" << activ_func_name; + else + fs << "activation_function_id" << activ_func; + if( activ_func != IDENTITY ) + { + fs << "f_param1" << f_param1; + fs << "f_param2" << f_param2; + } -void CvANN_MLP::read_params( CvFileStorage* fs, CvFileNode* node ) -{ - //CV_FUNCNAME( "CvANN_MLP::read_params" ); + fs << "min_val" << min_val << "max_val" << max_val << "min_val1" << min_val1 << "max_val1" << max_val1; - __BEGIN__; + fs << "training_params" << "{"; + if( params.trainMethod == Params::BACKPROP ) + { + fs << "train_method" << "BACKPROP"; + fs << "dw_scale" << params.bpDWScale; + fs << "moment_scale" << params.bpMomentScale; + } + else if( params.trainMethod == Params::RPROP ) + { + fs << "train_method" << "RPROP"; + fs << "dw0" << params.rpDW0; + fs << "dw_plus" << params.rpDWPlus; + fs << "dw_minus" << params.rpDWMinus; + fs << "dw_min" << params.rpDWMin; + fs << "dw_max" << params.rpDWMax; + } + else + CV_Error(CV_StsError, "Unknown training method"); + + fs << "term_criteria" << "{"; + if( params.termCrit.type & TermCriteria::EPS ) + fs << "epsilon" << params.termCrit.epsilon; + if( params.termCrit.type & TermCriteria::COUNT ) + fs << "iterations" << params.termCrit.maxCount; + fs << "}" << "}"; + } - const char* activ_func_name = cvReadStringByName( fs, node, "activation_function", 0 ); - CvFileNode* tparams_node; + void write( FileStorage& fs ) const + { + if( layer_sizes.empty() ) + return; + int i, l_count = layer_count(); - if( activ_func_name ) - activ_func = strcmp( activ_func_name, "SIGMOID_SYM" ) == 0 ? SIGMOID_SYM : - strcmp( activ_func_name, "IDENTITY" ) == 0 ? IDENTITY : - strcmp( activ_func_name, "GAUSSIAN" ) == 0 ? GAUSSIAN : 0; - else - activ_func = cvReadIntByName( fs, node, "activation_function" ); + fs << "layer_sizes" << layer_sizes; - f_param1 = cvReadRealByName( fs, node, "f_param1", 0 ); - f_param2 = cvReadRealByName( fs, node, "f_param2", 0 ); + write_params( fs ); - set_activ_func( activ_func, f_param1, f_param2 ); + size_t esz = weights[0].elemSize(); - min_val = cvReadRealByName( fs, node, "min_val", 0. ); - max_val = cvReadRealByName( fs, node, "max_val", 1. ); - min_val1 = cvReadRealByName( fs, node, "min_val1", 0. ); - max_val1 = cvReadRealByName( fs, node, "max_val1", 1. ); + fs << "input_scale" << "["; + fs.writeRaw("d", weights[0].data, weights[0].total()*esz); - tparams_node = cvGetFileNodeByName( fs, node, "training_params" ); - params = CvANN_MLP_TrainParams(); + fs << "]" << "output_scale" << "["; + fs.writeRaw("d", weights[l_count].data, weights[l_count].total()*esz); - if( tparams_node ) - { - const char* tmethod_name = cvReadStringByName( fs, tparams_node, "train_method", "" ); - CvFileNode* tcrit_node; + fs << "]" << "inv_output_scale" << "["; + fs.writeRaw("d", weights[l_count+1].data, weights[l_count+1].total()*esz); - if( strcmp( tmethod_name, "BACKPROP" ) == 0 ) - { - params.train_method = CvANN_MLP_TrainParams::BACKPROP; - params.bp_dw_scale = cvReadRealByName( fs, tparams_node, "dw_scale", 0 ); - params.bp_moment_scale = cvReadRealByName( fs, tparams_node, "moment_scale", 0 ); - } - else if( strcmp( tmethod_name, "RPROP" ) == 0 ) + fs << "]" << "weights" << "["; + for( i = 1; i < l_count; i++ ) { - params.train_method = CvANN_MLP_TrainParams::RPROP; - params.rp_dw0 = cvReadRealByName( fs, tparams_node, "dw0", 0 ); - params.rp_dw_plus = cvReadRealByName( fs, tparams_node, "dw_plus", 0 ); - params.rp_dw_minus = cvReadRealByName( fs, tparams_node, "dw_minus", 0 ); - params.rp_dw_min = cvReadRealByName( fs, tparams_node, "dw_min", 0 ); - params.rp_dw_max = cvReadRealByName( fs, tparams_node, "dw_max", 0 ); + fs << "["; + fs.writeRaw("d", weights[i].data, weights[i].total()*esz); + fs << "]"; } + fs << "]"; + } - tcrit_node = cvGetFileNodeByName( fs, tparams_node, "term_criteria" ); - if( tcrit_node ) + void read_params( const FileNode& fn ) + { + String activ_func_name = (String)fn["activation_function"]; + if( !activ_func_name.empty() ) { - params.term_crit.epsilon = cvReadRealByName( fs, tcrit_node, "epsilon", -1 ); - params.term_crit.max_iter = cvReadIntByName( fs, tcrit_node, "iterations", -1 ); - params.term_crit.type = (params.term_crit.epsilon >= 0 ? CV_TERMCRIT_EPS : 0) + - (params.term_crit.max_iter >= 0 ? CV_TERMCRIT_ITER : 0); + activ_func = activ_func_name == "SIGMOID_SYM" ? SIGMOID_SYM : + activ_func_name == "IDENTITY" ? IDENTITY : + activ_func_name == "GAUSSIAN" ? GAUSSIAN : -1; + CV_Assert( activ_func >= 0 ); } - } + else + activ_func = (int)fn["activation_function_id"]; - __END__; -} + f_param1 = (double)fn["f_param1"]; + f_param2 = (double)fn["f_param2"]; + set_activ_func( activ_func, f_param1, f_param2 ); -void CvANN_MLP::read( CvFileStorage* fs, CvFileNode* node ) -{ - CvMat* _layer_sizes = 0; + min_val = (double)fn["min_val"]; + max_val = (double)fn["max_val"]; + min_val1 = (double)fn["min_val1"]; + max_val1 = (double)fn["max_val1"]; - CV_FUNCNAME( "CvANN_MLP::read" ); + FileNode tpn = fn["training_params"]; + params = Params(); - __BEGIN__; + if( !tpn.empty() ) + { + String tmethod_name = (String)tpn["train_method"]; - CvFileNode* w; - CvSeqReader reader; - int i, l_count; + if( tmethod_name == "BACKPROP" ) + { + params.trainMethod = Params::BACKPROP; + params.bpDWScale = (double)tpn["dw_scale"]; + params.bpMomentScale = (double)tpn["moment_scale"]; + } + else if( tmethod_name == "RPROP" ) + { + params.trainMethod = Params::RPROP; + params.rpDW0 = (double)tpn["dw0"]; + params.rpDWPlus = (double)tpn["dw_plus"]; + params.rpDWMinus = (double)tpn["dw_minus"]; + params.rpDWMin = (double)tpn["dw_min"]; + params.rpDWMax = (double)tpn["dw_max"]; + } + else + CV_Error(CV_StsParseError, "Unknown training method (should be BACKPROP or RPROP)"); - _layer_sizes = (CvMat*)cvReadByName( fs, node, "layer_sizes" ); - CV_CALL( create( _layer_sizes, SIGMOID_SYM, 0, 0 )); - l_count = layer_sizes->cols; + FileNode tcn = tpn["term_criteria"]; + if( !tcn.empty() ) + { + FileNode tcn_e = tcn["epsilon"]; + FileNode tcn_i = tcn["iterations"]; + params.termCrit.type = 0; + if( !tcn_e.empty() ) + { + params.termCrit.type |= TermCriteria::EPS; + params.termCrit.epsilon = (double)tcn_e; + } + if( !tcn_i.empty() ) + { + params.termCrit.type |= TermCriteria::COUNT; + params.termCrit.maxCount = (int)tcn_i; + } + } + } + } - CV_CALL( read_params( fs, node )); + void read( const FileNode& fn ) + { + clear(); - w = cvGetFileNodeByName( fs, node, "input_scale" ); - if( !w || CV_NODE_TYPE(w->tag) != CV_NODE_SEQ || - w->data.seq->total != layer_sizes->data.i[0]*2 ) - CV_ERROR( CV_StsParseError, "input_scale tag is not found or is invalid" ); + vector _layer_sizes; + fn["layer_sizes"] >> _layer_sizes; + create( _layer_sizes ); - CV_CALL( cvReadRawData( fs, w, weights[0], "d" )); + int i, l_count = layer_count(); + read_params(fn); - w = cvGetFileNodeByName( fs, node, "output_scale" ); - if( !w || CV_NODE_TYPE(w->tag) != CV_NODE_SEQ || - w->data.seq->total != layer_sizes->data.i[l_count-1]*2 ) - CV_ERROR( CV_StsParseError, "output_scale tag is not found or is invalid" ); + size_t esz = weights[0].elemSize(); - CV_CALL( cvReadRawData( fs, w, weights[l_count], "d" )); + FileNode w = fn["input_scale"]; + w.readRaw("d", weights[0].data, weights[0].total()*esz); - w = cvGetFileNodeByName( fs, node, "inv_output_scale" ); - if( !w || CV_NODE_TYPE(w->tag) != CV_NODE_SEQ || - w->data.seq->total != layer_sizes->data.i[l_count-1]*2 ) - CV_ERROR( CV_StsParseError, "inv_output_scale tag is not found or is invalid" ); + w = fn["output_scale"]; + w.readRaw("d", weights[l_count].data, weights[l_count].total()*esz); - CV_CALL( cvReadRawData( fs, w, weights[l_count+1], "d" )); + w = fn["inv_output_scale"]; + w.readRaw("d", weights[l_count+1].data, weights[l_count+1].total()*esz); - w = cvGetFileNodeByName( fs, node, "weights" ); - if( !w || CV_NODE_TYPE(w->tag) != CV_NODE_SEQ || - w->data.seq->total != l_count - 1 ) - CV_ERROR( CV_StsParseError, "weights tag is not found or is invalid" ); + FileNodeIterator w_it = fn["weights"].begin(); - cvStartReadSeq( w->data.seq, &reader ); + for( i = 1; i < l_count; i++, ++w_it ) + (*w_it).readRaw("d", weights[i].data, weights[i].total()*esz); + trained = true; + } - for( i = 1; i < l_count; i++ ) + Mat getLayerSizes() const { - w = (CvFileNode*)reader.ptr; - CV_CALL( cvReadRawData( fs, w, weights[i], "d" )); - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + return Mat_(layer_sizes, true); } - __END__; -} + Mat getWeights(int layerIdx) const + { + CV_Assert( 0 <= layerIdx && layerIdx < (int)weights.size() ); + return weights[layerIdx]; + } -using namespace cv; + bool isTrained() const + { + return trained; + } -CvANN_MLP::CvANN_MLP( const Mat& _layer_sizes, int _activ_func, - double _f_param1, double _f_param2 ) -{ - layer_sizes = wbuf = 0; - min_val = max_val = min_val1 = max_val1 = 0.; - weights = 0; - rng = &cv::theRNG(); - default_model_name = "my_nn"; - create( _layer_sizes, _activ_func, _f_param1, _f_param2 ); -} + bool isClassifier() const + { + return false; + } -void CvANN_MLP::create( const Mat& _layer_sizes, int _activ_func, - double _f_param1, double _f_param2 ) -{ - CvMat cvlayer_sizes = _layer_sizes; - create( &cvlayer_sizes, _activ_func, _f_param1, _f_param2 ); -} + int getVarCount() const + { + return layer_sizes.empty() ? 0 : layer_sizes[0]; + } -int CvANN_MLP::train( const Mat& _inputs, const Mat& _outputs, - const Mat& _sample_weights, const Mat& _sample_idx, - CvANN_MLP_TrainParams _params, int flags ) -{ - CvMat inputs = _inputs, outputs = _outputs, sweights = _sample_weights, sidx = _sample_idx; - return train(&inputs, &outputs, sweights.data.ptr ? &sweights : 0, - sidx.data.ptr ? &sidx : 0, _params, flags); -} + String getDefaultModelName() const + { + return "opencv_ml_ann_mlp"; + } -float CvANN_MLP::predict( const Mat& _inputs, Mat& _outputs ) const -{ - CV_Assert(layer_sizes != 0); - _outputs.create(_inputs.rows, layer_sizes->data.i[layer_sizes->cols-1], _inputs.type()); - CvMat inputs = _inputs, outputs = _outputs; + vector layer_sizes; + vector weights; + double f_param1, f_param2; + double min_val, max_val, min_val1, max_val1; + int activ_func; + int max_lsize, max_buf_sz; + Params params; + RNG rng; + Mutex mtx; + bool trained; +}; - return predict(&inputs, &outputs); + +Ptr ANN_MLP::create(const ANN_MLP::Params& params) +{ + Ptr ann = makePtr(params); + return ann; } +}} + /* End of file. */ diff --git a/modules/ml/src/boost.cpp b/modules/ml/src/boost.cpp index a22e13a53209de9ca707cd696ebc853c03671f80..5e0b30733809cbbe6cd6ef90a91bb5b0e9f97016 100644 --- a/modules/ml/src/boost.cpp +++ b/modules/ml/src/boost.cpp @@ -7,9 +7,11 @@ // copy or use the software. // // -// Intel License Agreement +// License Agreement +// For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -22,7 +24,7 @@ // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // -// * The name of Intel Corporation may not be used to endorse or promote products +// * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and @@ -40,1309 +42,255 @@ #include "precomp.hpp" +namespace cv { namespace ml { + static inline double log_ratio( double val ) { const double eps = 1e-5; - - val = MAX( val, eps ); - val = MIN( val, 1. - eps ); + val = std::max( val, eps ); + val = std::min( val, 1. - eps ); return log( val/(1. - val) ); } -CvBoostParams::CvBoostParams() +Boost::Params::Params() { - boost_type = CvBoost::REAL; - weak_count = 100; - weight_trim_rate = 0.95; - cv_folds = 0; - max_depth = 1; + boostType = Boost::REAL; + weakCount = 100; + weightTrimRate = 0.95; + CVFolds = 0; + maxDepth = 1; } -CvBoostParams::CvBoostParams( int _boost_type, int _weak_count, - double _weight_trim_rate, int _max_depth, - bool _use_surrogates, const float* _priors ) +Boost::Params::Params( int _boostType, int _weak_count, + double _weightTrimRate, int _maxDepth, + bool _use_surrogates, const Mat& _priors ) { - boost_type = _boost_type; - weak_count = _weak_count; - weight_trim_rate = _weight_trim_rate; - split_criteria = CvBoost::DEFAULT; - cv_folds = 0; - max_depth = _max_depth; - use_surrogates = _use_surrogates; + boostType = _boostType; + weakCount = _weak_count; + weightTrimRate = _weightTrimRate; + CVFolds = 0; + maxDepth = _maxDepth; + useSurrogates = _use_surrogates; priors = _priors; } - -///////////////////////////////// CvBoostTree /////////////////////////////////// - -CvBoostTree::CvBoostTree() -{ - ensemble = 0; -} - - -CvBoostTree::~CvBoostTree() -{ - clear(); -} - - -void -CvBoostTree::clear() -{ - CvDTree::clear(); - ensemble = 0; -} - - -bool -CvBoostTree::train( CvDTreeTrainData* _train_data, - const CvMat* _subsample_idx, CvBoost* _ensemble ) -{ - clear(); - ensemble = _ensemble; - data = _train_data; - data->shared = true; - return do_train( _subsample_idx ); -} - - -bool -CvBoostTree::train( const CvMat*, int, const CvMat*, const CvMat*, - const CvMat*, const CvMat*, const CvMat*, CvDTreeParams ) -{ - assert(0); - return false; -} - - -bool -CvBoostTree::train( CvDTreeTrainData*, const CvMat* ) -{ - assert(0); - return false; -} - - -void -CvBoostTree::scale( double _scale ) -{ - CvDTreeNode* node = root; - - // traverse the tree and scale all the node values - for(;;) - { - CvDTreeNode* parent; - for(;;) - { - node->value *= _scale; - if( !node->left ) - break; - node = node->left; - } - - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) - ; - - if( !parent ) - break; - - node = parent->right; - } -} - - -void -CvBoostTree::try_split_node( CvDTreeNode* node ) -{ - CvDTree::try_split_node( node ); - - if( !node->left ) - { - // if the node has not been split, - // store the responses for the corresponding training samples - double* weak_eval = ensemble->get_weak_response()->data.db; - cv::AutoBuffer inn_buf(node->sample_count); - const int* labels = data->get_cv_labels( node, (int*)inn_buf ); - int i, count = node->sample_count; - double value = node->value; - - for( i = 0; i < count; i++ ) - weak_eval[labels[i]] = value; - } -} - - -double -CvBoostTree::calc_node_dir( CvDTreeNode* node ) -{ - char* dir = (char*)data->direction->data.ptr; - const double* weights = ensemble->get_subtree_weights()->data.db; - int i, n = node->sample_count, vi = node->split->var_idx; - double L, R; - - assert( !node->split->inversed ); - - if( data->get_var_type(vi) >= 0 ) // split on categorical var - { - cv::AutoBuffer inn_buf(n); - const int* cat_labels = data->get_cat_var_data( node, vi, (int*)inn_buf ); - const int* subset = node->split->subset; - double sum = 0, sum_abs = 0; - - for( i = 0; i < n; i++ ) - { - int idx = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; - double w = weights[i]; - int d = idx >= 0 ? CV_DTREE_CAT_DIR(idx,subset) : 0; - sum += d*w; sum_abs += (d & 1)*w; - dir[i] = (char)d; - } - - R = (sum_abs + sum) * 0.5; - L = (sum_abs - sum) * 0.5; - } - else // split on ordered var - { - cv::AutoBuffer inn_buf(2*n*sizeof(int)+n*sizeof(float)); - float* values_buf = (float*)(uchar*)inn_buf; - int* sorted_indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = sorted_indices_buf + n; - const float* values = 0; - const int* sorted_indices = 0; - data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); - int split_point = node->split->ord.split_point; - int n1 = node->get_num_valid(vi); - - assert( 0 <= split_point && split_point < n1-1 ); - L = R = 0; - - for( i = 0; i <= split_point; i++ ) - { - int idx = sorted_indices[i]; - double w = weights[idx]; - dir[idx] = (char)-1; - L += w; - } - - for( ; i < n1; i++ ) - { - int idx = sorted_indices[i]; - double w = weights[idx]; - dir[idx] = (char)1; - R += w; - } - - for( ; i < n; i++ ) - dir[sorted_indices[i]] = (char)0; - } - - node->maxlr = MAX( L, R ); - return node->split->quality/(L + R); -} - - -CvDTreeSplit* -CvBoostTree::find_split_ord_class( CvDTreeNode* node, int vi, float init_quality, - CvDTreeSplit* _split, uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - - const double* weights = ensemble->get_subtree_weights()->data.db; - int n = node->sample_count; - int n1 = node->get_num_valid(vi); - - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(n*(3*sizeof(int)+sizeof(float))); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - float* values_buf = (float*)ext_buf; - int* sorted_indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = sorted_indices_buf + n; - const float* values = 0; - const int* sorted_indices = 0; - data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); - int* responses_buf = sorted_indices_buf + n; - const int* responses = data->get_class_labels( node, responses_buf ); - const double* rcw0 = weights + n; - double lcw[2] = {0,0}, rcw[2]; - int i, best_i = -1; - double best_val = init_quality; - int boost_type = ensemble->get_params().boost_type; - int split_criteria = ensemble->get_params().split_criteria; - - rcw[0] = rcw0[0]; rcw[1] = rcw0[1]; - for( i = n1; i < n; i++ ) - { - int idx = sorted_indices[i]; - double w = weights[idx]; - rcw[responses[idx]] -= w; - } - - if( split_criteria != CvBoost::GINI && split_criteria != CvBoost::MISCLASS ) - split_criteria = boost_type == CvBoost::DISCRETE ? CvBoost::MISCLASS : CvBoost::GINI; - - if( split_criteria == CvBoost::GINI ) - { - double L = 0, R = rcw[0] + rcw[1]; - double lsum2 = 0, rsum2 = rcw[0]*rcw[0] + rcw[1]*rcw[1]; - - for( i = 0; i < n1 - 1; i++ ) - { - int idx = sorted_indices[i]; - double w = weights[idx], w2 = w*w; - double lv, rv; - idx = responses[idx]; - L += w; R -= w; - lv = lcw[idx]; rv = rcw[idx]; - lsum2 += 2*lv*w + w2; - rsum2 -= 2*rv*w - w2; - lcw[idx] = lv + w; rcw[idx] = rv - w; - - if( values[i] + epsilon < values[i+1] ) - { - double val = (lsum2*R + rsum2*L)/(L*R); - if( best_val < val ) - { - best_val = val; - best_i = i; - } - } - } - } - else - { - for( i = 0; i < n1 - 1; i++ ) - { - int idx = sorted_indices[i]; - double w = weights[idx]; - idx = responses[idx]; - lcw[idx] += w; - rcw[idx] -= w; - - if( values[i] + epsilon < values[i+1] ) - { - double val = lcw[0] + rcw[1], val2 = lcw[1] + rcw[0]; - val = MAX(val, val2); - if( best_val < val ) - { - best_val = val; - best_i = i; - } - } - } - } - - CvDTreeSplit* split = 0; - if( best_i >= 0 ) - { - split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); - split->var_idx = vi; - split->ord.c = (values[best_i] + values[best_i+1])*0.5f; - split->ord.split_point = best_i; - split->inversed = 0; - split->quality = (float)best_val; - } - return split; -} - -template -class LessThanPtr +class DTreesImplForBoost : public DTreesImpl { public: - bool operator()(T* a, T* b) const { return *a < *b; } -}; - -CvDTreeSplit* -CvBoostTree::find_split_cat_class( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) -{ - int ci = data->get_var_type(vi); - int n = node->sample_count; - int mi = data->cat_count->data.i[ci]; - - int base_size = (2*mi+3)*sizeof(double) + mi*sizeof(double*); - cv::AutoBuffer inn_buf((2*mi+3)*sizeof(double) + mi*sizeof(double*)); - if( !_ext_buf) - inn_buf.allocate( base_size + 2*n*sizeof(int) ); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - - int* cat_labels_buf = (int*)ext_buf; - const int* cat_labels = data->get_cat_var_data(node, vi, cat_labels_buf); - int* responses_buf = cat_labels_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - double lcw[2]={0,0}, rcw[2]={0,0}; - - double* cjk = (double*)cv::alignPtr(base_buf,sizeof(double))+2; - const double* weights = ensemble->get_subtree_weights()->data.db; - double** dbl_ptr = (double**)(cjk + 2*mi); - int i, j, k, idx; - double L = 0, R; - double best_val = init_quality; - int best_subset = -1, subset_i; - int boost_type = ensemble->get_params().boost_type; - int split_criteria = ensemble->get_params().split_criteria; - - // init array of counters: - // c_{jk} - number of samples that have vi-th input variable = j and response = k. - for( j = -1; j < mi; j++ ) - cjk[j*2] = cjk[j*2+1] = 0; - - for( i = 0; i < n; i++ ) - { - double w = weights[i]; - j = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; - k = responses[i]; - cjk[j*2 + k] += w; - } - - for( j = 0; j < mi; j++ ) - { - rcw[0] += cjk[j*2]; - rcw[1] += cjk[j*2+1]; - dbl_ptr[j] = cjk + j*2 + 1; - } - - R = rcw[0] + rcw[1]; - - if( split_criteria != CvBoost::GINI && split_criteria != CvBoost::MISCLASS ) - split_criteria = boost_type == CvBoost::DISCRETE ? CvBoost::MISCLASS : CvBoost::GINI; - - // sort rows of c_jk by increasing c_j,1 - // (i.e. by the weight of samples in j-th category that belong to class 1) - std::sort(dbl_ptr, dbl_ptr + mi, LessThanPtr()); - - for( subset_i = 0; subset_i < mi-1; subset_i++ ) - { - idx = (int)(dbl_ptr[subset_i] - cjk)/2; - const double* crow = cjk + idx*2; - double w0 = crow[0], w1 = crow[1]; - double weight = w0 + w1; - - if( weight < FLT_EPSILON ) - continue; - - lcw[0] += w0; rcw[0] -= w0; - lcw[1] += w1; rcw[1] -= w1; - - if( split_criteria == CvBoost::GINI ) - { - double lsum2 = lcw[0]*lcw[0] + lcw[1]*lcw[1]; - double rsum2 = rcw[0]*rcw[0] + rcw[1]*rcw[1]; - - L += weight; - R -= weight; - - if( L > FLT_EPSILON && R > FLT_EPSILON ) - { - double val = (lsum2*R + rsum2*L)/(L*R); - if( best_val < val ) - { - best_val = val; - best_subset = subset_i; - } - } - } - else - { - double val = lcw[0] + rcw[1]; - double val2 = lcw[1] + rcw[0]; + DTreesImplForBoost() {} + virtual ~DTreesImplForBoost() {} - val = MAX(val, val2); - if( best_val < val ) - { - best_val = val; - best_subset = subset_i; - } - } - } + bool isClassifier() const { return true; } - CvDTreeSplit* split = 0; - if( best_subset >= 0 ) + void setBParams(const Boost::Params& p) { - split = _split ? _split : data->new_split_cat( 0, -1.0f); - split->var_idx = vi; - split->quality = (float)best_val; - memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); - for( i = 0; i <= best_subset; i++ ) - { - idx = (int)(dbl_ptr[i] - cjk) >> 1; - split->subset[idx >> 5] |= 1 << (idx & 31); - } + bparams = p; } - return split; -} - -CvDTreeSplit* -CvBoostTree::find_split_ord_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - const double* weights = ensemble->get_subtree_weights()->data.db; - int n = node->sample_count; - int n1 = node->get_num_valid(vi); - - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(2*n*(sizeof(int)+sizeof(float))); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - - float* values_buf = (float*)ext_buf; - int* indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = indices_buf + n; - const float* values = 0; - const int* indices = 0; - data->get_ord_var_data( node, vi, values_buf, indices_buf, &values, &indices, sample_indices_buf ); - float* responses_buf = (float*)(indices_buf + n); - const float* responses = data->get_ord_responses( node, responses_buf, sample_indices_buf ); - - int i, best_i = -1; - double L = 0, R = weights[n]; - double best_val = init_quality, lsum = 0, rsum = node->value*R; - - // compensate for missing values - for( i = n1; i < n; i++ ) + Boost::Params getBParams() const { - int idx = indices[i]; - double w = weights[idx]; - rsum -= responses[idx]*w; - R -= w; + return bparams; } - // find the optimal split - for( i = 0; i < n1 - 1; i++ ) + void clear() { - int idx = indices[i]; - double w = weights[idx]; - double t = responses[idx]*w; - L += w; R -= w; - lsum += t; rsum -= t; - - if( values[i] + epsilon < values[i+1] ) - { - double val = (lsum*lsum*R + rsum*rsum*L)/(L*R); - if( best_val < val ) - { - best_val = val; - best_i = i; - } - } + DTreesImpl::clear(); } - CvDTreeSplit* split = 0; - if( best_i >= 0 ) + void startTraining( const Ptr& trainData, int flags ) { - split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); - split->var_idx = vi; - split->ord.c = (values[best_i] + values[best_i+1])*0.5f; - split->ord.split_point = best_i; - split->inversed = 0; - split->quality = (float)best_val; - } - return split; -} - + DTreesImpl::startTraining(trainData, flags); + sumResult.assign(w->sidx.size(), 0.); -CvDTreeSplit* -CvBoostTree::find_split_cat_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) -{ - const double* weights = ensemble->get_subtree_weights()->data.db; - int ci = data->get_var_type(vi); - int n = node->sample_count; - int mi = data->cat_count->data.i[ci]; - int base_size = (2*mi+3)*sizeof(double) + mi*sizeof(double*); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + n*(2*sizeof(int) + sizeof(float))); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - - int* cat_labels_buf = (int*)ext_buf; - const int* cat_labels = data->get_cat_var_data(node, vi, cat_labels_buf); - float* responses_buf = (float*)(cat_labels_buf + n); - int* sample_indices_buf = (int*)(responses_buf + n); - const float* responses = data->get_ord_responses(node, responses_buf, sample_indices_buf); - - double* sum = (double*)cv::alignPtr(base_buf,sizeof(double)) + 1; - double* counts = sum + mi + 1; - double** sum_ptr = (double**)(counts + mi); - double L = 0, R = 0, best_val = init_quality, lsum = 0, rsum = 0; - int i, best_subset = -1, subset_i; - - for( i = -1; i < mi; i++ ) - sum[i] = counts[i] = 0; - - // calculate sum response and weight of each category of the input var - for( i = 0; i < n; i++ ) - { - int idx = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; - double w = weights[i]; - double s = sum[idx] + responses[i]*w; - double nc = counts[idx] + w; - sum[idx] = s; - counts[idx] = nc; - } - - // calculate average response in each category - for( i = 0; i < mi; i++ ) - { - R += counts[i]; - rsum += sum[i]; - sum[i] = fabs(counts[i]) > DBL_EPSILON ? sum[i]/counts[i] : 0; - sum_ptr[i] = sum + i; - } - - std::sort(sum_ptr, sum_ptr + mi, LessThanPtr()); - - // revert back to unnormalized sums - // (there should be a very little loss in accuracy) - for( i = 0; i < mi; i++ ) - sum[i] *= counts[i]; - - for( subset_i = 0; subset_i < mi-1; subset_i++ ) - { - int idx = (int)(sum_ptr[subset_i] - sum); - double ni = counts[idx]; - - if( ni > FLT_EPSILON ) - { - double s = sum[idx]; - lsum += s; L += ni; - rsum -= s; R -= ni; - - if( L > FLT_EPSILON && R > FLT_EPSILON ) - { - double val = (lsum*lsum*R + rsum*rsum*L)/(L*R); - if( best_val < val ) - { - best_val = val; - best_subset = subset_i; - } - } - } - } - - CvDTreeSplit* split = 0; - if( best_subset >= 0 ) - { - split = _split ? _split : data->new_split_cat( 0, -1.0f); - split->var_idx = vi; - split->quality = (float)best_val; - memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); - for( i = 0; i <= best_subset; i++ ) + if( bparams.boostType != Boost::DISCRETE ) { - int idx = (int)(sum_ptr[i] - sum); - split->subset[idx >> 5] |= 1 << (idx & 31); - } - } - return split; -} - + _isClassifier = false; + int i, n = (int)w->cat_responses.size(); + w->ord_responses.resize(n); -CvDTreeSplit* -CvBoostTree::find_surrogate_split_ord( CvDTreeNode* node, int vi, uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - int n = node->sample_count; - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(n*(2*sizeof(int)+sizeof(float))); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - float* values_buf = (float*)ext_buf; - int* indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = indices_buf + n; - const float* values = 0; - const int* indices = 0; - data->get_ord_var_data( node, vi, values_buf, indices_buf, &values, &indices, sample_indices_buf ); - - const double* weights = ensemble->get_subtree_weights()->data.db; - const char* dir = (char*)data->direction->data.ptr; - int n1 = node->get_num_valid(vi); - // LL - number of samples that both the primary and the surrogate splits send to the left - // LR - ... primary split sends to the left and the surrogate split sends to the right - // RL - ... primary split sends to the right and the surrogate split sends to the left - // RR - ... both send to the right - int i, best_i = -1, best_inversed = 0; - double best_val; - double LL = 0, RL = 0, LR, RR; - double worst_val = node->maxlr; - double sum = 0, sum_abs = 0; - best_val = worst_val; - - for( i = 0; i < n1; i++ ) - { - int idx = indices[i]; - double w = weights[idx]; - int d = dir[idx]; - sum += d*w; sum_abs += (d & 1)*w; - } - - // sum_abs = R + L; sum = R - L - RR = (sum_abs + sum)*0.5; - LR = (sum_abs - sum)*0.5; - - // initially all the samples are sent to the right by the surrogate split, - // LR of them are sent to the left by primary split, and RR - to the right. - // now iteratively compute LL, LR, RL and RR for every possible surrogate split value. - for( i = 0; i < n1 - 1; i++ ) - { - int idx = indices[i]; - double w = weights[idx]; - int d = dir[idx]; - - if( d < 0 ) - { - LL += w; LR -= w; - if( LL + RR > best_val && values[i] + epsilon < values[i+1] ) + double a = -1, b = 1; + if( bparams.boostType == Boost::LOGIT ) { - best_val = LL + RR; - best_i = i; best_inversed = 0; + a = -2, b = 2; } + for( i = 0; i < n; i++ ) + w->ord_responses[i] = w->cat_responses[i] > 0 ? b : a; } - else if( d > 0 ) - { - RL += w; RR -= w; - if( RL + LR > best_val && values[i] + epsilon < values[i+1] ) - { - best_val = RL + LR; - best_i = i; best_inversed = 1; - } - } - } - - return best_i >= 0 && best_val > node->maxlr ? data->new_split_ord( vi, - (values[best_i] + values[best_i+1])*0.5f, best_i, - best_inversed, (float)best_val ) : 0; -} - - -CvDTreeSplit* -CvBoostTree::find_surrogate_split_cat( CvDTreeNode* node, int vi, uchar* _ext_buf ) -{ - const char* dir = (char*)data->direction->data.ptr; - const double* weights = ensemble->get_subtree_weights()->data.db; - int n = node->sample_count; - int i, mi = data->cat_count->data.i[data->get_var_type(vi)]; - - int base_size = (2*mi+3)*sizeof(double); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + n*sizeof(int)); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - int* cat_labels_buf = (int*)ext_buf; - const int* cat_labels = data->get_cat_var_data(node, vi, cat_labels_buf); - - // LL - number of samples that both the primary and the surrogate splits send to the left - // LR - ... primary split sends to the left and the surrogate split sends to the right - // RL - ... primary split sends to the right and the surrogate split sends to the left - // RR - ... both send to the right - CvDTreeSplit* split = data->new_split_cat( vi, 0 ); - double best_val = 0; - double* lc = (double*)cv::alignPtr(cat_labels_buf + n, sizeof(double)) + 1; - double* rc = lc + mi + 1; - - for( i = -1; i < mi; i++ ) - lc[i] = rc[i] = 0; - - // 1. for each category calculate the weight of samples - // sent to the left (lc) and to the right (rc) by the primary split - for( i = 0; i < n; i++ ) - { - int idx = ((cat_labels[i] == 65535) && data->is_buf_16u) ? -1 : cat_labels[i]; - double w = weights[i]; - int d = dir[i]; - double sum = lc[idx] + d*w; - double sum_abs = rc[idx] + (d & 1)*w; - lc[idx] = sum; rc[idx] = sum_abs; - } - - for( i = 0; i < mi; i++ ) - { - double sum = lc[i]; - double sum_abs = rc[i]; - lc[i] = (sum_abs - sum) * 0.5; - rc[i] = (sum_abs + sum) * 0.5; - } - - // 2. now form the split. - // in each category send all the samples to the same direction as majority - for( i = 0; i < mi; i++ ) - { - double lval = lc[i], rval = rc[i]; - if( lval > rval ) - { - split->subset[i >> 5] |= 1 << (i & 31); - best_val += lval; - } - else - best_val += rval; - } - - split->quality = (float)best_val; - if( split->quality <= node->maxlr ) - cvSetRemoveByPtr( data->split_heap, split ), split = 0; - - return split; -} - - -void -CvBoostTree::calc_node_value( CvDTreeNode* node ) -{ - int i, n = node->sample_count; - const double* weights = ensemble->get_weights()->data.db; - cv::AutoBuffer inn_buf(n*(sizeof(int) + ( data->is_classifier ? sizeof(int) : sizeof(int) + sizeof(float)))); - int* labels_buf = (int*)(uchar*)inn_buf; - const int* labels = data->get_cv_labels(node, labels_buf); - double* subtree_weights = ensemble->get_subtree_weights()->data.db; - double rcw[2] = {0,0}; - int boost_type = ensemble->get_params().boost_type; - - if( data->is_classifier ) - { - int* _responses_buf = labels_buf + n; - const int* _responses = data->get_class_labels(node, _responses_buf); - int m = data->get_num_classes(); - int* cls_count = data->counts->data.i; - for( int k = 0; k < m; k++ ) - cls_count[k] = 0; - - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - double w = weights[idx]; - int r = _responses[i]; - rcw[r] += w; - cls_count[r]++; - subtree_weights[i] = w; - } - - node->class_idx = rcw[1] > rcw[0]; - - if( boost_type == CvBoost::DISCRETE ) - { - // ignore cat_map for responses, and use {-1,1}, - // as the whole ensemble response is computes as sign(sum_i(weak_response_i) - node->value = node->class_idx*2 - 1; - } - else - { - double p = rcw[1]/(rcw[0] + rcw[1]); - assert( boost_type == CvBoost::REAL ); - - // store log-ratio of the probability - node->value = 0.5*log_ratio(p); - } - } - else - { - // in case of regression tree: - // * node value is 1/n*sum_i(Y_i), where Y_i is i-th response, - // n is the number of samples in the node. - // * node risk is the sum of squared errors: sum_i((Y_i - )^2) - double sum = 0, sum2 = 0, iw; - float* values_buf = (float*)(labels_buf + n); - int* sample_indices_buf = (int*)(values_buf + n); - const float* values = data->get_ord_responses(node, values_buf, sample_indices_buf); - - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - double w = weights[idx]/*priors[values[i] > 0]*/; - double t = values[i]; - rcw[0] += w; - subtree_weights[i] = w; - sum += t*w; - sum2 += t*t*w; - } - - iw = 1./rcw[0]; - node->value = sum*iw; - node->node_risk = sum2 - (sum*iw)*sum; - - // renormalize the risk, as in try_split_node the unweighted formula - // sqrt(risk)/n is used, rather than sqrt(risk)/sum(weights_i) - node->node_risk *= n*iw*n*iw; - } - - // store summary weights - subtree_weights[n] = rcw[0]; - subtree_weights[n+1] = rcw[1]; -} - - -void CvBoostTree::read( CvFileStorage* fs, CvFileNode* fnode, CvBoost* _ensemble, CvDTreeTrainData* _data ) -{ - CvDTree::read( fs, fnode, _data ); - ensemble = _ensemble; -} - -void CvBoostTree::read( CvFileStorage*, CvFileNode* ) -{ - assert(0); -} - -void CvBoostTree::read( CvFileStorage* _fs, CvFileNode* _node, - CvDTreeTrainData* _data ) -{ - CvDTree::read( _fs, _node, _data ); -} - - -/////////////////////////////////// CvBoost ///////////////////////////////////// - -CvBoost::CvBoost() -{ - data = 0; - weak = 0; - default_model_name = "my_boost_tree"; - - active_vars = active_vars_abs = orig_response = sum_response = weak_eval = - subsample_mask = weights = subtree_weights = 0; - have_active_cat_vars = have_subsample = false; - - clear(); -} - - -void CvBoost::prune( CvSlice slice ) -{ - if( weak && weak->total > 0 ) - { - CvSeqReader reader; - int i, count = cvSliceLength( slice, weak ); - - cvStartReadSeq( weak, &reader ); - cvSetSeqReaderPos( &reader, slice.start_index ); - - for( i = 0; i < count; i++ ) - { - CvBoostTree* w; - CV_READ_SEQ_ELEM( w, reader ); - delete w; - } - - cvSeqRemoveSlice( weak, slice ); - } -} - - -void CvBoost::clear() -{ - if( weak ) - { - prune( CV_WHOLE_SEQ ); - cvReleaseMemStorage( &weak->storage ); - } - if( data ) - delete data; - weak = 0; - data = 0; - cvReleaseMat( &active_vars ); - cvReleaseMat( &active_vars_abs ); - cvReleaseMat( &orig_response ); - cvReleaseMat( &sum_response ); - cvReleaseMat( &weak_eval ); - cvReleaseMat( &subsample_mask ); - cvReleaseMat( &weights ); - cvReleaseMat( &subtree_weights ); - - have_subsample = false; -} - - -CvBoost::~CvBoost() -{ - clear(); -} - - -CvBoost::CvBoost( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, - const CvMat* _sample_idx, const CvMat* _var_type, - const CvMat* _missing_mask, CvBoostParams _params ) -{ - weak = 0; - data = 0; - default_model_name = "my_boost_tree"; - - active_vars = active_vars_abs = orig_response = sum_response = weak_eval = - subsample_mask = weights = subtree_weights = 0; - - train( _train_data, _tflag, _responses, _var_idx, _sample_idx, - _var_type, _missing_mask, _params ); -} - - -bool -CvBoost::set_params( const CvBoostParams& _params ) -{ - bool ok = false; - - CV_FUNCNAME( "CvBoost::set_params" ); - - __BEGIN__; - - params = _params; - if( params.boost_type != DISCRETE && params.boost_type != REAL && - params.boost_type != LOGIT && params.boost_type != GENTLE ) - CV_ERROR( CV_StsBadArg, "Unknown/unsupported boosting type" ); - - params.weak_count = MAX( params.weak_count, 1 ); - params.weight_trim_rate = MAX( params.weight_trim_rate, 0. ); - params.weight_trim_rate = MIN( params.weight_trim_rate, 1. ); - if( params.weight_trim_rate < FLT_EPSILON ) - params.weight_trim_rate = 1.f; - - if( params.boost_type == DISCRETE && - params.split_criteria != GINI && params.split_criteria != MISCLASS ) - params.split_criteria = MISCLASS; - if( params.boost_type == REAL && - params.split_criteria != GINI && params.split_criteria != MISCLASS ) - params.split_criteria = GINI; - if( (params.boost_type == LOGIT || params.boost_type == GENTLE) && - params.split_criteria != SQERR ) - params.split_criteria = SQERR; - - ok = true; - - __END__; - - return ok; -} - - -bool -CvBoost::train( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, - const CvMat* _sample_idx, const CvMat* _var_type, - const CvMat* _missing_mask, - CvBoostParams _params, bool _update ) -{ - bool ok = false; - CvMemStorage* storage = 0; - - CV_FUNCNAME( "CvBoost::train" ); - - __BEGIN__; - - int i; - - set_params( _params ); - - cvReleaseMat( &active_vars ); - cvReleaseMat( &active_vars_abs ); - - if( !_update || !data ) - { - clear(); - data = new CvDTreeTrainData( _train_data, _tflag, _responses, _var_idx, - _sample_idx, _var_type, _missing_mask, _params, true, true ); - - if( data->get_num_classes() != 2 ) - CV_ERROR( CV_StsNotImplemented, - "Boosted trees can only be used for 2-class classification." ); - CV_CALL( storage = cvCreateMemStorage() ); - weak = cvCreateSeq( 0, sizeof(CvSeq), sizeof(CvBoostTree*), storage ); - storage = 0; - } - else - { - data->set_data( _train_data, _tflag, _responses, _var_idx, - _sample_idx, _var_type, _missing_mask, _params, true, true, true ); - } - - if ( (_params.boost_type == LOGIT) || (_params.boost_type == GENTLE) ) - data->do_responses_copy(); - - update_weights( 0 ); - - for( i = 0; i < params.weak_count; i++ ) - { - CvBoostTree* tree = new CvBoostTree; - if( !tree->train( data, subsample_mask, this ) ) - { - delete tree; - break; - } - //cvCheckArr( get_weak_response()); - cvSeqPush( weak, &tree ); - update_weights( tree ); - trim_weights(); - if( cvCountNonZero(subsample_mask) == 0 ) - break; - } - if(weak->total > 0) - { - get_active_vars(); // recompute active_vars* maps and condensed_idx's in the splits. - data->is_classifier = true; - data->free_train_data(); - ok = true; + normalizeWeights(); } - else - clear(); - - __END__; - return ok; -} - -bool CvBoost::train( CvMLData* _data, - CvBoostParams _params, - bool update ) -{ - bool result = false; - - CV_FUNCNAME( "CvBoost::train" ); - - __BEGIN__; - - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* missing = _data->get_missing(); - const CvMat* var_types = _data->get_var_types(); - const CvMat* train_sidx = _data->get_train_sample_idx(); - const CvMat* var_idx = _data->get_var_idx(); - - CV_CALL( result = train( values, CV_ROW_SAMPLE, response, var_idx, - train_sidx, var_types, missing, _params, update ) ); - - __END__; - - return result; -} - -void CvBoost::initialize_weights(double (&p)[2]) -{ - p[0] = 1.; - p[1] = 1.; -} - -void -CvBoost::update_weights( CvBoostTree* tree ) -{ - CV_FUNCNAME( "CvBoost::update_weights" ); - - __BEGIN__; - - int i, n = data->sample_count; - double sumw = 0.; - int step = 0; - float* fdata = 0; - int *sample_idx_buf; - const int* sample_idx = 0; - cv::AutoBuffer inn_buf; - size_t _buf_size = (params.boost_type == LOGIT) || (params.boost_type == GENTLE) ? (size_t)(data->sample_count)*sizeof(int) : 0; - if( !tree ) - _buf_size += n*sizeof(int); - else - { - if( have_subsample ) - _buf_size += data->get_length_subbuf()*(sizeof(float)+sizeof(uchar)); - } - inn_buf.allocate(_buf_size); - uchar* cur_buf_pos = (uchar*)inn_buf; - - if ( (params.boost_type == LOGIT) || (params.boost_type == GENTLE) ) - { - step = CV_IS_MAT_CONT(data->responses_copy->type) ? - 1 : data->responses_copy->step / CV_ELEM_SIZE(data->responses_copy->type); - fdata = data->responses_copy->data.fl; - sample_idx_buf = (int*)cur_buf_pos; - cur_buf_pos = (uchar*)(sample_idx_buf + data->sample_count); - sample_idx = data->get_sample_indices( data->data_root, sample_idx_buf ); - } - CvMat* dtree_data_buf = data->buf; - size_t length_buf_row = data->get_length_subbuf(); - if( !tree ) // before training the first tree, initialize weights and other parameters + void normalizeWeights() { - int* class_labels_buf = (int*)cur_buf_pos; - cur_buf_pos = (uchar*)(class_labels_buf + n); - const int* class_labels = data->get_class_labels(data->data_root, class_labels_buf); - // in case of logitboost and gentle adaboost each weak tree is a regression tree, - // so we need to convert class labels to floating-point values - - double w0 = 1./ n; - double p[2] = { 1., 1. }; - initialize_weights(p); - - cvReleaseMat( &orig_response ); - cvReleaseMat( &sum_response ); - cvReleaseMat( &weak_eval ); - cvReleaseMat( &subsample_mask ); - cvReleaseMat( &weights ); - cvReleaseMat( &subtree_weights ); - - CV_CALL( orig_response = cvCreateMat( 1, n, CV_32S )); - CV_CALL( weak_eval = cvCreateMat( 1, n, CV_64F )); - CV_CALL( subsample_mask = cvCreateMat( 1, n, CV_8U )); - CV_CALL( weights = cvCreateMat( 1, n, CV_64F )); - CV_CALL( subtree_weights = cvCreateMat( 1, n + 2, CV_64F )); - - if( data->have_priors ) + int i, n = (int)w->sidx.size(); + double sumw = 0, a, b; + for( i = 0; i < n; i++ ) + sumw += w->sample_weights[w->sidx[i]]; + if( sumw > DBL_EPSILON ) { - // compute weight scale for each class from their prior probabilities - int c1 = 0; - for( i = 0; i < n; i++ ) - c1 += class_labels[i]; - p[0] = data->priors->data.db[0]*(c1 < n ? 1./(n - c1) : 0.); - p[1] = data->priors->data.db[1]*(c1 > 0 ? 1./c1 : 0.); - p[0] /= p[0] + p[1]; - p[1] = 1. - p[0]; + a = 1./sumw; + b = 0; } - - if (data->is_buf_16u) + else { - unsigned short* labels = (unsigned short*)(dtree_data_buf->data.s + data->data_root->buf_idx*length_buf_row + - data->data_root->offset + (data->work_var_count-1)*data->sample_count); - for( i = 0; i < n; i++ ) - { - // save original categorical responses {0,1}, convert them to {-1,1} - orig_response->data.i[i] = class_labels[i]*2 - 1; - // make all the samples active at start. - // later, in trim_weights() deactivate/reactive again some, if need - subsample_mask->data.ptr[i] = (uchar)1; - // make all the initial weights the same. - weights->data.db[i] = w0*p[class_labels[i]]; - // set the labels to find (from within weak tree learning proc) - // the particular sample weight, and where to store the response. - labels[i] = (unsigned short)i; - } + a = 0; + b = 1; } - else + for( i = 0; i < n; i++ ) { - int* labels = dtree_data_buf->data.i + data->data_root->buf_idx*length_buf_row + - data->data_root->offset + (data->work_var_count-1)*data->sample_count; - - for( i = 0; i < n; i++ ) - { - // save original categorical responses {0,1}, convert them to {-1,1} - orig_response->data.i[i] = class_labels[i]*2 - 1; - // make all the samples active at start. - // later, in trim_weights() deactivate/reactive again some, if need - subsample_mask->data.ptr[i] = (uchar)1; - // make all the initial weights the same. - weights->data.db[i] = w0*p[class_labels[i]]; - // set the labels to find (from within weak tree learning proc) - // the particular sample weight, and where to store the response. - labels[i] = i; - } + double& wval = w->sample_weights[w->sidx[i]]; + wval = wval*a + b; } + } - if( params.boost_type == LOGIT ) - { - CV_CALL( sum_response = cvCreateMat( 1, n, CV_64F )); + void endTraining() + { + DTreesImpl::endTraining(); + vector e; + std::swap(sumResult, e); + } - for( i = 0; i < n; i++ ) + void scaleTree( int root, double scale ) + { + int nidx = root, pidx = 0; + Node *node = 0; + + // traverse the tree and save all the nodes in depth-first order + for(;;) + { + for(;;) { - sum_response->data.db[i] = 0; - fdata[sample_idx[i]*step] = orig_response->data.i[i] > 0 ? 2.f : -2.f; + node = &nodes[nidx]; + node->value *= scale; + if( node->left < 0 ) + break; + nidx = node->left; } - // in case of logitboost each weak tree is a regression tree. - // the target function values are recalculated for each of the trees - data->is_classifier = false; - } - else if( params.boost_type == GENTLE ) - { - for( i = 0; i < n; i++ ) - fdata[sample_idx[i]*step] = (float)orig_response->data.i[i]; + for( pidx = node->parent; pidx >= 0 && nodes[pidx].right == nidx; + nidx = pidx, pidx = nodes[pidx].parent ) + ; + + if( pidx < 0 ) + break; - data->is_classifier = false; + nidx = nodes[pidx].right; } } - else + + void calcValue( int nidx, const vector& _sidx ) { - // at this moment, for all the samples that participated in the training of the most - // recent weak classifier we know the responses. For other samples we need to compute them - if( have_subsample ) + DTreesImpl::calcValue(nidx, _sidx); + WNode* node = &w->wnodes[nidx]; + if( bparams.boostType == Boost::DISCRETE ) + { + node->value = node->class_idx == 0 ? -1 : 1; + } + else if( bparams.boostType == Boost::REAL ) { - float* values = (float*)cur_buf_pos; - cur_buf_pos = (uchar*)(values + data->get_length_subbuf()); - uchar* missing = cur_buf_pos; - cur_buf_pos = missing + data->get_length_subbuf() * (size_t)CV_ELEM_SIZE(data->buf->type); + double p = (node->value+1)*0.5; + node->value = 0.5*log_ratio(p); + } + } - CvMat _sample, _mask; + bool train( const Ptr& trainData, int flags ) + { + Params dp(bparams.maxDepth, bparams.minSampleCount, bparams.regressionAccuracy, + bparams.useSurrogates, bparams.maxCategories, 0, + false, false, bparams.priors); + setDParams(dp); + startTraining(trainData, flags); + int treeidx, ntrees = bparams.weakCount >= 0 ? bparams.weakCount : 10000; + vector sidx = w->sidx; - // invert the subsample mask - cvXorS( subsample_mask, cvScalar(1.), subsample_mask ); - data->get_vectors( subsample_mask, values, missing, 0 ); + for( treeidx = 0; treeidx < ntrees; treeidx++ ) + { + int root = addTree( sidx ); + if( root < 0 ) + return false; + updateWeightsAndTrim( treeidx, sidx ); + } + endTraining(); + return true; + } - _sample = cvMat( 1, data->var_count, CV_32F ); - _mask = cvMat( 1, data->var_count, CV_8U ); + void updateWeightsAndTrim( int treeidx, vector& sidx ) + { + int i, n = (int)w->sidx.size(); + int nvars = (int)varIdx.size(); + double sumw = 0., C = 1.; + cv::AutoBuffer buf(n + nvars); + double* result = buf; + float* sbuf = (float*)(result + n); + Mat sample(1, nvars, CV_32F, sbuf); + int predictFlags = bparams.boostType == Boost::DISCRETE ? (PREDICT_MAX_VOTE | RAW_OUTPUT) : PREDICT_SUM; + predictFlags |= COMPRESSED_INPUT; - // run tree through all the non-processed samples - for( i = 0; i < n; i++ ) - if( subsample_mask->data.ptr[i] ) - { - _sample.data.fl = values; - _mask.data.ptr = missing; - values += _sample.cols; - missing += _mask.cols; - weak_eval->data.db[i] = tree->predict( &_sample, &_mask, true )->value; - } + for( i = 0; i < n; i++ ) + { + w->data->getSample(varIdx, w->sidx[i], sbuf ); + result[i] = predictTrees(Range(treeidx, treeidx+1), sample, predictFlags); } // now update weights and other parameters for each type of boosting - if( params.boost_type == DISCRETE ) + if( bparams.boostType == Boost::DISCRETE ) { // Discrete AdaBoost: // weak_eval[i] (=f(x_i)) is in {-1,1} // err = sum(w_i*(f(x_i) != y_i))/sum(w_i) // C = log((1-err)/err) // w_i *= exp(C*(f(x_i) != y_i)) - - double C, err = 0.; - double scale[] = { 1., 0. }; + double err = 0.; for( i = 0; i < n; i++ ) { - double w = weights->data.db[i]; - sumw += w; - err += w*(weak_eval->data.db[i] != orig_response->data.i[i]); + int si = w->sidx[i]; + double wval = w->sample_weights[si]; + sumw += wval; + err += wval*(result[i] != w->cat_responses[si]); } if( sumw != 0 ) err /= sumw; - C = err = -log_ratio( err ); - scale[1] = exp(err); + C = -log_ratio( err ); + double scale = std::exp(C); sumw = 0; for( i = 0; i < n; i++ ) { - double w = weights->data.db[i]* - scale[weak_eval->data.db[i] != orig_response->data.i[i]]; - sumw += w; - weights->data.db[i] = w; + int si = w->sidx[i]; + double wval = w->sample_weights[si]; + if( result[i] != w->cat_responses[si] ) + wval *= scale; + sumw += wval; + w->sample_weights[si] = wval; } - tree->scale( C ); + scaleTree(roots[treeidx], C); } - else if( params.boost_type == REAL ) + else if( bparams.boostType == Boost::REAL || bparams.boostType == Boost::GENTLE ) { // Real AdaBoost: // weak_eval[i] = f(x_i) = 0.5*log(p(x_i)/(1-p(x_i))), p(x_i)=P(y=1|x_i) // w_i *= exp(-y_i*f(x_i)) - for( i = 0; i < n; i++ ) - weak_eval->data.db[i] *= -orig_response->data.i[i]; - - cvExp( weak_eval, weak_eval ); - + // Gentle AdaBoost: + // weak_eval[i] = f(x_i) in [-1,1] + // w_i *= exp(-y_i*f(x_i)) for( i = 0; i < n; i++ ) { - double w = weights->data.db[i]*weak_eval->data.db[i]; - sumw += w; - weights->data.db[i] = w; + int si = w->sidx[i]; + CV_Assert( std::abs(w->ord_responses[si]) == 1 ); + double wval = w->sample_weights[si]*std::exp(-result[i]*w->ord_responses[si]); + sumw += wval; + w->sample_weights[si] = wval; } } - else if( params.boost_type == LOGIT ) + else if( bparams.boostType == Boost::LOGIT ) { // LogitBoost: // weak_eval[i] = f(x_i) in [-z_max,z_max] @@ -1353,810 +301,223 @@ CvBoost::update_weights( CvBoostTree* tree ) // w_i = p(x_i)*1(1 - p(x_i)) // z_i = ((y_i+1)/2 - p(x_i))/(p(x_i)*(1 - p(x_i))) // store z_i to the data->data_root as the new target responses - const double lb_weight_thresh = FLT_EPSILON; const double lb_z_max = 10.; - /*float* responses_buf = data->get_resp_float_buf(); - const float* responses = 0; - data->get_ord_responses(data->data_root, responses_buf, &responses);*/ - - /*if( weak->total == 7 ) - putchar('*');*/ - - for( i = 0; i < n; i++ ) - { - double s = sum_response->data.db[i] + 0.5*weak_eval->data.db[i]; - sum_response->data.db[i] = s; - weak_eval->data.db[i] = -2*s; - } - - cvExp( weak_eval, weak_eval ); for( i = 0; i < n; i++ ) { - double p = 1./(1. + weak_eval->data.db[i]); - double w = p*(1 - p), z; - w = MAX( w, lb_weight_thresh ); - weights->data.db[i] = w; - sumw += w; - if( orig_response->data.i[i] > 0 ) + int si = w->sidx[i]; + sumResult[i] += 0.5*result[i]; + double p = 1./(1 + std::exp(-2*sumResult[i])); + double wval = std::max( p*(1 - p), lb_weight_thresh ), z; + w->sample_weights[si] = wval; + sumw += wval; + if( w->ord_responses[si] > 0 ) { z = 1./p; - fdata[sample_idx[i]*step] = (float)MIN(z, lb_z_max); + w->ord_responses[si] = std::min(z, lb_z_max); } else { z = 1./(1-p); - fdata[sample_idx[i]*step] = (float)-MIN(z, lb_z_max); + w->ord_responses[si] = -std::min(z, lb_z_max); } } } else - { - // Gentle AdaBoost: - // weak_eval[i] = f(x_i) in [-1,1] - // w_i *= exp(-y_i*f(x_i)) - assert( params.boost_type == GENTLE ); - - for( i = 0; i < n; i++ ) - weak_eval->data.db[i] *= -orig_response->data.i[i]; - - cvExp( weak_eval, weak_eval ); + CV_Error(CV_StsNotImplemented, "Unknown boosting type"); + /*if( bparams.boostType != Boost::LOGIT ) + { + double err = 0; for( i = 0; i < n; i++ ) { - double w = weights->data.db[i] * weak_eval->data.db[i]; - weights->data.db[i] = w; - sumw += w; + sumResult[i] += result[i]*C; + if( bparams.boostType != Boost::DISCRETE ) + err += sumResult[i]*w->ord_responses[w->sidx[i]] < 0; + else + err += sumResult[i]*w->cat_responses[w->sidx[i]] < 0; } - } - } - - // renormalize weights - if( sumw > FLT_EPSILON ) - { - sumw = 1./sumw; - for( i = 0; i < n; ++i ) - weights->data.db[i] *= sumw; - } - - __END__; -} - - -void -CvBoost::trim_weights() -{ - //CV_FUNCNAME( "CvBoost::trim_weights" ); - - __BEGIN__; - - int i, count = data->sample_count, nz_count = 0; - double sum, threshold; + printf("%d trees. C=%.2f, training error=%.1f%%, working set size=%d (out of %d)\n", (int)roots.size(), C, err*100./n, (int)sidx.size(), n); + }*/ - if( params.weight_trim_rate <= 0. || params.weight_trim_rate >= 1. ) - EXIT; - - // use weak_eval as temporary buffer for sorted weights - cvCopy( weights, weak_eval ); - - std::sort(weak_eval->data.db, weak_eval->data.db + count); - - // as weight trimming occurs immediately after updating the weights, - // where they are renormalized, we assume that the weight sum = 1. - sum = 1. - params.weight_trim_rate; - - for( i = 0; i < count; i++ ) - { - double w = weak_eval->data.db[i]; - if( sum <= 0 ) - break; - sum -= w; - } - - threshold = i < count ? weak_eval->data.db[i] : DBL_MAX; - - for( i = 0; i < count; i++ ) - { - double w = weights->data.db[i]; - int f = w >= threshold; - subsample_mask->data.ptr[i] = (uchar)f; - nz_count += f; - } - - have_subsample = nz_count < count; - - __END__; -} - - -const CvMat* -CvBoost::get_active_vars( bool absolute_idx ) -{ - CvMat* mask = 0; - CvMat* inv_map = 0; - CvMat* result = 0; + // renormalize weights + if( sumw > FLT_EPSILON ) + normalizeWeights(); - CV_FUNCNAME( "CvBoost::get_active_vars" ); + if( bparams.weightTrimRate <= 0. || bparams.weightTrimRate >= 1. ) + return; - __BEGIN__; + for( i = 0; i < n; i++ ) + result[i] = w->sample_weights[w->sidx[i]]; + std::sort(result, result + n); - if( !weak ) - CV_ERROR( CV_StsError, "The boosted tree ensemble has not been trained yet" ); + // as weight trimming occurs immediately after updating the weights, + // where they are renormalized, we assume that the weight sum = 1. + sumw = 1. - bparams.weightTrimRate; - if( !active_vars || !active_vars_abs ) - { - CvSeqReader reader; - int i, j, nactive_vars; - CvBoostTree* wtree; - const CvDTreeNode* node; - - assert(!active_vars && !active_vars_abs); - mask = cvCreateMat( 1, data->var_count, CV_8U ); - inv_map = cvCreateMat( 1, data->var_count, CV_32S ); - cvZero( mask ); - cvSet( inv_map, cvScalar(-1) ); - - // first pass: compute the mask of used variables - cvStartReadSeq( weak, &reader ); - for( i = 0; i < weak->total; i++ ) + for( i = 0; i < n; i++ ) { - CV_READ_SEQ_ELEM(wtree, reader); - - node = wtree->get_root(); - assert( node != 0 ); - for(;;) - { - const CvDTreeNode* parent; - for(;;) - { - CvDTreeSplit* split = node->split; - for( ; split != 0; split = split->next ) - mask->data.ptr[split->var_idx] = 1; - if( !node->left ) - break; - node = node->left; - } - - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) - ; - - if( !parent ) - break; - - node = parent->right; - } + double wval = result[i]; + if( sumw <= 0 ) + break; + sumw -= wval; } - nactive_vars = cvCountNonZero(mask); + double threshold = i < n ? result[i] : DBL_MAX; + sidx.clear(); - //if ( nactive_vars > 0 ) + for( i = 0; i < n; i++ ) { - active_vars = cvCreateMat( 1, nactive_vars, CV_32S ); - active_vars_abs = cvCreateMat( 1, nactive_vars, CV_32S ); - - have_active_cat_vars = false; - - for( i = j = 0; i < data->var_count; i++ ) - { - if( mask->data.ptr[i] ) - { - active_vars->data.i[j] = i; - active_vars_abs->data.i[j] = data->var_idx ? data->var_idx->data.i[i] : i; - inv_map->data.i[i] = j; - if( data->var_type->data.i[i] >= 0 ) - have_active_cat_vars = true; - j++; - } - } - - - // second pass: now compute the condensed indices - cvStartReadSeq( weak, &reader ); - for( i = 0; i < weak->total; i++ ) - { - CV_READ_SEQ_ELEM(wtree, reader); - node = wtree->get_root(); - for(;;) - { - const CvDTreeNode* parent; - for(;;) - { - CvDTreeSplit* split = node->split; - for( ; split != 0; split = split->next ) - { - split->condensed_idx = inv_map->data.i[split->var_idx]; - assert( split->condensed_idx >= 0 ); - } - - if( !node->left ) - break; - node = node->left; - } - - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) - ; - - if( !parent ) - break; - - node = parent->right; - } - } + int si = w->sidx[i]; + if( w->sample_weights[si] >= threshold ) + sidx.push_back(si); } } - result = absolute_idx ? active_vars_abs : active_vars; - - __END__; - - cvReleaseMat( &mask ); - cvReleaseMat( &inv_map ); - - return result; -} - - -float -CvBoost::predict( const CvMat* _sample, const CvMat* _missing, - CvMat* weak_responses, CvSlice slice, - bool raw_mode, bool return_sum ) const -{ - float value = -FLT_MAX; - - CvSeqReader reader; - double sum = 0; - int wstep = 0; - const float* sample_data; - - if( !weak ) - CV_Error( CV_StsError, "The boosted tree ensemble has not been trained yet" ); - - if( !CV_IS_MAT(_sample) || CV_MAT_TYPE(_sample->type) != CV_32FC1 || - (_sample->cols != 1 && _sample->rows != 1) || - (_sample->cols + _sample->rows - 1 != data->var_all && !raw_mode) || - (active_vars && _sample->cols + _sample->rows - 1 != active_vars->cols && raw_mode) ) - CV_Error( CV_StsBadArg, - "the input sample must be 1d floating-point vector with the same " - "number of elements as the total number of variables or " - "as the number of variables used for training" ); - - if( _missing ) + float predictTrees( const Range& range, const Mat& sample, int flags0 ) const { - if( !CV_IS_MAT(_missing) || !CV_IS_MASK_ARR(_missing) || - !CV_ARE_SIZES_EQ(_missing, _sample) ) - CV_Error( CV_StsBadArg, - "the missing data mask must be 8-bit vector of the same size as input sample" ); + int flags = (flags0 & ~PREDICT_MASK) | PREDICT_SUM; + float val = DTreesImpl::predictTrees(range, sample, flags); + if( flags != flags0 ) + { + int ival = (int)(val > 0); + if( !(flags0 & RAW_OUTPUT) ) + ival = classLabels[ival]; + val = (float)ival; + } + return val; } - int i, weak_count = cvSliceLength( slice, weak ); - if( weak_count >= weak->total ) + void writeTrainingParams( FileStorage& fs ) const { - weak_count = weak->total; - slice.start_index = 0; - } + fs << "boosting_type" << + (bparams.boostType == Boost::DISCRETE ? "DiscreteAdaboost" : + bparams.boostType == Boost::REAL ? "RealAdaboost" : + bparams.boostType == Boost::LOGIT ? "LogitBoost" : + bparams.boostType == Boost::GENTLE ? "GentleAdaboost" : "Unknown"); - if( weak_responses ) - { - if( !CV_IS_MAT(weak_responses) || - CV_MAT_TYPE(weak_responses->type) != CV_32FC1 || - (weak_responses->cols != 1 && weak_responses->rows != 1) || - weak_responses->cols + weak_responses->rows - 1 != weak_count ) - CV_Error( CV_StsBadArg, - "The output matrix of weak classifier responses must be valid " - "floating-point vector of the same number of components as the length of input slice" ); - wstep = CV_IS_MAT_CONT(weak_responses->type) ? 1 : weak_responses->step/sizeof(float); + DTreesImpl::writeTrainingParams(fs); + fs << "weight_trimming_rate" << bparams.weightTrimRate; } - int var_count = active_vars->cols; - const int* vtype = data->var_type->data.i; - const int* cmap = data->cat_map->data.i; - const int* cofs = data->cat_ofs->data.i; - - cv::Mat sample = cv::cvarrToMat(_sample); - cv::Mat missing; - if(!_missing) - missing = cv::cvarrToMat(_missing); - - // if need, preprocess the input vector - if( !raw_mode ) + void write( FileStorage& fs ) const { - int sstep, mstep = 0; - const float* src_sample; - const uchar* src_mask = 0; - float* dst_sample; - uchar* dst_mask; - const int* vidx = active_vars->data.i; - const int* vidx_abs = active_vars_abs->data.i; - bool have_mask = _missing != 0; + if( roots.empty() ) + CV_Error( CV_StsBadArg, "RTrees have not been trained" ); - sample = cv::Mat(1, var_count, CV_32FC1); - missing = cv::Mat(1, var_count, CV_8UC1); + writeParams(fs); - dst_sample = sample.ptr(); - dst_mask = missing.ptr(); + int k, ntrees = (int)roots.size(); - src_sample = _sample->data.fl; - sstep = CV_IS_MAT_CONT(_sample->type) ? 1 : _sample->step/sizeof(src_sample[0]); + fs << "ntrees" << ntrees + << "trees" << "["; - if( _missing ) + for( k = 0; k < ntrees; k++ ) { - src_mask = _missing->data.ptr; - mstep = CV_IS_MAT_CONT(_missing->type) ? 1 : _missing->step; - } - - for( i = 0; i < var_count; i++ ) - { - int idx = vidx[i], idx_abs = vidx_abs[i]; - float val = src_sample[idx_abs*sstep]; - int ci = vtype[idx]; - uchar m = src_mask ? src_mask[idx_abs*mstep] : (uchar)0; - - if( ci >= 0 ) - { - int a = cofs[ci], b = (ci+1 >= data->cat_ofs->cols) ? data->cat_map->cols : cofs[ci+1], - c = a; - int ival = cvRound(val); - if ( (ival != val) && (!m) ) - CV_Error( CV_StsBadArg, - "one of input categorical variable is not an integer" ); - - while( a < b ) - { - c = (a + b) >> 1; - if( ival < cmap[c] ) - b = c; - else if( ival > cmap[c] ) - a = c+1; - else - break; - } - - if( c < 0 || ival != cmap[c] ) - { - m = 1; - have_mask = true; - } - else - { - val = (float)(c - cofs[ci]); - } - } - - dst_sample[i] = val; - dst_mask[i] = m; + fs << "{"; + writeTree(fs, roots[k]); + fs << "}"; } - if( !have_mask ) - missing.release(); - } - else - { - if( !CV_IS_MAT_CONT(_sample->type & (_missing ? _missing->type : -1)) ) - CV_Error( CV_StsBadArg, "In raw mode the input vectors must be continuous" ); + fs << "]"; } - cvStartReadSeq( weak, &reader ); - cvSetSeqReaderPos( &reader, slice.start_index ); - - sample_data = sample.ptr(); - - if( !have_active_cat_vars && missing.empty() && !weak_responses ) - { - for( i = 0; i < weak_count; i++ ) - { - CvBoostTree* wtree; - const CvDTreeNode* node; - CV_READ_SEQ_ELEM( wtree, reader ); - - node = wtree->get_root(); - while( node->left ) - { - CvDTreeSplit* split = node->split; - int vi = split->condensed_idx; - float val = sample_data[vi]; - int dir = val <= split->ord.c ? -1 : 1; - if( split->inversed ) - dir = -dir; - node = dir < 0 ? node->left : node->right; - } - sum += node->value; - } - } - else + void readParams( const FileNode& fn ) { - const int* avars = active_vars->data.i; - const uchar* m = !missing.empty() ? missing.ptr() : 0; - - // full-featured version - for( i = 0; i < weak_count; i++ ) - { - CvBoostTree* wtree; - const CvDTreeNode* node; - CV_READ_SEQ_ELEM( wtree, reader ); - - node = wtree->get_root(); - while( node->left ) - { - const CvDTreeSplit* split = node->split; - int dir = 0; - for( ; !dir && split != 0; split = split->next ) - { - int vi = split->condensed_idx; - int ci = vtype[avars[vi]]; - float val = sample_data[vi]; - if( m && m[vi] ) - continue; - if( ci < 0 ) // ordered - dir = val <= split->ord.c ? -1 : 1; - else // categorical - { - int c = cvRound(val); - dir = CV_DTREE_CAT_DIR(c, split->subset); - } - if( split->inversed ) - dir = -dir; - } + DTreesImpl::readParams(fn); + bparams.maxDepth = params0.maxDepth; + bparams.minSampleCount = params0.minSampleCount; + bparams.regressionAccuracy = params0.regressionAccuracy; + bparams.useSurrogates = params0.useSurrogates; + bparams.maxCategories = params0.maxCategories; + bparams.priors = params0.priors; - if( !dir ) - { - int diff = node->right->sample_count - node->left->sample_count; - dir = diff < 0 ? -1 : 1; - } - node = dir < 0 ? node->left : node->right; - } - if( weak_responses ) - weak_responses->data.fl[i*wstep] = (float)node->value; - sum += node->value; - } + FileNode tparams_node = fn["training_params"]; + String bts = (String)tparams_node["boosting_type"]; + bparams.boostType = (bts == "DiscreteAdaboost" ? Boost::DISCRETE : + bts == "RealAdaboost" ? Boost::REAL : + bts == "LogitBoost" ? Boost::LOGIT : + bts == "GentleAdaboost" ? Boost::GENTLE : -1); + _isClassifier = bparams.boostType == Boost::DISCRETE; + bparams.weightTrimRate = (double)tparams_node["weight_trimming_rate"]; } - if( return_sum ) - value = (float)sum; - else + void read( const FileNode& fn ) { - int cls_idx = sum >= 0; - if( raw_mode ) - value = (float)cls_idx; - else - value = (float)cmap[cofs[vtype[data->var_count]] + cls_idx]; - } + clear(); - return value; -} + int ntrees = (int)fn["ntrees"]; + readParams(fn); -float CvBoost::calc_error( CvMLData* _data, int type, std::vector *resp ) -{ - float err = 0; - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* missing = _data->get_missing(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - const CvMat* var_types = _data->get_var_types(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(response->type) ? - 1 : response->step / CV_ELEM_SIZE(response->type); - bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; - float* pred_resp = 0; - if( resp && (sample_count > 0) ) - { - resp->resize( sample_count ); - pred_resp = &((*resp)[0]); - } - if ( is_classifier ) - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample, miss; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - if( missing ) - cvGetRow( missing, &miss, si ); - float r = (float)predict( &sample, missing ? &miss : 0 ); - if( pred_resp ) - pred_resp[i] = r; - int d = fabs((double)r - response->data.fl[si*r_step]) <= FLT_EPSILON ? 0 : 1; - err += d; - } - err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; - } - else - { - for( int i = 0; i < sample_count; i++ ) + FileNode trees_node = fn["trees"]; + FileNodeIterator it = trees_node.begin(); + CV_Assert( ntrees == (int)trees_node.size() ); + + for( int treeidx = 0; treeidx < ntrees; treeidx++, ++it ) { - CvMat sample, miss; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - if( missing ) - cvGetRow( missing, &miss, si ); - float r = (float)predict( &sample, missing ? &miss : 0 ); - if( pred_resp ) - pred_resp[i] = r; - float d = r - response->data.fl[si*r_step]; - err += d*d; + FileNode nfn = (*it)["nodes"]; + readTree(nfn); } - err = sample_count ? err / (float)sample_count : -FLT_MAX; } - return err; -} -void CvBoost::write_params( CvFileStorage* fs ) const -{ - const char* boost_type_str = - params.boost_type == DISCRETE ? "DiscreteAdaboost" : - params.boost_type == REAL ? "RealAdaboost" : - params.boost_type == LOGIT ? "LogitBoost" : - params.boost_type == GENTLE ? "GentleAdaboost" : 0; - - const char* split_crit_str = - params.split_criteria == DEFAULT ? "Default" : - params.split_criteria == GINI ? "Gini" : - params.boost_type == MISCLASS ? "Misclassification" : - params.boost_type == SQERR ? "SquaredErr" : 0; - - if( boost_type_str ) - cvWriteString( fs, "boosting_type", boost_type_str ); - else - cvWriteInt( fs, "boosting_type", params.boost_type ); - - if( split_crit_str ) - cvWriteString( fs, "splitting_criteria", split_crit_str ); - else - cvWriteInt( fs, "splitting_criteria", params.split_criteria ); - - cvWriteInt( fs, "ntrees", weak->total ); - cvWriteReal( fs, "weight_trimming_rate", params.weight_trim_rate ); - - data->write_params( fs ); -} + Boost::Params bparams; + vector sumResult; +}; -void CvBoost::read_params( CvFileStorage* fs, CvFileNode* fnode ) +class BoostImpl : public Boost { - CV_FUNCNAME( "CvBoost::read_params" ); - - __BEGIN__; - - CvFileNode* temp; - - if( !fnode || !CV_NODE_IS_MAP(fnode->tag) ) - return; - - data = new CvDTreeTrainData(); - CV_CALL( data->read_params(fs, fnode)); - data->shared = true; - - params.max_depth = data->params.max_depth; - params.min_sample_count = data->params.min_sample_count; - params.max_categories = data->params.max_categories; - params.priors = data->params.priors; - params.regression_accuracy = data->params.regression_accuracy; - params.use_surrogates = data->params.use_surrogates; +public: + BoostImpl() {} + virtual ~BoostImpl() {} - temp = cvGetFileNodeByName( fs, fnode, "boosting_type" ); - if( !temp ) - return; + String getDefaultModelName() const { return "opencv_ml_boost"; } - if( temp && CV_NODE_IS_STRING(temp->tag) ) + bool train( const Ptr& trainData, int flags ) { - const char* boost_type_str = cvReadString( temp, "" ); - params.boost_type = strcmp( boost_type_str, "DiscreteAdaboost" ) == 0 ? DISCRETE : - strcmp( boost_type_str, "RealAdaboost" ) == 0 ? REAL : - strcmp( boost_type_str, "LogitBoost" ) == 0 ? LOGIT : - strcmp( boost_type_str, "GentleAdaboost" ) == 0 ? GENTLE : -1; + return impl.train(trainData, flags); } - else - params.boost_type = cvReadInt( temp, -1 ); - - if( params.boost_type < DISCRETE || params.boost_type > GENTLE ) - CV_ERROR( CV_StsBadArg, "Unknown boosting type" ); - temp = cvGetFileNodeByName( fs, fnode, "splitting_criteria" ); - if( temp && CV_NODE_IS_STRING(temp->tag) ) + float predict( InputArray samples, OutputArray results, int flags ) const { - const char* split_crit_str = cvReadString( temp, "" ); - params.split_criteria = strcmp( split_crit_str, "Default" ) == 0 ? DEFAULT : - strcmp( split_crit_str, "Gini" ) == 0 ? GINI : - strcmp( split_crit_str, "Misclassification" ) == 0 ? MISCLASS : - strcmp( split_crit_str, "SquaredErr" ) == 0 ? SQERR : -1; + return impl.predict(samples, results, flags); } - else - params.split_criteria = cvReadInt( temp, -1 ); - - if( params.split_criteria < DEFAULT || params.boost_type > SQERR ) - CV_ERROR( CV_StsBadArg, "Unknown boosting type" ); - - params.weak_count = cvReadIntByName( fs, fnode, "ntrees" ); - params.weight_trim_rate = cvReadRealByName( fs, fnode, "weight_trimming_rate", 0. ); - - __END__; -} - - - -void -CvBoost::read( CvFileStorage* fs, CvFileNode* node ) -{ - CV_FUNCNAME( "CvBoost::read" ); - - __BEGIN__; - - CvSeqReader reader; - CvFileNode* trees_fnode; - CvMemStorage* storage; - int i, ntrees; - - clear(); - read_params( fs, node ); - - if( !data ) - EXIT; - trees_fnode = cvGetFileNodeByName( fs, node, "trees" ); - if( !trees_fnode || !CV_NODE_IS_SEQ(trees_fnode->tag) ) - CV_ERROR( CV_StsParseError, " tag is missing" ); - - cvStartReadSeq( trees_fnode->data.seq, &reader ); - ntrees = trees_fnode->data.seq->total; - - if( ntrees != params.weak_count ) - CV_ERROR( CV_StsUnmatchedSizes, - "The number of trees stored does not match tag value" ); - - CV_CALL( storage = cvCreateMemStorage() ); - weak = cvCreateSeq( 0, sizeof(CvSeq), sizeof(CvBoostTree*), storage ); - - for( i = 0; i < ntrees; i++ ) + void write( FileStorage& fs ) const { - CvBoostTree* tree = new CvBoostTree(); - CV_CALL(tree->read( fs, (CvFileNode*)reader.ptr, this, data )); - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); - cvSeqPush( weak, &tree ); + impl.write(fs); } - get_active_vars(); - - __END__; -} - - -void -CvBoost::write( CvFileStorage* fs, const char* name ) const -{ - CV_FUNCNAME( "CvBoost::write" ); - - __BEGIN__; - - CvSeqReader reader; - int i; - - cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_BOOSTING ); - if( !weak ) - CV_ERROR( CV_StsBadArg, "The classifier has not been trained yet" ); - - write_params( fs ); - cvStartWriteStruct( fs, "trees", CV_NODE_SEQ ); - - cvStartReadSeq( weak, &reader ); - - for( i = 0; i < weak->total; i++ ) + void read( const FileNode& fn ) { - CvBoostTree* tree; - CV_READ_SEQ_ELEM( tree, reader ); - cvStartWriteStruct( fs, 0, CV_NODE_MAP ); - tree->write( fs ); - cvEndWriteStruct( fs ); + impl.read(fn); } - cvEndWriteStruct( fs ); - cvEndWriteStruct( fs ); - - __END__; -} - - -CvMat* -CvBoost::get_weights() -{ - return weights; -} - - -CvMat* -CvBoost::get_subtree_weights() -{ - return subtree_weights; -} - - -CvMat* -CvBoost::get_weak_response() -{ - return weak_eval; -} - - -const CvBoostParams& -CvBoost::get_params() const -{ - return params; -} + void setBParams(const Params& p) { impl.setBParams(p); } + Params getBParams() const { return impl.getBParams(); } -CvSeq* CvBoost::get_weak_predictors() -{ - return weak; -} + int getVarCount() const { return impl.getVarCount(); } -const CvDTreeTrainData* CvBoost::get_data() const -{ - return data; -} + bool isTrained() const { return impl.isTrained(); } + bool isClassifier() const { return impl.isClassifier(); } -using namespace cv; + const vector& getRoots() const { return impl.getRoots(); } + const vector& getNodes() const { return impl.getNodes(); } + const vector& getSplits() const { return impl.getSplits(); } + const vector& getSubsets() const { return impl.getSubsets(); } -CvBoost::CvBoost( const Mat& _train_data, int _tflag, - const Mat& _responses, const Mat& _var_idx, - const Mat& _sample_idx, const Mat& _var_type, - const Mat& _missing_mask, - CvBoostParams _params ) -{ - weak = 0; - data = 0; - default_model_name = "my_boost_tree"; - active_vars = active_vars_abs = orig_response = sum_response = weak_eval = - subsample_mask = weights = subtree_weights = 0; - - train( _train_data, _tflag, _responses, _var_idx, _sample_idx, - _var_type, _missing_mask, _params ); -} + DTreesImplForBoost impl; +}; -bool -CvBoost::train( const Mat& _train_data, int _tflag, - const Mat& _responses, const Mat& _var_idx, - const Mat& _sample_idx, const Mat& _var_type, - const Mat& _missing_mask, - CvBoostParams _params, bool _update ) +Ptr Boost::create(const Params& params) { - train_data_hdr = _train_data; - train_data_mat = _train_data; - responses_hdr = _responses; - responses_mat = _responses; - - CvMat vidx = _var_idx, sidx = _sample_idx, vtype = _var_type, mmask = _missing_mask; - - return train(&train_data_hdr, _tflag, &responses_hdr, vidx.data.ptr ? &vidx : 0, - sidx.data.ptr ? &sidx : 0, vtype.data.ptr ? &vtype : 0, - mmask.data.ptr ? &mmask : 0, _params, _update); + Ptr p = makePtr(); + p->setBParams(params); + return p; } -float -CvBoost::predict( const Mat& _sample, const Mat& _missing, - const Range& slice, bool raw_mode, bool return_sum ) const -{ - CvMat sample = _sample, mmask = _missing; - /*if( weak_responses ) - { - int weak_count = cvSliceLength( slice, weak ); - if( weak_count >= weak->total ) - { - weak_count = weak->total; - slice.start_index = 0; - } - - if( !(weak_responses->data && weak_responses->type() == CV_32FC1 && - (weak_responses->cols == 1 || weak_responses->rows == 1) && - weak_responses->cols + weak_responses->rows - 1 == weak_count) ) - weak_responses->create(weak_count, 1, CV_32FC1); - pwr = &(wr = *weak_responses); - }*/ - return predict(&sample, _missing.empty() ? 0 : &mmask, 0, - slice == Range::all() ? CV_WHOLE_SEQ : cvSlice(slice.start, slice.end), - raw_mode, return_sum); -} +}} /* End of file. */ diff --git a/modules/ml/src/cnn.cpp b/modules/ml/src/cnn.cpp deleted file mode 100644 index 0e0b1d08b71ec40f5bb00b3b33d2e1414433c41b..0000000000000000000000000000000000000000 --- a/modules/ml/src/cnn.cpp +++ /dev/null @@ -1,1675 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// Intel License Agreement -// -// Copyright (C) 2000, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of Intel Corporation may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#include "precomp.hpp" - -#if 0 -/****************************************************************************************\ -* Auxilary functions declarations * -\****************************************************************************************/ -/*---------------------- functions for the CNN classifier ------------------------------*/ -static float icvCNNModelPredict( - const CvStatModel* cnn_model, - const CvMat* image, - CvMat* probs CV_DEFAULT(0) ); - -static void icvCNNModelUpdate( - CvStatModel* cnn_model, const CvMat* images, int tflag, - const CvMat* responses, const CvStatModelParams* params, - const CvMat* CV_DEFAULT(0), const CvMat* sample_idx CV_DEFAULT(0), - const CvMat* CV_DEFAULT(0), const CvMat* CV_DEFAULT(0)); - -static void icvCNNModelRelease( CvStatModel** cnn_model ); - -static void icvTrainCNNetwork( CvCNNetwork* network, - const float** images, - const CvMat* responses, - const CvMat* etalons, - int grad_estim_type, - int max_iter, - int start_iter ); - -/*------------------------- functions for the CNN network ------------------------------*/ -static void icvCNNetworkAddLayer( CvCNNetwork* network, CvCNNLayer* layer ); -static void icvCNNetworkRelease( CvCNNetwork** network ); - -/* In all layer functions we denote input by X and output by Y, where - X and Y are column-vectors, so that - length(X)==**, - length(Y)==**. -*/ -/*------------------------ functions for convolutional layer ---------------------------*/ -static void icvCNNConvolutionRelease( CvCNNLayer** p_layer ); - -static void icvCNNConvolutionForward( CvCNNLayer* layer, const CvMat* X, CvMat* Y ); - -static void icvCNNConvolutionBackward( CvCNNLayer* layer, int t, - const CvMat* X, const CvMat* dE_dY, CvMat* dE_dX ); - -/*------------------------ functions for sub-sampling layer ----------------------------*/ -static void icvCNNSubSamplingRelease( CvCNNLayer** p_layer ); - -static void icvCNNSubSamplingForward( CvCNNLayer* layer, const CvMat* X, CvMat* Y ); - -static void icvCNNSubSamplingBackward( CvCNNLayer* layer, int t, - const CvMat* X, const CvMat* dE_dY, CvMat* dE_dX ); - -/*------------------------ functions for full connected layer --------------------------*/ -static void icvCNNFullConnectRelease( CvCNNLayer** p_layer ); - -static void icvCNNFullConnectForward( CvCNNLayer* layer, const CvMat* X, CvMat* Y ); - -static void icvCNNFullConnectBackward( CvCNNLayer* layer, int, - const CvMat*, const CvMat* dE_dY, CvMat* dE_dX ); - -/****************************************************************************************\ -* Functions implementations * -\****************************************************************************************/ - -#define ICV_CHECK_CNN_NETWORK(network) \ -{ \ - CvCNNLayer* first_layer, *layer, *last_layer; \ - int n_layers, i; \ - if( !network ) \ - CV_ERROR( CV_StsNullPtr, \ - "Null pointer. Network must be created by user." ); \ - n_layers = network->n_layers; \ - first_layer = last_layer = network->layers; \ - for( i = 0, layer = first_layer; i < n_layers && layer; i++ ) \ - { \ - if( !ICV_IS_CNN_LAYER(layer) ) \ - CV_ERROR( CV_StsNullPtr, "Invalid network" ); \ - last_layer = layer; \ - layer = layer->next_layer; \ - } \ - \ - if( i == 0 || i != n_layers || first_layer->prev_layer || layer ) \ - CV_ERROR( CV_StsNullPtr, "Invalid network" ); \ - \ - if( first_layer->n_input_planes != 1 ) \ - CV_ERROR( CV_StsBadArg, "First layer must contain only one input plane" ); \ - \ - if( img_size != first_layer->input_height*first_layer->input_width ) \ - CV_ERROR( CV_StsBadArg, "Invalid input sizes of the first layer" ); \ - \ - if( params->etalons->cols != last_layer->n_output_planes* \ - last_layer->output_height*last_layer->output_width ) \ - CV_ERROR( CV_StsBadArg, "Invalid output sizes of the last layer" ); \ -} - -#define ICV_CHECK_CNN_MODEL_PARAMS(params) \ -{ \ - if( !params ) \ - CV_ERROR( CV_StsNullPtr, "Null pointer" ); \ - \ - if( !ICV_IS_MAT_OF_TYPE(params->etalons, CV_32FC1) ) \ - CV_ERROR( CV_StsBadArg, " must be CV_32FC1 type" ); \ - if( params->etalons->rows != cnn_model->cls_labels->cols ) \ - CV_ERROR( CV_StsBadArg, "Invalid size" ); \ - \ - if( params->grad_estim_type != CV_CNN_GRAD_ESTIM_RANDOM && \ - params->grad_estim_type != CV_CNN_GRAD_ESTIM_BY_WORST_IMG ) \ - CV_ERROR( CV_StsBadArg, "Invalid " ); \ - \ - if( params->start_iter < 0 ) \ - CV_ERROR( CV_StsBadArg, "Parameter must be positive or zero" ); \ - \ - if( params->max_iter < 1 ) \ - params->max_iter = 1; \ -} - -/****************************************************************************************\ -* Classifier functions * -\****************************************************************************************/ -ML_IMPL CvStatModel* -cvTrainCNNClassifier( const CvMat* _train_data, int tflag, - const CvMat* _responses, - const CvStatModelParams* _params, - const CvMat*, const CvMat* _sample_idx, const CvMat*, const CvMat* ) -{ - CvCNNStatModel* cnn_model = 0; - const float** out_train_data = 0; - CvMat* responses = 0; - - CV_FUNCNAME("cvTrainCNNClassifier"); - __BEGIN__; - - int n_images; - int img_size; - CvCNNStatModelParams* params = (CvCNNStatModelParams*)_params; - - CV_CALL(cnn_model = (CvCNNStatModel*)cvCreateStatModel( - CV_STAT_MODEL_MAGIC_VAL|CV_CNN_MAGIC_VAL, sizeof(CvCNNStatModel), - icvCNNModelRelease, icvCNNModelPredict, icvCNNModelUpdate )); - - CV_CALL(cvPrepareTrainData( "cvTrainCNNClassifier", - _train_data, tflag, _responses, CV_VAR_CATEGORICAL, - 0, _sample_idx, false, &out_train_data, - &n_images, &img_size, &img_size, &responses, - &cnn_model->cls_labels, 0 )); - - ICV_CHECK_CNN_MODEL_PARAMS(params); - ICV_CHECK_CNN_NETWORK(params->network); - - cnn_model->network = params->network; - CV_CALL(cnn_model->etalons = (CvMat*)cvClone( params->etalons )); - - CV_CALL( icvTrainCNNetwork( cnn_model->network, out_train_data, responses, - cnn_model->etalons, params->grad_estim_type, params->max_iter, - params->start_iter )); - - __END__; - - if( cvGetErrStatus() < 0 && cnn_model ) - { - cnn_model->release( (CvStatModel**)&cnn_model ); - } - cvFree( &out_train_data ); - cvReleaseMat( &responses ); - - return (CvStatModel*)cnn_model; -} - -/****************************************************************************************/ -static void icvTrainCNNetwork( CvCNNetwork* network, - const float** images, - const CvMat* responses, - const CvMat* etalons, - int grad_estim_type, - int max_iter, - int start_iter ) -{ - CvMat** X = 0; - CvMat** dE_dX = 0; - const int n_layers = network->n_layers; - int k; - - CV_FUNCNAME("icvTrainCNNetwork"); - __BEGIN__; - - CvCNNLayer* first_layer = network->layers; - const int img_height = first_layer->input_height; - const int img_width = first_layer->input_width; - const int img_size = img_width*img_height; - const int n_images = responses->cols; - CvMat image = cvMat( 1, img_size, CV_32FC1 ); - CvCNNLayer* layer; - int n; - CvRNG rng = cvRNG(-1); - - CV_CALL(X = (CvMat**)cvAlloc( (n_layers+1)*sizeof(CvMat*) )); - CV_CALL(dE_dX = (CvMat**)cvAlloc( (n_layers+1)*sizeof(CvMat*) )); - memset( X, 0, (n_layers+1)*sizeof(CvMat*) ); - memset( dE_dX, 0, (n_layers+1)*sizeof(CvMat*) ); - - CV_CALL(X[0] = cvCreateMat( img_height*img_width,1,CV_32FC1 )); - CV_CALL(dE_dX[0] = cvCreateMat( 1, X[0]->rows, CV_32FC1 )); - for( k = 0, layer = first_layer; k < n_layers; k++, layer = layer->next_layer ) - { - CV_CALL(X[k+1] = cvCreateMat( layer->n_output_planes*layer->output_height* - layer->output_width, 1, CV_32FC1 )); - CV_CALL(dE_dX[k+1] = cvCreateMat( 1, X[k+1]->rows, CV_32FC1 )); - } - - for( n = 1; n <= max_iter; n++ ) - { - float loss, max_loss = 0; - int i; - int worst_img_idx = -1; - int* right_etal_idx = responses->data.i; - CvMat etalon; - - // Find the worst image (which produces the greatest loss) or use the random image - if( grad_estim_type == CV_CNN_GRAD_ESTIM_BY_WORST_IMG ) - { - for( i = 0; i < n_images; i++, right_etal_idx++ ) - { - image.data.fl = (float*)images[i]; - cvTranspose( &image, X[0] ); - - for( k = 0, layer = first_layer; k < n_layers; k++, layer = layer->next_layer ) - CV_CALL(layer->forward( layer, X[k], X[k+1] )); - - cvTranspose( X[n_layers], dE_dX[n_layers] ); - cvGetRow( etalons, &etalon, *right_etal_idx ); - loss = (float)cvNorm( dE_dX[n_layers], &etalon ); - if( loss > max_loss ) - { - max_loss = loss; - worst_img_idx = i; - } - } - } - else - worst_img_idx = cvRandInt(&rng) % n_images; - - // Train network on the worst image - // 1) Compute the network output on the - image.data.fl = (float*)images[worst_img_idx]; - CV_CALL(cvTranspose( &image, X[0] )); - - for( k = 0, layer = first_layer; k < n_layers - 1; k++, layer = layer->next_layer ) - CV_CALL(layer->forward( layer, X[k], X[k+1] )); - CV_CALL(layer->forward( layer, X[k], X[k+1] )); - - // 2) Compute the gradient - cvTranspose( X[n_layers], dE_dX[n_layers] ); - cvGetRow( etalons, &etalon, responses->data.i[worst_img_idx] ); - cvSub( dE_dX[n_layers], &etalon, dE_dX[n_layers] ); - - // 3) Update weights by the gradient descent - for( k = n_layers; k > 0; k--, layer = layer->prev_layer ) - CV_CALL(layer->backward( layer, n + start_iter, X[k-1], dE_dX[k], dE_dX[k-1] )); - } - - __END__; - - for( k = 0; k <= n_layers; k++ ) - { - cvReleaseMat( &X[k] ); - cvReleaseMat( &dE_dX[k] ); - } - cvFree( &X ); - cvFree( &dE_dX ); -} - -/****************************************************************************************/ -static float icvCNNModelPredict( const CvStatModel* model, - const CvMat* _image, - CvMat* probs ) -{ - CvMat** X = 0; - float* img_data = 0; - int n_layers = 0; - int best_etal_idx = -1; - int k; - - CV_FUNCNAME("icvCNNModelPredict"); - __BEGIN__; - - CvCNNStatModel* cnn_model = (CvCNNStatModel*)model; - CvCNNLayer* first_layer, *layer = 0; - int img_height, img_width, img_size; - int nclasses, i; - float loss, min_loss = FLT_MAX; - float* probs_data; - CvMat etalon, image; - - if( !CV_IS_CNN(model) ) - CV_ERROR( CV_StsBadArg, "Invalid model" ); - - nclasses = cnn_model->cls_labels->cols; - n_layers = cnn_model->network->n_layers; - first_layer = cnn_model->network->layers; - img_height = first_layer->input_height; - img_width = first_layer->input_width; - img_size = img_height*img_width; - - cvPreparePredictData( _image, img_size, 0, nclasses, probs, &img_data ); - - CV_CALL(X = (CvMat**)cvAlloc( (n_layers+1)*sizeof(CvMat*) )); - memset( X, 0, (n_layers+1)*sizeof(CvMat*) ); - - CV_CALL(X[0] = cvCreateMat( img_size,1,CV_32FC1 )); - for( k = 0, layer = first_layer; k < n_layers; k++, layer = layer->next_layer ) - { - CV_CALL(X[k+1] = cvCreateMat( layer->n_output_planes*layer->output_height* - layer->output_width, 1, CV_32FC1 )); - } - - image = cvMat( 1, img_size, CV_32FC1, img_data ); - cvTranspose( &image, X[0] ); - for( k = 0, layer = first_layer; k < n_layers; k++, layer = layer->next_layer ) - CV_CALL(layer->forward( layer, X[k], X[k+1] )); - - probs_data = probs ? probs->data.fl : 0; - etalon = cvMat( cnn_model->etalons->cols, 1, CV_32FC1, cnn_model->etalons->data.fl ); - for( i = 0; i < nclasses; i++, etalon.data.fl += cnn_model->etalons->cols ) - { - loss = (float)cvNorm( X[n_layers], &etalon ); - if( loss < min_loss ) - { - min_loss = loss; - best_etal_idx = i; - } - if( probs ) - *probs_data++ = -loss; - } - - if( probs ) - { - cvExp( probs, probs ); - CvScalar sum = cvSum( probs ); - cvConvertScale( probs, probs, 1./sum.val[0] ); - } - - __END__; - - for( k = 0; k <= n_layers; k++ ) - cvReleaseMat( &X[k] ); - cvFree( &X ); - if( img_data != _image->data.fl ) - cvFree( &img_data ); - - return ((float) ((CvCNNStatModel*)model)->cls_labels->data.i[best_etal_idx]); -} - -/****************************************************************************************/ -static void icvCNNModelUpdate( - CvStatModel* _cnn_model, const CvMat* _train_data, int tflag, - const CvMat* _responses, const CvStatModelParams* _params, - const CvMat*, const CvMat* _sample_idx, - const CvMat*, const CvMat* ) -{ - const float** out_train_data = 0; - CvMat* responses = 0; - CvMat* cls_labels = 0; - - CV_FUNCNAME("icvCNNModelUpdate"); - __BEGIN__; - - int n_images, img_size, i; - CvCNNStatModelParams* params = (CvCNNStatModelParams*)_params; - CvCNNStatModel* cnn_model = (CvCNNStatModel*)_cnn_model; - - if( !CV_IS_CNN(cnn_model) ) - CV_ERROR( CV_StsBadArg, "Invalid model" ); - - CV_CALL(cvPrepareTrainData( "cvTrainCNNClassifier", - _train_data, tflag, _responses, CV_VAR_CATEGORICAL, - 0, _sample_idx, false, &out_train_data, - &n_images, &img_size, &img_size, &responses, - &cls_labels, 0, 0 )); - - ICV_CHECK_CNN_MODEL_PARAMS(params); - - // Number of classes must be the same as when classifiers was created - if( !CV_ARE_SIZES_EQ(cls_labels, cnn_model->cls_labels) ) - CV_ERROR( CV_StsBadArg, "Number of classes must be left unchanged" ); - for( i = 0; i < cls_labels->cols; i++ ) - { - if( cls_labels->data.i[i] != cnn_model->cls_labels->data.i[i] ) - CV_ERROR( CV_StsBadArg, "Number of classes must be left unchanged" ); - } - - CV_CALL( icvTrainCNNetwork( cnn_model->network, out_train_data, responses, - cnn_model->etalons, params->grad_estim_type, params->max_iter, - params->start_iter )); - - __END__; - - cvFree( &out_train_data ); - cvReleaseMat( &responses ); -} - -/****************************************************************************************/ -static void icvCNNModelRelease( CvStatModel** cnn_model ) -{ - CV_FUNCNAME("icvCNNModelRelease"); - __BEGIN__; - - CvCNNStatModel* cnn; - if( !cnn_model ) - CV_ERROR( CV_StsNullPtr, "Null double pointer" ); - - cnn = *(CvCNNStatModel**)cnn_model; - - cvReleaseMat( &cnn->cls_labels ); - cvReleaseMat( &cnn->etalons ); - cnn->network->release( &cnn->network ); - - cvFree( &cnn ); - - __END__; - -} - -/****************************************************************************************\ -* Network functions * -\****************************************************************************************/ -ML_IMPL CvCNNetwork* cvCreateCNNetwork( CvCNNLayer* first_layer ) -{ - CvCNNetwork* network = 0; - - CV_FUNCNAME( "cvCreateCNNetwork" ); - __BEGIN__; - - if( !ICV_IS_CNN_LAYER(first_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - CV_CALL(network = (CvCNNetwork*)cvAlloc( sizeof(CvCNNetwork) )); - memset( network, 0, sizeof(CvCNNetwork) ); - - network->layers = first_layer; - network->n_layers = 1; - network->release = icvCNNetworkRelease; - network->add_layer = icvCNNetworkAddLayer; - - __END__; - - if( cvGetErrStatus() < 0 && network ) - cvFree( &network ); - - return network; - -} - -/****************************************************************************************/ -static void icvCNNetworkAddLayer( CvCNNetwork* network, CvCNNLayer* layer ) -{ - CV_FUNCNAME( "icvCNNetworkAddLayer" ); - __BEGIN__; - - CvCNNLayer* prev_layer; - - if( network == NULL ) - CV_ERROR( CV_StsNullPtr, "Null pointer" ); - - prev_layer = network->layers; - while( prev_layer->next_layer ) - prev_layer = prev_layer->next_layer; - - if( ICV_IS_CNN_FULLCONNECT_LAYER(layer) ) - { - if( layer->n_input_planes != prev_layer->output_width*prev_layer->output_height* - prev_layer->n_output_planes ) - CV_ERROR( CV_StsBadArg, "Unmatched size of the new layer" ); - if( layer->input_height != 1 || layer->output_height != 1 || - layer->input_width != 1 || layer->output_width != 1 ) - CV_ERROR( CV_StsBadArg, "Invalid size of the new layer" ); - } - else if( ICV_IS_CNN_CONVOLUTION_LAYER(layer) || ICV_IS_CNN_SUBSAMPLING_LAYER(layer) ) - { - if( prev_layer->n_output_planes != layer->n_input_planes || - prev_layer->output_height != layer->input_height || - prev_layer->output_width != layer->input_width ) - CV_ERROR( CV_StsBadArg, "Unmatched size of the new layer" ); - } - else - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - layer->prev_layer = prev_layer; - prev_layer->next_layer = layer; - network->n_layers++; - - __END__; -} - -/****************************************************************************************/ -static void icvCNNetworkRelease( CvCNNetwork** network_pptr ) -{ - CV_FUNCNAME( "icvReleaseCNNetwork" ); - __BEGIN__; - - CvCNNetwork* network = 0; - CvCNNLayer* layer = 0, *next_layer = 0; - int k; - - if( network_pptr == NULL ) - CV_ERROR( CV_StsBadArg, "Null double pointer" ); - if( *network_pptr == NULL ) - return; - - network = *network_pptr; - layer = network->layers; - if( layer == NULL ) - CV_ERROR( CV_StsBadArg, "CNN is empty (does not contain any layer)" ); - - // k is the number of the layer to be deleted - for( k = 0; k < network->n_layers && layer; k++ ) - { - next_layer = layer->next_layer; - layer->release( &layer ); - layer = next_layer; - } - - if( k != network->n_layers || layer) - CV_ERROR( CV_StsBadArg, "Invalid network" ); - - cvFree( &network ); - - __END__; -} - -/****************************************************************************************\ -* Layer functions * -\****************************************************************************************/ -static CvCNNLayer* icvCreateCNNLayer( int layer_type, int header_size, - int n_input_planes, int input_height, int input_width, - int n_output_planes, int output_height, int output_width, - float init_learn_rate, int learn_rate_decrease_type, - CvCNNLayerRelease release, CvCNNLayerForward forward, CvCNNLayerBackward backward ) -{ - CvCNNLayer* layer = 0; - - CV_FUNCNAME("icvCreateCNNLayer"); - __BEGIN__; - - CV_ASSERT( release && forward && backward ) - CV_ASSERT( header_size >= sizeof(CvCNNLayer) ) - - if( n_input_planes < 1 || n_output_planes < 1 || - input_height < 1 || input_width < 1 || - output_height < 1 || output_width < 1 || - input_height < output_height || - input_width < output_width ) - CV_ERROR( CV_StsBadArg, "Incorrect input or output parameters" ); - if( init_learn_rate < FLT_EPSILON ) - CV_ERROR( CV_StsBadArg, "Initial learning rate must be positive" ); - if( learn_rate_decrease_type != CV_CNN_LEARN_RATE_DECREASE_HYPERBOLICALLY && - learn_rate_decrease_type != CV_CNN_LEARN_RATE_DECREASE_SQRT_INV && - learn_rate_decrease_type != CV_CNN_LEARN_RATE_DECREASE_LOG_INV ) - CV_ERROR( CV_StsBadArg, "Invalid type of learning rate dynamics" ); - - CV_CALL(layer = (CvCNNLayer*)cvAlloc( header_size )); - memset( layer, 0, header_size ); - - layer->flags = ICV_CNN_LAYER|layer_type; - CV_ASSERT( ICV_IS_CNN_LAYER(layer) ) - - layer->n_input_planes = n_input_planes; - layer->input_height = input_height; - layer->input_width = input_width; - - layer->n_output_planes = n_output_planes; - layer->output_height = output_height; - layer->output_width = output_width; - - layer->init_learn_rate = init_learn_rate; - layer->learn_rate_decrease_type = learn_rate_decrease_type; - - layer->release = release; - layer->forward = forward; - layer->backward = backward; - - __END__; - - if( cvGetErrStatus() < 0 && layer) - cvFree( &layer ); - - return layer; -} - -/****************************************************************************************/ -ML_IMPL CvCNNLayer* cvCreateCNNConvolutionLayer( - int n_input_planes, int input_height, int input_width, - int n_output_planes, int K, - float init_learn_rate, int learn_rate_decrease_type, - CvMat* connect_mask, CvMat* weights ) - -{ - CvCNNConvolutionLayer* layer = 0; - - CV_FUNCNAME("cvCreateCNNConvolutionLayer"); - __BEGIN__; - - const int output_height = input_height - K + 1; - const int output_width = input_width - K + 1; - - if( K < 1 || init_learn_rate <= 0 ) - CV_ERROR( CV_StsBadArg, "Incorrect parameters" ); - - CV_CALL(layer = (CvCNNConvolutionLayer*)icvCreateCNNLayer( ICV_CNN_CONVOLUTION_LAYER, - sizeof(CvCNNConvolutionLayer), n_input_planes, input_height, input_width, - n_output_planes, output_height, output_width, - init_learn_rate, learn_rate_decrease_type, - icvCNNConvolutionRelease, icvCNNConvolutionForward, icvCNNConvolutionBackward )); - - layer->K = K; - CV_CALL(layer->weights = cvCreateMat( n_output_planes, K*K+1, CV_32FC1 )); - CV_CALL(layer->connect_mask = cvCreateMat( n_output_planes, n_input_planes, CV_8UC1)); - - if( weights ) - { - if( !ICV_IS_MAT_OF_TYPE( weights, CV_32FC1 ) ) - CV_ERROR( CV_StsBadSize, "Type of initial weights matrix must be CV_32FC1" ); - if( !CV_ARE_SIZES_EQ( weights, layer->weights ) ) - CV_ERROR( CV_StsBadSize, "Invalid size of initial weights matrix" ); - CV_CALL(cvCopy( weights, layer->weights )); - } - else - { - CvRNG rng = cvRNG( 0xFFFFFFFF ); - cvRandArr( &rng, layer->weights, CV_RAND_UNI, cvRealScalar(-1), cvRealScalar(1) ); - } - - if( connect_mask ) - { - if( !ICV_IS_MAT_OF_TYPE( connect_mask, CV_8UC1 ) ) - CV_ERROR( CV_StsBadSize, "Type of connection matrix must be CV_32FC1" ); - if( !CV_ARE_SIZES_EQ( connect_mask, layer->connect_mask ) ) - CV_ERROR( CV_StsBadSize, "Invalid size of connection matrix" ); - CV_CALL(cvCopy( connect_mask, layer->connect_mask )); - } - else - CV_CALL(cvSet( layer->connect_mask, cvRealScalar(1) )); - - __END__; - - if( cvGetErrStatus() < 0 && layer ) - { - cvReleaseMat( &layer->weights ); - cvReleaseMat( &layer->connect_mask ); - cvFree( &layer ); - } - - return (CvCNNLayer*)layer; -} - -/****************************************************************************************/ -ML_IMPL CvCNNLayer* cvCreateCNNSubSamplingLayer( - int n_input_planes, int input_height, int input_width, - int sub_samp_scale, float a, float s, - float init_learn_rate, int learn_rate_decrease_type, CvMat* weights ) - -{ - CvCNNSubSamplingLayer* layer = 0; - - CV_FUNCNAME("cvCreateCNNSubSamplingLayer"); - __BEGIN__; - - const int output_height = input_height/sub_samp_scale; - const int output_width = input_width/sub_samp_scale; - const int n_output_planes = n_input_planes; - - if( sub_samp_scale < 1 || a <= 0 || s <= 0) - CV_ERROR( CV_StsBadArg, "Incorrect parameters" ); - - CV_CALL(layer = (CvCNNSubSamplingLayer*)icvCreateCNNLayer( ICV_CNN_SUBSAMPLING_LAYER, - sizeof(CvCNNSubSamplingLayer), n_input_planes, input_height, input_width, - n_output_planes, output_height, output_width, - init_learn_rate, learn_rate_decrease_type, - icvCNNSubSamplingRelease, icvCNNSubSamplingForward, icvCNNSubSamplingBackward )); - - layer->sub_samp_scale = sub_samp_scale; - layer->a = a; - layer->s = s; - - CV_CALL(layer->sumX = - cvCreateMat( n_output_planes*output_width*output_height, 1, CV_32FC1 )); - CV_CALL(layer->exp2ssumWX = - cvCreateMat( n_output_planes*output_width*output_height, 1, CV_32FC1 )); - - cvZero( layer->sumX ); - cvZero( layer->exp2ssumWX ); - - CV_CALL(layer->weights = cvCreateMat( n_output_planes, 2, CV_32FC1 )); - if( weights ) - { - if( !ICV_IS_MAT_OF_TYPE( weights, CV_32FC1 ) ) - CV_ERROR( CV_StsBadSize, "Type of initial weights matrix must be CV_32FC1" ); - if( !CV_ARE_SIZES_EQ( weights, layer->weights ) ) - CV_ERROR( CV_StsBadSize, "Invalid size of initial weights matrix" ); - CV_CALL(cvCopy( weights, layer->weights )); - } - else - { - CvRNG rng = cvRNG( 0xFFFFFFFF ); - cvRandArr( &rng, layer->weights, CV_RAND_UNI, cvRealScalar(-1), cvRealScalar(1) ); - } - - __END__; - - if( cvGetErrStatus() < 0 && layer ) - { - cvReleaseMat( &layer->exp2ssumWX ); - cvFree( &layer ); - } - - return (CvCNNLayer*)layer; -} - -/****************************************************************************************/ -ML_IMPL CvCNNLayer* cvCreateCNNFullConnectLayer( - int n_inputs, int n_outputs, float a, float s, - float init_learn_rate, int learn_rate_decrease_type, CvMat* weights ) -{ - CvCNNFullConnectLayer* layer = 0; - - CV_FUNCNAME("cvCreateCNNFullConnectLayer"); - __BEGIN__; - - if( a <= 0 || s <= 0 || init_learn_rate <= 0) - CV_ERROR( CV_StsBadArg, "Incorrect parameters" ); - - CV_CALL(layer = (CvCNNFullConnectLayer*)icvCreateCNNLayer( ICV_CNN_FULLCONNECT_LAYER, - sizeof(CvCNNFullConnectLayer), n_inputs, 1, 1, n_outputs, 1, 1, - init_learn_rate, learn_rate_decrease_type, - icvCNNFullConnectRelease, icvCNNFullConnectForward, icvCNNFullConnectBackward )); - - layer->a = a; - layer->s = s; - - CV_CALL(layer->exp2ssumWX = cvCreateMat( n_outputs, 1, CV_32FC1 )); - cvZero( layer->exp2ssumWX ); - - CV_CALL(layer->weights = cvCreateMat( n_outputs, n_inputs+1, CV_32FC1 )); - if( weights ) - { - if( !ICV_IS_MAT_OF_TYPE( weights, CV_32FC1 ) ) - CV_ERROR( CV_StsBadSize, "Type of initial weights matrix must be CV_32FC1" ); - if( !CV_ARE_SIZES_EQ( weights, layer->weights ) ) - CV_ERROR( CV_StsBadSize, "Invalid size of initial weights matrix" ); - CV_CALL(cvCopy( weights, layer->weights )); - } - else - { - CvRNG rng = cvRNG( 0xFFFFFFFF ); - cvRandArr( &rng, layer->weights, CV_RAND_UNI, cvRealScalar(-1), cvRealScalar(1) ); - } - - __END__; - - if( cvGetErrStatus() < 0 && layer ) - { - cvReleaseMat( &layer->exp2ssumWX ); - cvReleaseMat( &layer->weights ); - cvFree( &layer ); - } - - return (CvCNNLayer*)layer; -} - - -/****************************************************************************************\ -* Layer FORWARD functions * -\****************************************************************************************/ -static void icvCNNConvolutionForward( CvCNNLayer* _layer, - const CvMat* X, - CvMat* Y ) -{ - CV_FUNCNAME("icvCNNConvolutionForward"); - - if( !ICV_IS_CNN_CONVOLUTION_LAYER(_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - {__BEGIN__; - - const CvCNNConvolutionLayer* layer = (CvCNNConvolutionLayer*) _layer; - - const int K = layer->K; - const int n_weights_for_Yplane = K*K + 1; - - const int nXplanes = layer->n_input_planes; - const int Xheight = layer->input_height; - const int Xwidth = layer->input_width ; - const int Xsize = Xwidth*Xheight; - - const int nYplanes = layer->n_output_planes; - const int Yheight = layer->output_height; - const int Ywidth = layer->output_width; - const int Ysize = Ywidth*Yheight; - - int xx, yy, ni, no, kx, ky; - float *Yplane = 0, *Xplane = 0, *w = 0; - uchar* connect_mask_data = 0; - - CV_ASSERT( X->rows == nXplanes*Xsize && X->cols == 1 ); - CV_ASSERT( Y->rows == nYplanes*Ysize && Y->cols == 1 ); - - cvSetZero( Y ); - - Yplane = Y->data.fl; - connect_mask_data = layer->connect_mask->data.ptr; - w = layer->weights->data.fl; - for( no = 0; no < nYplanes; no++, Yplane += Ysize, w += n_weights_for_Yplane ) - { - Xplane = X->data.fl; - for( ni = 0; ni < nXplanes; ni++, Xplane += Xsize, connect_mask_data++ ) - { - if( *connect_mask_data ) - { - float* Yelem = Yplane; - - // Xheight-K+1 == Yheight && Xwidth-K+1 == Ywidth - for( yy = 0; yy < Xheight-K+1; yy++ ) - { - for( xx = 0; xx < Xwidth-K+1; xx++, Yelem++ ) - { - float* templ = Xplane+yy*Xwidth+xx; - float WX = 0; - for( ky = 0; ky < K; ky++, templ += Xwidth-K ) - { - for( kx = 0; kx < K; kx++, templ++ ) - { - WX += *templ*w[ky*K+kx]; - } - } - *Yelem += WX + w[K*K]; - } - } - } - } - } - }__END__; -} - -/****************************************************************************************/ -static void icvCNNSubSamplingForward( CvCNNLayer* _layer, - const CvMat* X, - CvMat* Y ) -{ - CV_FUNCNAME("icvCNNSubSamplingForward"); - - if( !ICV_IS_CNN_SUBSAMPLING_LAYER(_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - {__BEGIN__; - - const CvCNNSubSamplingLayer* layer = (CvCNNSubSamplingLayer*) _layer; - - const int sub_sampl_scale = layer->sub_samp_scale; - const int nplanes = layer->n_input_planes; - - const int Xheight = layer->input_height; - const int Xwidth = layer->input_width ; - const int Xsize = Xwidth*Xheight; - - const int Yheight = layer->output_height; - const int Ywidth = layer->output_width; - const int Ysize = Ywidth*Yheight; - - int xx, yy, ni, kx, ky; - float* sumX_data = 0, *w = 0; - CvMat sumX_sub_col, exp2ssumWX_sub_col; - - CV_ASSERT(X->rows == nplanes*Xsize && X->cols == 1); - CV_ASSERT(layer->exp2ssumWX->cols == 1 && layer->exp2ssumWX->rows == nplanes*Ysize); - - // update inner variable layer->exp2ssumWX, which will be used in back-progation - cvZero( layer->sumX ); - cvZero( layer->exp2ssumWX ); - - for( ky = 0; ky < sub_sampl_scale; ky++ ) - for( kx = 0; kx < sub_sampl_scale; kx++ ) - { - float* Xplane = X->data.fl; - sumX_data = layer->sumX->data.fl; - for( ni = 0; ni < nplanes; ni++, Xplane += Xsize ) - { - for( yy = 0; yy < Yheight; yy++ ) - for( xx = 0; xx < Ywidth; xx++, sumX_data++ ) - *sumX_data += Xplane[((yy+ky)*Xwidth+(xx+kx))]; - } - } - - w = layer->weights->data.fl; - cvGetRows( layer->sumX, &sumX_sub_col, 0, Ysize ); - cvGetRows( layer->exp2ssumWX, &exp2ssumWX_sub_col, 0, Ysize ); - for( ni = 0; ni < nplanes; ni++, w += 2 ) - { - CV_CALL(cvConvertScale( &sumX_sub_col, &exp2ssumWX_sub_col, w[0], w[1] )); - sumX_sub_col.data.fl += Ysize; - exp2ssumWX_sub_col.data.fl += Ysize; - } - - CV_CALL(cvScale( layer->exp2ssumWX, layer->exp2ssumWX, 2.0*layer->s )); - CV_CALL(cvExp( layer->exp2ssumWX, layer->exp2ssumWX )); - CV_CALL(cvMinS( layer->exp2ssumWX, FLT_MAX, layer->exp2ssumWX )); -//#ifdef _DEBUG - { - float* exp2ssumWX_data = layer->exp2ssumWX->data.fl; - for( ni = 0; ni < layer->exp2ssumWX->rows; ni++, exp2ssumWX_data++ ) - { - if( *exp2ssumWX_data == FLT_MAX ) - cvSetErrStatus( 1 ); - } - } -//#endif - // compute the output variable Y == ( a - 2a/(layer->exp2ssumWX + 1)) - CV_CALL(cvAddS( layer->exp2ssumWX, cvRealScalar(1), Y )); - CV_CALL(cvDiv( 0, Y, Y, -2.0*layer->a )); - CV_CALL(cvAddS( Y, cvRealScalar(layer->a), Y )); - - }__END__; -} - -/****************************************************************************************/ -static void icvCNNFullConnectForward( CvCNNLayer* _layer, const CvMat* X, CvMat* Y ) -{ - CV_FUNCNAME("icvCNNFullConnectForward"); - - if( !ICV_IS_CNN_FULLCONNECT_LAYER(_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - {__BEGIN__; - - const CvCNNFullConnectLayer* layer = (CvCNNFullConnectLayer*)_layer; - CvMat* weights = layer->weights; - CvMat sub_weights, bias; - - CV_ASSERT(X->cols == 1 && X->rows == layer->n_input_planes); - CV_ASSERT(Y->cols == 1 && Y->rows == layer->n_output_planes); - - CV_CALL(cvGetSubRect( weights, &sub_weights, - cvRect(0, 0, weights->cols-1, weights->rows ))); - CV_CALL(cvGetCol( weights, &bias, weights->cols-1)); - - // update inner variable layer->exp2ssumWX, which will be used in Back-Propagation - CV_CALL(cvGEMM( &sub_weights, X, 2*layer->s, &bias, 2*layer->s, layer->exp2ssumWX )); - CV_CALL(cvExp( layer->exp2ssumWX, layer->exp2ssumWX )); - CV_CALL(cvMinS( layer->exp2ssumWX, FLT_MAX, layer->exp2ssumWX )); -//#ifdef _DEBUG - { - float* exp2ssumWX_data = layer->exp2ssumWX->data.fl; - int i; - for( i = 0; i < layer->exp2ssumWX->rows; i++, exp2ssumWX_data++ ) - { - if( *exp2ssumWX_data == FLT_MAX ) - cvSetErrStatus( 1 ); - } - } -//#endif - // compute the output variable Y == ( a - 2a/(layer->exp2ssumWX + 1)) - CV_CALL(cvAddS( layer->exp2ssumWX, cvRealScalar(1), Y )); - CV_CALL(cvDiv( 0, Y, Y, -2.0*layer->a )); - CV_CALL(cvAddS( Y, cvRealScalar(layer->a), Y )); - - }__END__; -} - -/****************************************************************************************\ -* Layer BACKWARD functions * -\****************************************************************************************/ - -/* , should be row-vectors. - Function computes partial derivatives - of the loss function with respect to the planes components - of the previous layer (X). - It is a basic function for back propagation method. - Input parameter is the partial derivative of the - loss function with respect to the planes components - of the current layer. */ -static void icvCNNConvolutionBackward( - CvCNNLayer* _layer, int t, const CvMat* X, const CvMat* dE_dY, CvMat* dE_dX ) -{ - CvMat* dY_dX = 0; - CvMat* dY_dW = 0; - CvMat* dE_dW = 0; - - CV_FUNCNAME("icvCNNConvolutionBackward"); - - if( !ICV_IS_CNN_CONVOLUTION_LAYER(_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - {__BEGIN__; - - const CvCNNConvolutionLayer* layer = (CvCNNConvolutionLayer*) _layer; - - const int K = layer->K; - - const int n_X_planes = layer->n_input_planes; - const int X_plane_height = layer->input_height; - const int X_plane_width = layer->input_width; - const int X_plane_size = X_plane_height*X_plane_width; - - const int n_Y_planes = layer->n_output_planes; - const int Y_plane_height = layer->output_height; - const int Y_plane_width = layer->output_width; - const int Y_plane_size = Y_plane_height*Y_plane_width; - - int no, ni, yy, xx, ky, kx; - int X_idx = 0, Y_idx = 0; - - float *X_plane = 0, *w = 0; - - CvMat* weights = layer->weights; - - CV_ASSERT( t >= 1 ); - CV_ASSERT( n_Y_planes == weights->rows ); - - dY_dX = cvCreateMat( n_Y_planes*Y_plane_size, X->rows, CV_32FC1 ); - dY_dW = cvCreateMat( dY_dX->rows, weights->cols*weights->rows, CV_32FC1 ); - dE_dW = cvCreateMat( 1, dY_dW->cols, CV_32FC1 ); - - cvZero( dY_dX ); - cvZero( dY_dW ); - - // compute gradient of the loss function with respect to X and W - for( no = 0; no < n_Y_planes; no++, Y_idx += Y_plane_size ) - { - w = weights->data.fl + no*(K*K+1); - X_idx = 0; - X_plane = X->data.fl; - for( ni = 0; ni < n_X_planes; ni++, X_plane += X_plane_size ) - { - if( layer->connect_mask->data.ptr[ni*n_Y_planes+no] ) - { - for( yy = 0; yy < X_plane_height - K + 1; yy++ ) - { - for( xx = 0; xx < X_plane_width - K + 1; xx++ ) - { - for( ky = 0; ky < K; ky++ ) - { - for( kx = 0; kx < K; kx++ ) - { - CV_MAT_ELEM(*dY_dX, float, Y_idx+yy*Y_plane_width+xx, - X_idx+(yy+ky)*X_plane_width+(xx+kx)) = w[ky*K+kx]; - - // dY_dWi, i=1,...,K*K - CV_MAT_ELEM(*dY_dW, float, Y_idx+yy*Y_plane_width+xx, - no*(K*K+1)+ky*K+kx) += - X_plane[(yy+ky)*X_plane_width+(xx+kx)]; - } - } - // dY_dW(K*K+1)==1 because W(K*K+1) is bias - CV_MAT_ELEM(*dY_dW, float, Y_idx+yy*Y_plane_width+xx, - no*(K*K+1)+K*K) += 1; - } - } - } - X_idx += X_plane_size; - } - } - - CV_CALL(cvMatMul( dE_dY, dY_dW, dE_dW )); - CV_CALL(cvMatMul( dE_dY, dY_dX, dE_dX )); - - // update weights - { - CvMat dE_dW_mat; - float eta; - if( layer->learn_rate_decrease_type == CV_CNN_LEARN_RATE_DECREASE_LOG_INV ) - eta = -layer->init_learn_rate/logf(1+(float)t); - else if( layer->learn_rate_decrease_type == CV_CNN_LEARN_RATE_DECREASE_SQRT_INV ) - eta = -layer->init_learn_rate/sqrtf((float)t); - else - eta = -layer->init_learn_rate/(float)t; - cvReshape( dE_dW, &dE_dW_mat, 0, weights->rows ); - cvScaleAdd( &dE_dW_mat, cvRealScalar(eta), weights, weights ); - } - - }__END__; - - cvReleaseMat( &dY_dX ); - cvReleaseMat( &dY_dW ); - cvReleaseMat( &dE_dW ); -} - -/****************************************************************************************/ -static void icvCNNSubSamplingBackward( - CvCNNLayer* _layer, int t, const CvMat*, const CvMat* dE_dY, CvMat* dE_dX ) -{ - // derivative of activation function - CvMat* dY_dX_elems = 0; // elements of matrix dY_dX - CvMat* dY_dW_elems = 0; // elements of matrix dY_dW - CvMat* dE_dW = 0; - - CV_FUNCNAME("icvCNNSubSamplingBackward"); - - if( !ICV_IS_CNN_SUBSAMPLING_LAYER(_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - {__BEGIN__; - - const CvCNNSubSamplingLayer* layer = (CvCNNSubSamplingLayer*) _layer; - - const int Xwidth = layer->input_width; - const int Ywidth = layer->output_width; - const int Yheight = layer->output_height; - const int Ysize = Ywidth * Yheight; - const int scale = layer->sub_samp_scale; - const int k_max = layer->n_output_planes * Yheight; - - int k, i, j, m; - float* dY_dX_current_elem = 0, *dE_dX_start = 0, *dE_dW_data = 0, *w = 0; - CvMat dy_dw0, dy_dw1; - CvMat activ_func_der, sumX_row; - CvMat dE_dY_sub_row, dY_dX_sub_col, dy_dw0_sub_row, dy_dw1_sub_row; - - CV_CALL(dY_dX_elems = cvCreateMat( layer->sumX->rows, 1, CV_32FC1 )); - CV_CALL(dY_dW_elems = cvCreateMat( 2, layer->sumX->rows, CV_32FC1 )); - CV_CALL(dE_dW = cvCreateMat( 1, 2*layer->n_output_planes, CV_32FC1 )); - - // compute derivative of activ.func. - // == = 4as*(layer->exp2ssumWX)/(layer->exp2ssumWX + 1)^2 - CV_CALL(cvAddS( layer->exp2ssumWX, cvRealScalar(1), dY_dX_elems )); - CV_CALL(cvPow( dY_dX_elems, dY_dX_elems, -2.0 )); - CV_CALL(cvMul( dY_dX_elems, layer->exp2ssumWX, dY_dX_elems, 4.0*layer->a*layer->s )); - - // compute - // a) compute - cvReshape( dY_dX_elems, &activ_func_der, 0, 1 ); - cvGetRow( dY_dW_elems, &dy_dw0, 0 ); - cvGetRow( dY_dW_elems, &dy_dw1, 1 ); - CV_CALL(cvCopy( &activ_func_der, &dy_dw0 )); - CV_CALL(cvCopy( &activ_func_der, &dy_dw1 )); - - cvReshape( layer->sumX, &sumX_row, 0, 1 ); - cvMul( &dy_dw0, &sumX_row, &dy_dw0 ); - - // b) compute = * - cvGetCols( dE_dY, &dE_dY_sub_row, 0, Ysize ); - cvGetCols( &dy_dw0, &dy_dw0_sub_row, 0, Ysize ); - cvGetCols( &dy_dw1, &dy_dw1_sub_row, 0, Ysize ); - dE_dW_data = dE_dW->data.fl; - for( i = 0; i < layer->n_output_planes; i++ ) - { - *dE_dW_data++ = (float)cvDotProduct( &dE_dY_sub_row, &dy_dw0_sub_row ); - *dE_dW_data++ = (float)cvDotProduct( &dE_dY_sub_row, &dy_dw1_sub_row ); - - dE_dY_sub_row.data.fl += Ysize; - dy_dw0_sub_row.data.fl += Ysize; - dy_dw1_sub_row.data.fl += Ysize; - } - - // compute = layer->weights* - w = layer->weights->data.fl; - cvGetRows( dY_dX_elems, &dY_dX_sub_col, 0, Ysize ); - for( i = 0; i < layer->n_input_planes; i++, w++, dY_dX_sub_col.data.fl += Ysize ) - CV_CALL(cvConvertScale( &dY_dX_sub_col, &dY_dX_sub_col, (float)*w )); - - // compute - CV_CALL(cvReshape( dY_dX_elems, dY_dX_elems, 0, 1 )); - CV_CALL(cvMul( dY_dX_elems, dE_dY, dY_dX_elems )); - - dY_dX_current_elem = dY_dX_elems->data.fl; - dE_dX_start = dE_dX->data.fl; - for( k = 0; k < k_max; k++ ) - { - for( i = 0; i < Ywidth; i++, dY_dX_current_elem++ ) - { - float* dE_dX_current_elem = dE_dX_start; - for( j = 0; j < scale; j++, dE_dX_current_elem += Xwidth - scale ) - { - for( m = 0; m < scale; m++, dE_dX_current_elem++ ) - *dE_dX_current_elem = *dY_dX_current_elem; - } - dE_dX_start += scale; - } - dE_dX_start += Xwidth * (scale - 1); - } - - // update weights - { - CvMat dE_dW_mat, *weights = layer->weights; - float eta; - if( layer->learn_rate_decrease_type == CV_CNN_LEARN_RATE_DECREASE_LOG_INV ) - eta = -layer->init_learn_rate/logf(1+(float)t); - else if( layer->learn_rate_decrease_type == CV_CNN_LEARN_RATE_DECREASE_SQRT_INV ) - eta = -layer->init_learn_rate/sqrtf((float)t); - else - eta = -layer->init_learn_rate/(float)t; - cvReshape( dE_dW, &dE_dW_mat, 0, weights->rows ); - cvScaleAdd( &dE_dW_mat, cvRealScalar(eta), weights, weights ); - } - - }__END__; - - cvReleaseMat( &dY_dX_elems ); - cvReleaseMat( &dY_dW_elems ); - cvReleaseMat( &dE_dW ); -} - -/****************************************************************************************/ -/* , should be row-vectors. - Function computes partial derivatives , - of the loss function with respect to the planes components - of the previous layer (X) and the weights of the current layer (W) - and updates weights od the current layer by using . - It is a basic function for back propagation method. - Input parameter is the partial derivative of the - loss function with respect to the planes components - of the current layer. */ -static void icvCNNFullConnectBackward( CvCNNLayer* _layer, - int t, - const CvMat* X, - const CvMat* dE_dY, - CvMat* dE_dX ) -{ - CvMat* dE_dY_activ_func_der = 0; - CvMat* dE_dW = 0; - - CV_FUNCNAME( "icvCNNFullConnectBackward" ); - - if( !ICV_IS_CNN_FULLCONNECT_LAYER(_layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - {__BEGIN__; - - const CvCNNFullConnectLayer* layer = (CvCNNFullConnectLayer*)_layer; - const int n_outputs = layer->n_output_planes; - const int n_inputs = layer->n_input_planes; - - int i; - float* dE_dY_activ_func_der_data; - CvMat* weights = layer->weights; - CvMat sub_weights, Xtemplate, Xrow, exp2ssumWXrow; - - CV_ASSERT(X->cols == 1 && X->rows == n_inputs); - CV_ASSERT(dE_dY->rows == 1 && dE_dY->cols == n_outputs ); - CV_ASSERT(dE_dX->rows == 1 && dE_dX->cols == n_inputs ); - - // we violate the convetion about vector's orientation because - // here is more convenient to make this parameter a row-vector - CV_CALL(dE_dY_activ_func_der = cvCreateMat( 1, n_outputs, CV_32FC1 )); - CV_CALL(dE_dW = cvCreateMat( 1, weights->rows*weights->cols, CV_32FC1 )); - - // 1) compute gradients dE_dX and dE_dW - // activ_func_der == 4as*(layer->exp2ssumWX)/(layer->exp2ssumWX + 1)^2 - CV_CALL(cvReshape( layer->exp2ssumWX, &exp2ssumWXrow, 0, layer->exp2ssumWX->cols )); - CV_CALL(cvAddS( &exp2ssumWXrow, cvRealScalar(1), dE_dY_activ_func_der )); - CV_CALL(cvPow( dE_dY_activ_func_der, dE_dY_activ_func_der, -2.0 )); - CV_CALL(cvMul( dE_dY_activ_func_der, &exp2ssumWXrow, dE_dY_activ_func_der, - 4.0*layer->a*layer->s )); - CV_CALL(cvMul( dE_dY, dE_dY_activ_func_der, dE_dY_activ_func_der )); - - // sub_weights = d(W*(X|1))/dX - CV_CALL(cvGetSubRect( weights, &sub_weights, - cvRect(0, 0, weights->cols-1, weights->rows) )); - CV_CALL(cvMatMul( dE_dY_activ_func_der, &sub_weights, dE_dX )); - - cvReshape( X, &Xrow, 0, 1 ); - dE_dY_activ_func_der_data = dE_dY_activ_func_der->data.fl; - Xtemplate = cvMat( 1, n_inputs, CV_32FC1, dE_dW->data.fl ); - for( i = 0; i < n_outputs; i++, Xtemplate.data.fl += n_inputs + 1 ) - { - CV_CALL(cvConvertScale( &Xrow, &Xtemplate, *dE_dY_activ_func_der_data )); - Xtemplate.data.fl[n_inputs] = *dE_dY_activ_func_der_data++; - } - - // 2) update weights - { - CvMat dE_dW_mat; - float eta; - if( layer->learn_rate_decrease_type == CV_CNN_LEARN_RATE_DECREASE_LOG_INV ) - eta = -layer->init_learn_rate/logf(1+(float)t); - else if( layer->learn_rate_decrease_type == CV_CNN_LEARN_RATE_DECREASE_SQRT_INV ) - eta = -layer->init_learn_rate/sqrtf((float)t); - else - eta = -layer->init_learn_rate/(float)t; - cvReshape( dE_dW, &dE_dW_mat, 0, n_outputs ); - cvScaleAdd( &dE_dW_mat, cvRealScalar(eta), weights, weights ); - } - - }__END__; - - cvReleaseMat( &dE_dY_activ_func_der ); - cvReleaseMat( &dE_dW ); -} - -/****************************************************************************************\ -* Layer RELEASE functions * -\****************************************************************************************/ -static void icvCNNConvolutionRelease( CvCNNLayer** p_layer ) -{ - CV_FUNCNAME("icvCNNConvolutionRelease"); - __BEGIN__; - - CvCNNConvolutionLayer* layer = 0; - - if( !p_layer ) - CV_ERROR( CV_StsNullPtr, "Null double pointer" ); - - layer = *(CvCNNConvolutionLayer**)p_layer; - - if( !layer ) - return; - if( !ICV_IS_CNN_CONVOLUTION_LAYER(layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - cvReleaseMat( &layer->weights ); - cvReleaseMat( &layer->connect_mask ); - cvFree( p_layer ); - - __END__; -} - -/****************************************************************************************/ -static void icvCNNSubSamplingRelease( CvCNNLayer** p_layer ) -{ - CV_FUNCNAME("icvCNNSubSamplingRelease"); - __BEGIN__; - - CvCNNSubSamplingLayer* layer = 0; - - if( !p_layer ) - CV_ERROR( CV_StsNullPtr, "Null double pointer" ); - - layer = *(CvCNNSubSamplingLayer**)p_layer; - - if( !layer ) - return; - if( !ICV_IS_CNN_SUBSAMPLING_LAYER(layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - cvReleaseMat( &layer->exp2ssumWX ); - cvReleaseMat( &layer->weights ); - cvFree( p_layer ); - - __END__; -} - -/****************************************************************************************/ -static void icvCNNFullConnectRelease( CvCNNLayer** p_layer ) -{ - CV_FUNCNAME("icvCNNFullConnectRelease"); - __BEGIN__; - - CvCNNFullConnectLayer* layer = 0; - - if( !p_layer ) - CV_ERROR( CV_StsNullPtr, "Null double pointer" ); - - layer = *(CvCNNFullConnectLayer**)p_layer; - - if( !layer ) - return; - if( !ICV_IS_CNN_FULLCONNECT_LAYER(layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - cvReleaseMat( &layer->exp2ssumWX ); - cvReleaseMat( &layer->weights ); - cvFree( p_layer ); - - __END__; -} - -/****************************************************************************************\ -* Read/Write CNN classifier * -\****************************************************************************************/ -static int icvIsCNNModel( const void* ptr ) -{ - return CV_IS_CNN(ptr); -} - -/****************************************************************************************/ -static void icvReleaseCNNModel( void** ptr ) -{ - CV_FUNCNAME("icvReleaseCNNModel"); - __BEGIN__; - - if( !ptr ) - CV_ERROR( CV_StsNullPtr, "NULL double pointer" ); - CV_ASSERT(CV_IS_CNN(*ptr)); - - icvCNNModelRelease( (CvStatModel**)ptr ); - - __END__; -} - -/****************************************************************************************/ -static CvCNNLayer* icvReadCNNLayer( CvFileStorage* fs, CvFileNode* node ) -{ - CvCNNLayer* layer = 0; - CvMat* weights = 0; - CvMat* connect_mask = 0; - - CV_FUNCNAME("icvReadCNNLayer"); - __BEGIN__; - - int n_input_planes, input_height, input_width; - int n_output_planes, output_height, output_width; - int learn_type, layer_type; - float init_learn_rate; - - CV_CALL(n_input_planes = cvReadIntByName( fs, node, "n_input_planes", -1 )); - CV_CALL(input_height = cvReadIntByName( fs, node, "input_height", -1 )); - CV_CALL(input_width = cvReadIntByName( fs, node, "input_width", -1 )); - CV_CALL(n_output_planes = cvReadIntByName( fs, node, "n_output_planes", -1 )); - CV_CALL(output_height = cvReadIntByName( fs, node, "output_height", -1 )); - CV_CALL(output_width = cvReadIntByName( fs, node, "output_width", -1 )); - CV_CALL(layer_type = cvReadIntByName( fs, node, "layer_type", -1 )); - - CV_CALL(init_learn_rate = (float)cvReadRealByName( fs, node, "init_learn_rate", -1 )); - CV_CALL(learn_type = cvReadIntByName( fs, node, "learn_rate_decrease_type", -1 )); - CV_CALL(weights = (CvMat*)cvReadByName( fs, node, "weights" )); - - if( n_input_planes < 0 || input_height < 0 || input_width < 0 || - n_output_planes < 0 || output_height < 0 || output_width < 0 || - init_learn_rate < 0 || learn_type < 0 || layer_type < 0 || !weights ) - CV_ERROR( CV_StsParseError, "" ); - - if( layer_type == ICV_CNN_CONVOLUTION_LAYER ) - { - const int K = input_height - output_height + 1; - if( K <= 0 || K != input_width - output_width + 1 ) - CV_ERROR( CV_StsBadArg, "Invalid " ); - - CV_CALL(connect_mask = (CvMat*)cvReadByName( fs, node, "connect_mask" )); - if( !connect_mask ) - CV_ERROR( CV_StsParseError, "Missing " ); - - CV_CALL(layer = cvCreateCNNConvolutionLayer( - n_input_planes, input_height, input_width, n_output_planes, K, - init_learn_rate, learn_type, connect_mask, weights )); - } - else if( layer_type == ICV_CNN_SUBSAMPLING_LAYER ) - { - float a, s; - const int sub_samp_scale = input_height/output_height; - - if( sub_samp_scale <= 0 || sub_samp_scale != input_width/output_width ) - CV_ERROR( CV_StsBadArg, "Invalid " ); - - CV_CALL(a = (float)cvReadRealByName( fs, node, "a", -1 )); - CV_CALL(s = (float)cvReadRealByName( fs, node, "s", -1 )); - if( a < 0 || s < 0 ) - CV_ERROR( CV_StsParseError, "Missing or " ); - - CV_CALL(layer = cvCreateCNNSubSamplingLayer( - n_input_planes, input_height, input_width, sub_samp_scale, - a, s, init_learn_rate, learn_type, weights )); - } - else if( layer_type == ICV_CNN_FULLCONNECT_LAYER ) - { - float a, s; - CV_CALL(a = (float)cvReadRealByName( fs, node, "a", -1 )); - CV_CALL(s = (float)cvReadRealByName( fs, node, "s", -1 )); - if( a < 0 || s < 0 ) - CV_ERROR( CV_StsParseError, "" ); - if( input_height != 1 || input_width != 1 || - output_height != 1 || output_width != 1 ) - CV_ERROR( CV_StsBadArg, "" ); - - CV_CALL(layer = cvCreateCNNFullConnectLayer( n_input_planes, n_output_planes, - a, s, init_learn_rate, learn_type, weights )); - } - else - CV_ERROR( CV_StsBadArg, "Invalid " ); - - __END__; - - if( cvGetErrStatus() < 0 && layer ) - layer->release( &layer ); - - cvReleaseMat( &weights ); - cvReleaseMat( &connect_mask ); - - return layer; -} - -/****************************************************************************************/ -static void icvWriteCNNLayer( CvFileStorage* fs, CvCNNLayer* layer ) -{ - CV_FUNCNAME ("icvWriteCNNLayer"); - __BEGIN__; - - if( !ICV_IS_CNN_LAYER(layer) ) - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - CV_CALL( cvStartWriteStruct( fs, NULL, CV_NODE_MAP, "opencv-ml-cnn-layer" )); - - CV_CALL(cvWriteInt( fs, "n_input_planes", layer->n_input_planes )); - CV_CALL(cvWriteInt( fs, "input_height", layer->input_height )); - CV_CALL(cvWriteInt( fs, "input_width", layer->input_width )); - CV_CALL(cvWriteInt( fs, "n_output_planes", layer->n_output_planes )); - CV_CALL(cvWriteInt( fs, "output_height", layer->output_height )); - CV_CALL(cvWriteInt( fs, "output_width", layer->output_width )); - CV_CALL(cvWriteInt( fs, "learn_rate_decrease_type", layer->learn_rate_decrease_type)); - CV_CALL(cvWriteReal( fs, "init_learn_rate", layer->init_learn_rate )); - CV_CALL(cvWrite( fs, "weights", layer->weights )); - - if( ICV_IS_CNN_CONVOLUTION_LAYER( layer )) - { - CvCNNConvolutionLayer* l = (CvCNNConvolutionLayer*)layer; - CV_CALL(cvWriteInt( fs, "layer_type", ICV_CNN_CONVOLUTION_LAYER )); - CV_CALL(cvWrite( fs, "connect_mask", l->connect_mask )); - } - else if( ICV_IS_CNN_SUBSAMPLING_LAYER( layer ) ) - { - CvCNNSubSamplingLayer* l = (CvCNNSubSamplingLayer*)layer; - CV_CALL(cvWriteInt( fs, "layer_type", ICV_CNN_SUBSAMPLING_LAYER )); - CV_CALL(cvWriteReal( fs, "a", l->a )); - CV_CALL(cvWriteReal( fs, "s", l->s )); - } - else if( ICV_IS_CNN_FULLCONNECT_LAYER( layer ) ) - { - CvCNNFullConnectLayer* l = (CvCNNFullConnectLayer*)layer; - CV_CALL(cvWriteInt( fs, "layer_type", ICV_CNN_FULLCONNECT_LAYER )); - CV_CALL(cvWriteReal( fs, "a", l->a )); - CV_CALL(cvWriteReal( fs, "s", l->s )); - } - else - CV_ERROR( CV_StsBadArg, "Invalid layer" ); - - CV_CALL( cvEndWriteStruct( fs )); //"opencv-ml-cnn-layer" - - __END__; -} - -/****************************************************************************************/ -static void* icvReadCNNModel( CvFileStorage* fs, CvFileNode* root_node ) -{ - CvCNNStatModel* cnn = 0; - CvCNNLayer* layer = 0; - - CV_FUNCNAME("icvReadCNNModel"); - __BEGIN__; - - CvFileNode* node; - CvSeq* seq; - CvSeqReader reader; - int i; - - CV_CALL(cnn = (CvCNNStatModel*)cvCreateStatModel( - CV_STAT_MODEL_MAGIC_VAL|CV_CNN_MAGIC_VAL, sizeof(CvCNNStatModel), - icvCNNModelRelease, icvCNNModelPredict, icvCNNModelUpdate )); - - CV_CALL(cnn->etalons = (CvMat*)cvReadByName( fs, root_node, "etalons" )); - CV_CALL(cnn->cls_labels = (CvMat*)cvReadByName( fs, root_node, "cls_labels" )); - - if( !cnn->etalons || !cnn->cls_labels ) - CV_ERROR( CV_StsParseError, "No or in CNN model" ); - - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "network" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) ) - CV_ERROR( CV_StsBadArg, "" ); - - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - CV_CALL(layer = icvReadCNNLayer( fs, (CvFileNode*)reader.ptr )); - CV_CALL(cnn->network = cvCreateCNNetwork( layer )); - - for( i = 1; i < seq->total; i++ ) - { - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - CV_CALL(layer = icvReadCNNLayer( fs, (CvFileNode*)reader.ptr )); - CV_CALL(cnn->network->add_layer( cnn->network, layer )); - } - - __END__; - - if( cvGetErrStatus() < 0 ) - { - if( cnn ) cnn->release( (CvStatModel**)&cnn ); - if( layer ) layer->release( &layer ); - } - return (void*)cnn; -} - -/****************************************************************************************/ -static void -icvWriteCNNModel( CvFileStorage* fs, const char* name, - const void* struct_ptr, CvAttrList ) - -{ - CV_FUNCNAME ("icvWriteCNNModel"); - __BEGIN__; - - CvCNNStatModel* cnn = (CvCNNStatModel*)struct_ptr; - int n_layers, i; - CvCNNLayer* layer; - - if( !CV_IS_CNN(cnn) ) - CV_ERROR( CV_StsBadArg, "Invalid pointer" ); - - n_layers = cnn->network->n_layers; - - CV_CALL( cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_CNN )); - - CV_CALL(cvWrite( fs, "etalons", cnn->etalons )); - CV_CALL(cvWrite( fs, "cls_labels", cnn->cls_labels )); - - CV_CALL( cvStartWriteStruct( fs, "network", CV_NODE_SEQ )); - - layer = cnn->network->layers; - for( i = 0; i < n_layers && layer; i++, layer = layer->next_layer ) - CV_CALL(icvWriteCNNLayer( fs, layer )); - if( i < n_layers || layer ) - CV_ERROR( CV_StsBadArg, "Invalid network" ); - - CV_CALL( cvEndWriteStruct( fs )); //"network" - CV_CALL( cvEndWriteStruct( fs )); //"opencv-ml-cnn" - - __END__; -} - -static int icvRegisterCNNStatModelType() -{ - CvTypeInfo info; - - info.header_size = sizeof( info ); - info.is_instance = icvIsCNNModel; - info.release = icvReleaseCNNModel; - info.read = icvReadCNNModel; - info.write = icvWriteCNNModel; - info.clone = NULL; - info.type_name = CV_TYPE_NAME_ML_CNN; - cvRegisterType( &info ); - - return 1; -} // End of icvRegisterCNNStatModelType - -static int cnn = icvRegisterCNNStatModelType(); - -#endif - -// End of file diff --git a/modules/ml/src/data.cpp b/modules/ml/src/data.cpp index 3af1e3bd02a1217a5c30818469fc43b72f744353..b5d0527985760bd4be7c929ddbeef3d22f357aec 100644 --- a/modules/ml/src/data.cpp +++ b/modules/ml/src/data.cpp @@ -40,753 +40,958 @@ #include "precomp.hpp" #include +#include +#include -#define MISS_VAL FLT_MAX -#define CV_VAR_MISS 0 +namespace cv { namespace ml { -CvTrainTestSplit::CvTrainTestSplit() -{ - train_sample_part_mode = CV_COUNT; - train_sample_part.count = -1; - mix = false; -} +static const float MISSED_VAL = TrainData::missingValue(); +static const int VAR_MISSED = VAR_ORDERED; -CvTrainTestSplit::CvTrainTestSplit( int _train_sample_count, bool _mix ) -{ - train_sample_part_mode = CV_COUNT; - train_sample_part.count = _train_sample_count; - mix = _mix; -} +TrainData::~TrainData() {} -CvTrainTestSplit::CvTrainTestSplit( float _train_sample_portion, bool _mix ) +Mat TrainData::getSubVector(const Mat& vec, const Mat& idx) { - train_sample_part_mode = CV_PORTION; - train_sample_part.portion = _train_sample_portion; - mix = _mix; -} - -//////////////// + if( idx.empty() ) + return vec; + int i, j, n = idx.checkVector(1, CV_32S); + int type = vec.type(); + CV_Assert( type == CV_32S || type == CV_32F || type == CV_64F ); + int dims = 1, m; -CvMLData::CvMLData() -{ - values = missing = var_types = var_idx_mask = response_out = var_idx_out = var_types_out = 0; - train_sample_idx = test_sample_idx = 0; - header_lines_number = 0; - sample_idx = 0; - response_idx = -1; - - train_sample_count = -1; - - delimiter = ','; - miss_ch = '?'; - //flt_separator = '.'; - - rng = &cv::theRNG(); -} + if( vec.cols == 1 || vec.rows == 1 ) + { + dims = 1; + m = vec.cols + vec.rows - 1; + } + else + { + dims = vec.cols; + m = vec.rows; + } -CvMLData::~CvMLData() -{ - clear(); -} + Mat subvec; -void CvMLData::free_train_test_idx() -{ - cvReleaseMat( &train_sample_idx ); - cvReleaseMat( &test_sample_idx ); - sample_idx = 0; + if( vec.cols == m ) + subvec.create(dims, n, type); + else + subvec.create(n, dims, type); + if( type == CV_32S ) + for( i = 0; i < n; i++ ) + { + int k = idx.at(i); + CV_Assert( 0 <= k && k < m ); + if( dims == 1 ) + subvec.at(i) = vec.at(k); + else + for( j = 0; j < dims; j++ ) + subvec.at(i, j) = vec.at(k, j); + } + else if( type == CV_32F ) + for( i = 0; i < n; i++ ) + { + int k = idx.at(i); + CV_Assert( 0 <= k && k < m ); + if( dims == 1 ) + subvec.at(i) = vec.at(k); + else + for( j = 0; j < dims; j++ ) + subvec.at(i, j) = vec.at(k, j); + } + else + for( i = 0; i < n; i++ ) + { + int k = idx.at(i); + CV_Assert( 0 <= k && k < m ); + if( dims == 1 ) + subvec.at(i) = vec.at(k); + else + for( j = 0; j < dims; j++ ) + subvec.at(i, j) = vec.at(k, j); + } + return subvec; } -void CvMLData::clear() +class TrainDataImpl : public TrainData { - class_map.clear(); +public: + typedef std::map MapType; - cvReleaseMat( &values ); - cvReleaseMat( &missing ); - cvReleaseMat( &var_types ); - cvReleaseMat( &var_idx_mask ); - - cvReleaseMat( &response_out ); - cvReleaseMat( &var_idx_out ); - cvReleaseMat( &var_types_out ); + TrainDataImpl() + { + file = 0; + clear(); + } - free_train_test_idx(); + virtual ~TrainDataImpl() { closeFile(); } - total_class_count = 0; + int getLayout() const { return layout; } + int getNSamples() const + { + return !sampleIdx.empty() ? (int)sampleIdx.total() : + layout == ROW_SAMPLE ? samples.rows : samples.cols; + } + int getNTrainSamples() const + { + return !trainSampleIdx.empty() ? (int)trainSampleIdx.total() : getNSamples(); + } + int getNTestSamples() const + { + return !testSampleIdx.empty() ? (int)testSampleIdx.total() : 0; + } + int getNVars() const + { + return !varIdx.empty() ? (int)varIdx.total() : getNAllVars(); + } + int getNAllVars() const + { + return layout == ROW_SAMPLE ? samples.cols : samples.rows; + } - response_idx = -1; + Mat getSamples() const { return samples; } + Mat getResponses() const { return responses; } + Mat getMissing() const { return missing; } + Mat getVarIdx() const { return varIdx; } + Mat getVarType() const { return varType; } + int getResponseType() const + { + return classLabels.empty() ? VAR_ORDERED : VAR_CATEGORICAL; + } + Mat getTrainSampleIdx() const { return !trainSampleIdx.empty() ? trainSampleIdx : sampleIdx; } + Mat getTestSampleIdx() const { return testSampleIdx; } + Mat getSampleWeights() const + { + return sampleWeights; + } + Mat getTrainSampleWeights() const + { + return getSubVector(sampleWeights, getTrainSampleIdx()); + } + Mat getTestSampleWeights() const + { + Mat idx = getTestSampleIdx(); + return idx.empty() ? Mat() : getSubVector(sampleWeights, idx); + } + Mat getTrainResponses() const + { + return getSubVector(responses, getTrainSampleIdx()); + } + Mat getTrainNormCatResponses() const + { + return getSubVector(normCatResponses, getTrainSampleIdx()); + } + Mat getTestResponses() const + { + Mat idx = getTestSampleIdx(); + return idx.empty() ? Mat() : getSubVector(responses, idx); + } + Mat getTestNormCatResponses() const + { + Mat idx = getTestSampleIdx(); + return idx.empty() ? Mat() : getSubVector(normCatResponses, idx); + } + Mat getNormCatResponses() const { return normCatResponses; } + Mat getClassLabels() const { return classLabels; } + Mat getClassCounters() const { return classCounters; } + int getCatCount(int vi) const + { + int n = (int)catOfs.total(); + CV_Assert( 0 <= vi && vi < n ); + Vec2i ofs = catOfs.at(vi); + return ofs[1] - ofs[0]; + } - train_sample_count = -1; -} + Mat getCatOfs() const { return catOfs; } + Mat getCatMap() const { return catMap; } + Mat getDefaultSubstValues() const { return missingSubst; } -void CvMLData::set_header_lines_number( int idx ) -{ - header_lines_number = std::max(0, idx); -} + void closeFile() { if(file) fclose(file); file=0; } + void clear() + { + closeFile(); + samples.release(); + missing.release(); + varType.release(); + responses.release(); + sampleIdx.release(); + trainSampleIdx.release(); + testSampleIdx.release(); + normCatResponses.release(); + classLabels.release(); + classCounters.release(); + catMap.release(); + catOfs.release(); + nameMap = MapType(); + layout = ROW_SAMPLE; + } -int CvMLData::get_header_lines_number() const -{ - return header_lines_number; -} + typedef std::map CatMapHash; -static char *fgets_chomp(char *str, int n, FILE *stream) -{ - char *head = fgets(str, n, stream); - if( head ) + void setData(InputArray _samples, int _layout, InputArray _responses, + InputArray _varIdx, InputArray _sampleIdx, InputArray _sampleWeights, + InputArray _varType, InputArray _missing) { - for(char *tail = head + strlen(head) - 1; tail >= head; --tail) - { - if( *tail != '\r' && *tail != '\n' ) - break; - *tail = '\0'; - } - } - return head; -} + clear(); + CV_Assert(_layout == ROW_SAMPLE || _layout == COL_SAMPLE ); + samples = _samples.getMat(); + layout = _layout; + responses = _responses.getMat(); + varIdx = _varIdx.getMat(); + sampleIdx = _sampleIdx.getMat(); + sampleWeights = _sampleWeights.getMat(); + varType = _varType.getMat(); + missing = _missing.getMat(); -int CvMLData::read_csv(const char* filename) -{ - const int M = 1000000; - const char str_delimiter[3] = { ' ', delimiter, '\0' }; - FILE* file = 0; - CvMemStorage* storage; - CvSeq* seq; - char *ptr; - float* el_ptr; - CvSeqReader reader; - int cols_count = 0; - uchar *var_types_ptr = 0; + int nsamples = layout == ROW_SAMPLE ? samples.rows : samples.cols; + int ninputvars = layout == ROW_SAMPLE ? samples.cols : samples.rows; + int i, noutputvars = 0; - clear(); + CV_Assert( samples.type() == CV_32F || samples.type() == CV_32S ); - file = fopen( filename, "rt" ); + if( !sampleIdx.empty() ) + { + CV_Assert( (sampleIdx.checkVector(1, CV_32S, true) > 0 && + checkRange(sampleIdx, true, 0, 0, nsamples-1)) || + sampleIdx.checkVector(1, CV_8U, true) == nsamples ); + if( sampleIdx.type() == CV_8U ) + sampleIdx = convertMaskToIdx(sampleIdx); + } - if( !file ) - return -1; + if( !sampleWeights.empty() ) + { + CV_Assert( sampleWeights.checkVector(1, CV_32F, true) == nsamples ); + } + else + { + sampleWeights = Mat::ones(nsamples, 1, CV_32F); + } - std::vector _buf(M); - char* buf = &_buf[0]; + if( !varIdx.empty() ) + { + CV_Assert( (varIdx.checkVector(1, CV_32S, true) > 0 && + checkRange(varIdx, true, 0, 0, ninputvars)) || + varIdx.checkVector(1, CV_8U, true) == ninputvars ); + if( varIdx.type() == CV_8U ) + varIdx = convertMaskToIdx(varIdx); + varIdx = varIdx.clone(); + std::sort(varIdx.ptr(), varIdx.ptr() + varIdx.total()); + } - // skip header lines - for( int i = 0; i < header_lines_number; i++ ) - { - if( fgets( buf, M, file ) == 0 ) + if( !responses.empty() ) { - fclose(file); - return -1; + CV_Assert( responses.type() == CV_32F || responses.type() == CV_32S ); + if( (responses.cols == 1 || responses.rows == 1) && (int)responses.total() == nsamples ) + noutputvars = 1; + else + { + CV_Assert( (layout == ROW_SAMPLE && responses.rows == nsamples) || + (layout == COL_SAMPLE && responses.cols == nsamples) ); + noutputvars = layout == ROW_SAMPLE ? responses.cols : responses.rows; + } + if( !responses.isContinuous() || (layout == COL_SAMPLE && noutputvars > 1) ) + { + Mat temp; + transpose(responses, temp); + responses = temp; + } } - } - // read the first data line and determine the number of variables - if( !fgets_chomp( buf, M, file )) - { - fclose(file); - return -1; - } + int nvars = ninputvars + noutputvars; - ptr = buf; - while( *ptr == ' ' ) - ptr++; - for( ; *ptr != '\0'; ) - { - if(*ptr == delimiter || *ptr == ' ') + if( !varType.empty() ) { - cols_count++; - ptr++; - while( *ptr == ' ' ) ptr++; + CV_Assert( varType.checkVector(1, CV_8U, true) == nvars && + checkRange(varType, true, 0, VAR_ORDERED, VAR_CATEGORICAL+1) ); } else - ptr++; - } + { + varType.create(1, nvars, CV_8U); + varType = Scalar::all(VAR_ORDERED); + if( noutputvars == 1 ) + varType.at(ninputvars) = (uchar)(responses.type() < CV_32F ? VAR_CATEGORICAL : VAR_ORDERED); + } - cols_count++; + if( noutputvars > 1 ) + { + for( i = 0; i < noutputvars; i++ ) + CV_Assert( varType.at(ninputvars + i) == VAR_ORDERED ); + } - if ( cols_count == 0) - { - fclose(file); - return -1; - } + catOfs = Mat::zeros(1, nvars, CV_32SC2); + missingSubst = Mat::zeros(1, nvars, CV_32F); - // create temporary memory storage to store the whole database - el_ptr = new float[cols_count]; - storage = cvCreateMemStorage(); - seq = cvCreateSeq( 0, sizeof(*seq), cols_count*sizeof(float), storage ); + vector labels, counters, sortbuf, tempCatMap; + vector tempCatOfs; + CatMapHash ofshash; - var_types = cvCreateMat( 1, cols_count, CV_8U ); - cvZero( var_types ); - var_types_ptr = var_types->data.ptr; + AutoBuffer buf(nsamples); + Mat non_missing(layout == ROW_SAMPLE ? Size(1, nsamples) : Size(nsamples, 1), CV_8U, (uchar*)buf); + bool haveMissing = !missing.empty(); + if( haveMissing ) + { + CV_Assert( missing.size() == samples.size() && missing.type() == CV_8U ); + } - for(;;) - { - char *token = NULL; - int type; - token = strtok(buf, str_delimiter); - if (!token) - break; - for (int i = 0; i < cols_count-1; i++) + // we iterate through all the variables. For each categorical variable we build a map + // in order to convert input values of the variable into normalized values (0..catcount_vi-1) + // often many categorical variables are similar, so we compress the map - try to re-use + // maps for different variables if they are identical + for( i = 0; i < ninputvars; i++ ) { - str_to_flt_elem( token, el_ptr[i], type); - var_types_ptr[i] |= type; - token = strtok(NULL, str_delimiter); - if (!token) + Mat values_i = layout == ROW_SAMPLE ? samples.col(i) : samples.row(i); + + if( varType.at(i) == VAR_CATEGORICAL ) + { + preprocessCategorical(values_i, 0, labels, 0, sortbuf); + missingSubst.at(i) = -1.f; + int j, m = (int)labels.size(); + CV_Assert( m > 0 ); + int a = labels.front(), b = labels.back(); + const int* currmap = &labels[0]; + int hashval = ((unsigned)a*127 + (unsigned)b)*127 + m; + CatMapHash::iterator it = ofshash.find(hashval); + if( it != ofshash.end() ) + { + int vi = it->second; + Vec2i ofs0 = tempCatOfs[vi]; + int m0 = ofs0[1] - ofs0[0]; + const int* map0 = &tempCatMap[ofs0[0]]; + if( m0 == m && map0[0] == a && map0[m0-1] == b ) + { + for( j = 0; j < m; j++ ) + if( map0[j] != currmap[j] ) + break; + if( j == m ) + { + // re-use the map + tempCatOfs.push_back(ofs0); + continue; + } + } + } + else + ofshash[hashval] = i; + Vec2i ofs; + ofs[0] = (int)tempCatMap.size(); + ofs[1] = ofs[0] + m; + tempCatOfs.push_back(ofs); + std::copy(labels.begin(), labels.end(), std::back_inserter(tempCatMap)); + } + else { - fclose(file); - delete [] el_ptr; - return -1; + tempCatOfs.push_back(Vec2i(0, 0)); + /*Mat missing_i = layout == ROW_SAMPLE ? missing.col(i) : missing.row(i); + compare(missing_i, Scalar::all(0), non_missing, CMP_EQ); + missingSubst.at(i) = (float)(mean(values_i, non_missing)[0]);*/ + missingSubst.at(i) = 0.f; } } - str_to_flt_elem( token, el_ptr[cols_count-1], type); - var_types_ptr[cols_count-1] |= type; - cvSeqPush( seq, el_ptr ); - if( !fgets_chomp( buf, M, file ) ) - break; - } - fclose(file); - - values = cvCreateMat( seq->total, cols_count, CV_32FC1 ); - missing = cvCreateMat( seq->total, cols_count, CV_8U ); - var_idx_mask = cvCreateMat( 1, values->cols, CV_8UC1 ); - cvSet( var_idx_mask, cvRealScalar(1) ); - train_sample_count = seq->total; - cvStartReadSeq( seq, &reader ); - for(int i = 0; i < seq->total; i++ ) - { - const float* sdata = (float*)reader.ptr; - float* ddata = values->data.fl + cols_count*i; - uchar* dm = missing->data.ptr + cols_count*i; + if( !tempCatOfs.empty() ) + { + Mat(tempCatOfs).copyTo(catOfs); + Mat(tempCatMap).copyTo(catMap); + } - for( int j = 0; j < cols_count; j++ ) + if( varType.at(ninputvars) == VAR_CATEGORICAL ) { - ddata[j] = sdata[j]; - dm[j] = ( fabs( MISS_VAL - sdata[j] ) <= FLT_EPSILON ); + preprocessCategorical(responses, &normCatResponses, labels, &counters, sortbuf); + Mat(labels).copyTo(classLabels); + Mat(counters).copyTo(classCounters); } - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); } - if ( cvNorm( missing, 0, CV_L1 ) <= FLT_EPSILON ) - cvReleaseMat( &missing ); + Mat convertMaskToIdx(const Mat& mask) + { + int i, j, nz = countNonZero(mask), n = mask.cols + mask.rows - 1; + Mat idx(1, nz, CV_32S); + for( i = j = 0; i < n; i++ ) + if( mask.at(i) ) + idx.at(j++) = i; + return idx; + } - cvReleaseMemStorage( &storage ); - delete []el_ptr; - return 0; -} + struct CmpByIdx + { + CmpByIdx(const int* _data, int _step) : data(_data), step(_step) {} + bool operator ()(int i, int j) const { return data[i*step] < data[j*step]; } + const int* data; + int step; + }; + + void preprocessCategorical(const Mat& data, Mat* normdata, vector& labels, + vector* counters, vector& sortbuf) + { + CV_Assert((data.cols == 1 || data.rows == 1) && (data.type() == CV_32S || data.type() == CV_32F)); + int* odata = 0; + int ostep = 0; -const CvMat* CvMLData::get_values() const -{ - return values; -} + if(normdata) + { + normdata->create(data.size(), CV_32S); + odata = normdata->ptr(); + ostep = normdata->isContinuous() ? 1 : (int)normdata->step1(); + } -const CvMat* CvMLData::get_missing() const -{ - CV_FUNCNAME( "CvMLData::get_missing" ); - __BEGIN__; + int i, n = data.cols + data.rows - 1; + sortbuf.resize(n*2); + int* idx = &sortbuf[0]; + int* idata = (int*)data.ptr(); + int istep = data.isContinuous() ? 1 : (int)data.step1(); - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); + if( data.type() == CV_32F ) + { + idata = idx + n; + const float* fdata = data.ptr(); + for( i = 0; i < n; i++ ) + { + if( fdata[i*istep] == MISSED_VAL ) + idata[i] = -1; + else + { + idata[i] = cvRound(fdata[i*istep]); + CV_Assert( (float)idata[i] == fdata[i*istep] ); + } + } + istep = 1; + } - __END__; + for( i = 0; i < n; i++ ) + idx[i] = i; - return missing; -} + std::sort(idx, idx + n, CmpByIdx(idata, istep)); -const std::map& CvMLData::get_class_labels_map() const -{ - return class_map; -} + int clscount = 1; + for( i = 1; i < n; i++ ) + clscount += idata[idx[i]*istep] != idata[idx[i-1]*istep]; -void CvMLData::str_to_flt_elem( const char* token, float& flt_elem, int& type) -{ + int clslabel = -1; + int prev = ~idata[idx[0]*istep]; + int previdx = 0; - char* stopstring = NULL; - flt_elem = (float)strtod( token, &stopstring ); - assert( stopstring ); - type = CV_VAR_ORDERED; - if ( *stopstring == miss_ch && strlen(stopstring) == 1 ) // missed value - { - flt_elem = MISS_VAL; - type = CV_VAR_MISS; - } - else - { - if ( (*stopstring != 0) && (*stopstring != '\n') && (strcmp(stopstring, "\r\n") != 0) ) // class label + labels.resize(clscount); + if(counters) + counters->resize(clscount); + + for( i = 0; i < n; i++ ) { - int idx = class_map[token]; - if ( idx == 0) + int l = idata[idx[i]*istep]; + if( l != prev ) { - total_class_count++; - idx = total_class_count; - class_map[token] = idx; + clslabel++; + labels[clslabel] = l; + int k = i - previdx; + if( clslabel > 0 && counters ) + counters->at(clslabel-1) = k; + prev = l; + previdx = i; } - flt_elem = (float)idx; - type = CV_VAR_CATEGORICAL; + if(odata) + odata[idx[i]*ostep] = clslabel; } + if(counters) + counters->at(clslabel) = i - previdx; } -} - -void CvMLData::set_delimiter(char ch) -{ - CV_FUNCNAME( "CvMLData::set_delimited" ); - __BEGIN__; - - if (ch == miss_ch /*|| ch == flt_separator*/) - CV_ERROR(CV_StsBadArg, "delimited, miss_character and flt_separator must be different"); - - delimiter = ch; - __END__; -} + bool loadCSV(const String& filename, int headerLines, + int responseStartIdx, int responseEndIdx, + const String& varTypeSpec, char delimiter, char missch) + { + const int M = 1000000; + const char delimiters[3] = { ' ', delimiter, '\0' }; + int nvars = 0; + bool varTypesSet = false; -char CvMLData::get_delimiter() const -{ - return delimiter; -} + clear(); -void CvMLData::set_miss_ch(char ch) -{ - CV_FUNCNAME( "CvMLData::set_miss_ch" ); - __BEGIN__; + file = fopen( filename.c_str(), "rt" ); - if (ch == delimiter/* || ch == flt_separator*/) - CV_ERROR(CV_StsBadArg, "delimited, miss_character and flt_separator must be different"); + if( !file ) + return false; - miss_ch = ch; + std::vector _buf(M); + std::vector allresponses; + std::vector rowvals; + std::vector vtypes, rowtypes; + bool haveMissed = false; + char* buf = &_buf[0]; - __END__; -} + int i, ridx0 = responseStartIdx, ridx1 = responseEndIdx; + int ninputvars = 0, noutputvars = 0; -char CvMLData::get_miss_ch() const -{ - return miss_ch; -} + Mat tempSamples, tempMissing, tempResponses; + MapType tempNameMap; + int catCounter = 1; -void CvMLData::set_response_idx( int idx ) -{ - CV_FUNCNAME( "CvMLData::set_response_idx" ); - __BEGIN__; + // skip header lines + int lineno = 0; + for(;;lineno++) + { + if( !fgets(buf, M, file) ) + break; + if(lineno < headerLines ) + continue; + // trim trailing spaces + int idx = (int)strlen(buf)-1; + while( idx >= 0 && isspace(buf[idx]) ) + buf[idx--] = '\0'; + // skip spaces in the beginning + char* ptr = buf; + while( *ptr != '\0' && isspace(*ptr) ) + ptr++; + // skip commented off lines + if(*ptr == '#') + continue; + rowvals.clear(); + rowtypes.clear(); + + char* token = strtok(buf, delimiters); + if (!token) + break; - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); + for(;;) + { + float val=0.f; int tp = 0; + decodeElem( token, val, tp, missch, tempNameMap, catCounter ); + if( tp == VAR_MISSED ) + haveMissed = true; + rowvals.push_back(val); + rowtypes.push_back((uchar)tp); + token = strtok(NULL, delimiters); + if (!token) + break; + } - if ( idx >= values->cols) - CV_ERROR( CV_StsBadArg, "idx value is not correct" ); + if( nvars == 0 ) + { + if( rowvals.empty() ) + CV_Error(CV_StsBadArg, "invalid CSV format; no data found"); + nvars = (int)rowvals.size(); + if( !varTypeSpec.empty() && varTypeSpec.size() > 0 ) + { + setVarTypes(varTypeSpec, nvars, vtypes); + varTypesSet = true; + } + else + vtypes = rowtypes; - if ( response_idx >= 0 ) - chahge_var_idx( response_idx, true ); - if ( idx >= 0 ) - chahge_var_idx( idx, false ); - response_idx = idx; + ridx0 = ridx0 >= 0 ? ridx0 : ridx0 == -1 ? nvars - 1 : -1; + ridx1 = ridx1 >= 0 ? ridx1 : ridx0 >= 0 ? ridx0+1 : -1; + CV_Assert(ridx1 > ridx0); + noutputvars = ridx0 >= 0 ? ridx1 - ridx0 : 0; + ninputvars = nvars - noutputvars; + } + else + CV_Assert( nvars == (int)rowvals.size() ); - __END__; -} + // check var types + for( i = 0; i < nvars; i++ ) + { + CV_Assert( (!varTypesSet && vtypes[i] == rowtypes[i]) || + (varTypesSet && (vtypes[i] == rowtypes[i] || rowtypes[i] == VAR_ORDERED)) ); + } -int CvMLData::get_response_idx() const -{ - CV_FUNCNAME( "CvMLData::get_response_idx" ); - __BEGIN__; + if( ridx0 >= 0 ) + { + for( i = ridx1; i < nvars; i++ ) + std::swap(rowvals[i], rowvals[i-noutputvars]); + for( i = ninputvars; i < nvars; i++ ) + allresponses.push_back(rowvals[i]); + rowvals.pop_back(); + } + Mat rmat(1, ninputvars, CV_32F, &rowvals[0]); + tempSamples.push_back(rmat); + } - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - __END__; - return response_idx; -} + closeFile(); -void CvMLData::change_var_type( int var_idx, int type ) -{ - CV_FUNCNAME( "CvMLData::change_var_type" ); - __BEGIN__; + int nsamples = tempSamples.rows; + if( nsamples == 0 ) + return false; - int var_count = 0; + if( haveMissed ) + compare(tempSamples, MISSED_VAL, tempMissing, CMP_EQ); - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); + if( ridx0 >= 0 ) + { + for( i = ridx1; i < nvars; i++ ) + std::swap(vtypes[i], vtypes[i-noutputvars]); + if( noutputvars > 1 ) + { + for( i = ninputvars; i < nvars; i++ ) + if( vtypes[i] == VAR_CATEGORICAL ) + CV_Error(CV_StsBadArg, + "If responses are vector values, not scalars, they must be marked as ordered responses"); + } + } - var_count = values->cols; + if( !varTypesSet && noutputvars == 1 && vtypes[ninputvars] == VAR_ORDERED ) + { + for( i = 0; i < nsamples; i++ ) + if( allresponses[i] != cvRound(allresponses[i]) ) + break; + if( i == nsamples ) + vtypes[ninputvars] = VAR_CATEGORICAL; + } - if ( var_idx < 0 || var_idx >= var_count) - CV_ERROR( CV_StsBadArg, "var_idx is not correct" ); + Mat(nsamples, noutputvars, CV_32F, &allresponses[0]).copyTo(tempResponses); + setData(tempSamples, ROW_SAMPLE, tempResponses, noArray(), noArray(), + noArray(), Mat(vtypes).clone(), tempMissing); + bool ok = !samples.empty(); + if(ok) + std::swap(tempNameMap, nameMap); + return ok; + } - if ( type != CV_VAR_ORDERED && type != CV_VAR_CATEGORICAL) - CV_ERROR( CV_StsBadArg, "type is not correct" ); + void decodeElem( const char* token, float& elem, int& type, + char missch, MapType& namemap, int& counter ) const + { + char* stopstring = NULL; + elem = (float)strtod( token, &stopstring ); + if( *stopstring == missch && strlen(stopstring) == 1 ) // missed value + { + elem = MISSED_VAL; + type = VAR_MISSED; + } + else if( *stopstring != '\0' ) + { + MapType::iterator it = namemap.find(token); + if( it == namemap.end() ) + { + elem = (float)counter; + namemap[token] = counter++; + } + else + elem = (float)it->second; + type = VAR_CATEGORICAL; + } + else + type = VAR_ORDERED; + } - assert( var_types ); - if ( var_types->data.ptr[var_idx] == CV_VAR_CATEGORICAL && type == CV_VAR_ORDERED) - CV_ERROR( CV_StsBadArg, "it`s impossible to assign CV_VAR_ORDERED type to categorical variable" ); - var_types->data.ptr[var_idx] = (uchar)type; + void setVarTypes( const String& s, int nvars, std::vector& vtypes ) const + { + const char* errmsg = "type spec is not correct; it should have format \"cat\", \"ord\" or " + "\"ord[n1,n2-n3,n4-n5,...]cat[m1-m2,m3,m4-m5,...]\", where n's and m's are 0-based variable indices"; + const char* str = s.c_str(); + int specCounter = 0; - __END__; + vtypes.resize(nvars); - return; -} + for( int k = 0; k < 2; k++ ) + { + const char* ptr = strstr(str, k == 0 ? "ord" : "cat"); + int tp = k == 0 ? VAR_ORDERED : VAR_CATEGORICAL; + if( ptr ) // parse ord/cat str + { + char* stopstring = NULL; -void CvMLData::set_var_types( const char* str ) -{ - CV_FUNCNAME( "CvMLData::set_var_types" ); - __BEGIN__; + if( ptr[3] == '\0' ) + { + for( int i = 0; i < nvars; i++ ) + vtypes[i] = (uchar)tp; + specCounter = nvars; + break; + } - const char* ord = 0, *cat = 0; - int var_count = 0, set_var_type_count = 0; - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); + if ( ptr[3] != '[') + CV_Error( CV_StsBadArg, errmsg ); - var_count = values->cols; + ptr += 4; // pass "ord[" + do + { + int b1 = (int)strtod( ptr, &stopstring ); + if( *stopstring == 0 || (*stopstring != ',' && *stopstring != ']' && *stopstring != '-') ) + CV_Error( CV_StsBadArg, errmsg ); + ptr = stopstring + 1; + if( (stopstring[0] == ',') || (stopstring[0] == ']')) + { + CV_Assert( 0 <= b1 && b1 < nvars ); + vtypes[b1] = (uchar)tp; + specCounter++; + } + else + { + if( stopstring[0] == '-') + { + int b2 = (int)strtod( ptr, &stopstring); + if ( (*stopstring == 0) || (*stopstring != ',' && *stopstring != ']') ) + CV_Error( CV_StsBadArg, errmsg ); + ptr = stopstring + 1; + CV_Assert( 0 <= b1 && b1 <= b2 && b2 < nvars ); + for (int i = b1; i <= b2; i++) + vtypes[i] = (uchar)tp; + specCounter += b2 - b1 + 1; + } + else + CV_Error( CV_StsBadArg, errmsg ); - assert( var_types ); + } + } + while(*stopstring != ']'); - ord = strstr( str, "ord" ); - cat = strstr( str, "cat" ); - if ( !ord && !cat ) - CV_ERROR( CV_StsBadArg, "types string is not correct" ); + if( stopstring[1] != '\0' && stopstring[1] != ',') + CV_Error( CV_StsBadArg, errmsg ); + } + } - if ( !ord && strlen(cat) == 3 ) // str == "cat" - { - cvSet( var_types, cvScalarAll(CV_VAR_CATEGORICAL) ); - return; + if( specCounter != nvars ) + CV_Error( CV_StsBadArg, "type of some variables is not specified" ); } - if ( !cat && strlen(ord) == 3 ) // str == "ord" + void setTrainTestSplitRatio(double ratio, bool shuffle) { - cvSet( var_types, cvScalarAll(CV_VAR_ORDERED) ); - return; + CV_Assert( 0. <= ratio && ratio <= 1. ); + setTrainTestSplit(cvRound(getNSamples()*ratio), shuffle); } - if ( ord ) // parse ord str + void setTrainTestSplit(int count, bool shuffle) { - char* stopstring = NULL; - if ( ord[3] != '[') - CV_ERROR( CV_StsBadArg, "types string is not correct" ); + int i, nsamples = getNSamples(); + CV_Assert( 0 <= count && count < nsamples ); + + trainSampleIdx.release(); + testSampleIdx.release(); - ord += 4; // pass "ord[" - do + if( count == 0 ) + trainSampleIdx = sampleIdx; + else if( count == nsamples ) + testSampleIdx = sampleIdx; + else { - int b1 = (int)strtod( ord, &stopstring ); - if ( *stopstring == 0 || (*stopstring != ',' && *stopstring != ']' && *stopstring != '-') ) - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - ord = stopstring + 1; - if ( (stopstring[0] == ',') || (stopstring[0] == ']')) + Mat mask(1, nsamples, CV_8U); + uchar* mptr = mask.data; + for( i = 0; i < nsamples; i++ ) + mptr[i] = (uchar)(i < count); + trainSampleIdx.create(1, count, CV_32S); + testSampleIdx.create(1, nsamples - count, CV_32S); + int j0 = 0, j1 = 0; + const int* sptr = !sampleIdx.empty() ? sampleIdx.ptr() : 0; + int* trainptr = trainSampleIdx.ptr(); + int* testptr = testSampleIdx.ptr(); + for( i = 0; i < nsamples; i++ ) { - if ( var_types->data.ptr[b1] == CV_VAR_CATEGORICAL) - CV_ERROR( CV_StsBadArg, "it`s impossible to assign CV_VAR_ORDERED type to categorical variable" ); - var_types->data.ptr[b1] = CV_VAR_ORDERED; - set_var_type_count++; - } - else - { - if ( stopstring[0] == '-') - { - int b2 = (int)strtod( ord, &stopstring); - if ( (*stopstring == 0) || (*stopstring != ',' && *stopstring != ']') ) - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - ord = stopstring + 1; - for (int i = b1; i <= b2; i++) - { - if ( var_types->data.ptr[i] == CV_VAR_CATEGORICAL) - CV_ERROR( CV_StsBadArg, "it`s impossible to assign CV_VAR_ORDERED type to categorical variable" ); - var_types->data.ptr[i] = CV_VAR_ORDERED; - } - set_var_type_count += b2 - b1 + 1; - } + int idx = sptr ? sptr[i] : i; + if( mptr[i] ) + trainptr[j0++] = idx; else - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - + testptr[j1++] = idx; } + if( shuffle ) + shuffleTrainTest(); } - while (*stopstring != ']'); - - if ( stopstring[1] != '\0' && stopstring[1] != ',') - CV_ERROR( CV_StsBadArg, "types string is not correct" ); } - if ( cat ) // parse cat str + void shuffleTrainTest() { - char* stopstring = NULL; - if ( cat[3] != '[') - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - - cat += 4; // pass "cat[" - do + if( !trainSampleIdx.empty() && !testSampleIdx.empty() ) { - int b1 = (int)strtod( cat, &stopstring ); - if ( *stopstring == 0 || (*stopstring != ',' && *stopstring != ']' && *stopstring != '-') ) - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - cat = stopstring + 1; - if ( (stopstring[0] == ',') || (stopstring[0] == ']')) - { - var_types->data.ptr[b1] = CV_VAR_CATEGORICAL; - set_var_type_count++; - } - else + int i, nsamples = getNSamples(), ntrain = getNTrainSamples(), ntest = getNTestSamples(); + int* trainIdx = trainSampleIdx.ptr(); + int* testIdx = testSampleIdx.ptr(); + RNG& rng = theRNG(); + + for( i = 0; i < nsamples; i++) { - if ( stopstring[0] == '-') + int a = rng.uniform(0, nsamples); + int b = rng.uniform(0, nsamples); + int* ptra = trainIdx; + int* ptrb = trainIdx; + if( a >= ntrain ) { - int b2 = (int)strtod( cat, &stopstring); - if ( (*stopstring == 0) || (*stopstring != ',' && *stopstring != ']') ) - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - cat = stopstring + 1; - for (int i = b1; i <= b2; i++) - var_types->data.ptr[i] = CV_VAR_CATEGORICAL; - set_var_type_count += b2 - b1 + 1; + ptra = testIdx; + a -= ntrain; + CV_Assert( a < ntest ); } - else - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - + if( b >= ntrain ) + { + ptrb = testIdx; + b -= ntrain; + CV_Assert( b < ntest ); + } + std::swap(ptra[a], ptrb[b]); } } - while (*stopstring != ']'); - - if ( stopstring[1] != '\0' && stopstring[1] != ',') - CV_ERROR( CV_StsBadArg, "types string is not correct" ); } - if (set_var_type_count != var_count) - CV_ERROR( CV_StsBadArg, "types string is not correct" ); - - __END__; -} - -const CvMat* CvMLData::get_var_types() -{ - CV_FUNCNAME( "CvMLData::get_var_types" ); - __BEGIN__; - - uchar *var_types_out_ptr = 0; - int avcount, vt_size; - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - - assert( var_idx_mask ); - - avcount = cvFloor( cvNorm( var_idx_mask, 0, CV_L1 ) ); - vt_size = avcount + (response_idx >= 0); - - if ( avcount == values->cols || (avcount == values->cols-1 && response_idx == values->cols-1) ) - return var_types; - - if ( !var_types_out || ( var_types_out && var_types_out->cols != vt_size ) ) + Mat getTrainSamples(int _layout, + bool compressSamples, + bool compressVars) const { - cvReleaseMat( &var_types_out ); - var_types_out = cvCreateMat( 1, vt_size, CV_8UC1 ); - } - - var_types_out_ptr = var_types_out->data.ptr; - for( int i = 0; i < var_types->cols; i++) - { - if (i == response_idx || !var_idx_mask->data.ptr[i]) continue; - *var_types_out_ptr = var_types->data.ptr[i]; - var_types_out_ptr++; - } - if ( response_idx >= 0 ) - *var_types_out_ptr = var_types->data.ptr[response_idx]; - - __END__; - - return var_types_out; -} - -int CvMLData::get_var_type( int var_idx ) const -{ - return var_types->data.ptr[var_idx]; -} - -const CvMat* CvMLData::get_responses() -{ - CV_FUNCNAME( "CvMLData::get_responses_ptr" ); - __BEGIN__; - - int var_count = 0; - - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - var_count = values->cols; - - if ( response_idx < 0 || response_idx >= var_count ) - return 0; - if ( !response_out ) - response_out = cvCreateMatHeader( values->rows, 1, CV_32FC1 ); - else - cvInitMatHeader( response_out, values->rows, 1, CV_32FC1); - cvGetCol( values, response_out, response_idx ); - - __END__; - - return response_out; -} - -void CvMLData::set_train_test_split( const CvTrainTestSplit * spl) -{ - CV_FUNCNAME( "CvMLData::set_division" ); - __BEGIN__; - - int sample_count = 0; + if( samples.empty() ) + return samples; + + if( (!compressSamples || (trainSampleIdx.empty() && sampleIdx.empty())) && + (!compressVars || varIdx.empty()) && + layout == _layout ) + return samples; + + int drows = getNTrainSamples(), dcols = getNVars(); + Mat sidx = getTrainSampleIdx(), vidx = getVarIdx(); + const float* src0 = samples.ptr(); + const int* sptr = !sidx.empty() ? sidx.ptr() : 0; + const int* vptr = !vidx.empty() ? vidx.ptr() : 0; + size_t sstep0 = samples.step/samples.elemSize(); + size_t sstep = layout == ROW_SAMPLE ? sstep0 : 1; + size_t vstep = layout == ROW_SAMPLE ? 1 : sstep0; + + if( _layout == COL_SAMPLE ) + { + std::swap(drows, dcols); + std::swap(sptr, vptr); + std::swap(sstep, vstep); + } - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); + Mat dsamples(drows, dcols, CV_32F); - sample_count = values->rows; + for( int i = 0; i < drows; i++ ) + { + const float* src = src0 + (sptr ? sptr[i] : i)*sstep; + float* dst = dsamples.ptr(i); - float train_sample_portion; + for( int j = 0; j < dcols; j++ ) + dst[j] = src[(vptr ? vptr[j] : j)*vstep]; + } - if (spl->train_sample_part_mode == CV_COUNT) - { - train_sample_count = spl->train_sample_part.count; - if (train_sample_count > sample_count) - CV_ERROR( CV_StsBadArg, "train samples count is not correct" ); - train_sample_count = train_sample_count<=0 ? sample_count : train_sample_count; - } - else // dtype.train_sample_part_mode == CV_PORTION - { - train_sample_portion = spl->train_sample_part.portion; - if ( train_sample_portion > 1) - CV_ERROR( CV_StsBadArg, "train samples count is not correct" ); - train_sample_portion = train_sample_portion <= FLT_EPSILON || - 1 - train_sample_portion <= FLT_EPSILON ? 1 : train_sample_portion; - train_sample_count = std::max(1, cvFloor( train_sample_portion * sample_count )); + return dsamples; } - if ( train_sample_count == sample_count ) + void getValues( int vi, InputArray _sidx, float* values ) const { - free_train_test_idx(); - return; + Mat sidx = _sidx.getMat(); + int i, n, nsamples = getNSamples(); + CV_Assert( 0 <= vi && vi < getNAllVars() ); + CV_Assert( (n = sidx.checkVector(1, CV_32S)) >= 0 ); + const int* s = n > 0 ? sidx.ptr() : 0; + if( n == 0 ) + n = nsamples; + + size_t step = samples.step/samples.elemSize(); + size_t sstep = layout == ROW_SAMPLE ? step : 1; + size_t vstep = layout == ROW_SAMPLE ? 1 : step; + + const float* src = samples.ptr() + vi*vstep; + float subst = missingSubst.at(vi); + for( i = 0; i < n; i++ ) + { + int j = i; + if( s ) + { + j = s[i]; + CV_Assert( 0 <= j && j < nsamples ); + } + values[i] = src[j*sstep]; + if( values[i] == MISSED_VAL ) + values[i] = subst; + } } - if ( train_sample_idx && train_sample_idx->cols != train_sample_count ) - free_train_test_idx(); - - if ( !sample_idx) + void getNormCatValues( int vi, InputArray _sidx, int* values ) const { - int test_sample_count = sample_count- train_sample_count; - sample_idx = (int*)cvAlloc( sample_count * sizeof(sample_idx[0]) ); - for (int i = 0; i < sample_count; i++ ) - sample_idx[i] = i; - train_sample_idx = cvCreateMatHeader( 1, train_sample_count, CV_32SC1 ); - *train_sample_idx = cvMat( 1, train_sample_count, CV_32SC1, &sample_idx[0] ); - - CV_Assert(test_sample_count > 0); - test_sample_idx = cvCreateMatHeader( 1, test_sample_count, CV_32SC1 ); - *test_sample_idx = cvMat( 1, test_sample_count, CV_32SC1, &sample_idx[train_sample_count] ); - } - - mix = spl->mix; - if ( mix ) - mix_train_and_test_idx(); - - __END__; -} - -const CvMat* CvMLData::get_train_sample_idx() const -{ - CV_FUNCNAME( "CvMLData::get_train_sample_idx" ); - __BEGIN__; - - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - __END__; - - return train_sample_idx; -} + float* fvalues = (float*)values; + getValues(vi, _sidx, fvalues); + int i, n = (int)_sidx.total(); + Vec2i ofs = catOfs.at(vi); + int m = ofs[1] - ofs[0]; -const CvMat* CvMLData::get_test_sample_idx() const -{ - CV_FUNCNAME( "CvMLData::get_test_sample_idx" ); - __BEGIN__; - - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - __END__; - - return test_sample_idx; -} - -void CvMLData::mix_train_and_test_idx() -{ - CV_FUNCNAME( "CvMLData::mix_train_and_test_idx" ); - __BEGIN__; + CV_Assert( m > 0 ); // if m==0, vi is an ordered variable + const int* cmap = &catMap.at(ofs[0]); + bool fastMap = (m == cmap[m] - cmap[0]); - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - __END__; - - if ( !sample_idx) - return; - - if ( train_sample_count > 0 && train_sample_count < values->rows ) - { - int n = values->rows; - for (int i = 0; i < n; i++) + if( fastMap ) { - int a = (*rng)(n); - int b = (*rng)(n); - int t; - CV_SWAP( sample_idx[a], sample_idx[b], t ); + for( i = 0; i < n; i++ ) + { + int val = cvRound(fvalues[i]); + int idx = val - cmap[0]; + CV_Assert(cmap[idx] == val); + values[i] = idx; + } } - } -} - -const CvMat* CvMLData::get_var_idx() -{ - CV_FUNCNAME( "CvMLData::get_var_idx" ); - __BEGIN__; - - int avcount = 0; - - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - - assert( var_idx_mask ); - - avcount = cvFloor( cvNorm( var_idx_mask, 0, CV_L1 ) ); - int* vidx; + else + { + for( i = 0; i < n; i++ ) + { + int val = cvRound(fvalues[i]); + int a = 0, b = m, c = -1; - if ( avcount == values->cols ) - return 0; + while( a < b ) + { + c = (a + b) >> 1; + if( val < cmap[c] ) + b = c; + else if( val > cmap[c] ) + a = c+1; + else + break; + } - if ( !var_idx_out || ( var_idx_out && var_idx_out->cols != avcount ) ) - { - cvReleaseMat( &var_idx_out ); - var_idx_out = cvCreateMat( 1, avcount, CV_32SC1); - if ( response_idx >=0 ) - var_idx_mask->data.ptr[response_idx] = 0; + CV_DbgAssert( c >= 0 && val == cmap[c] ); + values[i] = c; + } + } } - vidx = var_idx_out->data.i; - - for(int i = 0; i < var_idx_mask->cols; i++) - if ( var_idx_mask->data.ptr[i] ) + void getSample(InputArray _vidx, int sidx, float* buf) const + { + CV_Assert(buf != 0 && 0 <= sidx && sidx < getNSamples()); + Mat vidx = _vidx.getMat(); + int i, n, nvars = getNAllVars(); + CV_Assert( (n = vidx.checkVector(1, CV_32S)) >= 0 ); + const int* vptr = n > 0 ? vidx.ptr() : 0; + if( n == 0 ) + n = nvars; + + size_t step = samples.step/samples.elemSize(); + size_t sstep = layout == ROW_SAMPLE ? step : 1; + size_t vstep = layout == ROW_SAMPLE ? 1 : step; + + const float* src = samples.ptr() + sidx*sstep; + for( i = 0; i < n; i++ ) { - *vidx = i; - vidx++; + int j = i; + if( vptr ) + { + j = vptr[i]; + CV_Assert( 0 <= j && j < nvars ); + } + buf[i] = src[j*vstep]; } + } - __END__; - - return var_idx_out; -} + FILE* file; + int layout; + Mat samples, missing, varType, varIdx, responses, missingSubst; + Mat sampleIdx, trainSampleIdx, testSampleIdx; + Mat sampleWeights, catMap, catOfs; + Mat normCatResponses, classLabels, classCounters; + MapType nameMap; +}; -void CvMLData::chahge_var_idx( int vi, bool state ) +Ptr TrainData::loadFromCSV(const String& filename, + int headerLines, + int responseStartIdx, + int responseEndIdx, + const String& varTypeSpec, + char delimiter, char missch) { - change_var_idx( vi, state ); + Ptr td = makePtr(); + if(!td->loadCSV(filename, headerLines, responseStartIdx, responseEndIdx, varTypeSpec, delimiter, missch)) + td.release(); + return td; } -void CvMLData::change_var_idx( int vi, bool state ) +Ptr TrainData::create(InputArray samples, int layout, InputArray responses, + InputArray varIdx, InputArray sampleIdx, InputArray sampleWeights, + InputArray varType) { - CV_FUNCNAME( "CvMLData::change_var_idx" ); - __BEGIN__; - - int var_count = 0; - - if ( !values ) - CV_ERROR( CV_StsInternal, "data is empty" ); - - var_count = values->cols; - - if ( vi < 0 || vi >= var_count) - CV_ERROR( CV_StsBadArg, "variable index is not correct" ); - - assert( var_idx_mask ); - var_idx_mask->data.ptr[vi] = state; - - __END__; + Ptr td = makePtr(); + td->setData(samples, layout, responses, varIdx, sampleIdx, sampleWeights, varType, noArray()); + return td; } +}} + /* End of file. */ diff --git a/modules/ml/src/em.cpp b/modules/ml/src/em.cpp index 0bd44f27206efc743d0c795a59f12e35260f881e..351ca39fc77deee0fdd3f87b6b0933e22ee22747 100644 --- a/modules/ml/src/em.cpp +++ b/modules/ml/src/em.cpp @@ -43,635 +43,839 @@ namespace cv { +namespace ml +{ const double minEigenValue = DBL_EPSILON; -/////////////////////////////////////////////////////////////////////////////////////////////////////// - -EM::EM(int _nclusters, int _covMatType, const TermCriteria& _termCrit) +EM::Params::Params(int _nclusters, int _covMatType, const TermCriteria& _termCrit) { nclusters = _nclusters; covMatType = _covMatType; - maxIters = (_termCrit.type & TermCriteria::MAX_ITER) ? _termCrit.maxCount : DEFAULT_MAX_ITERS; - epsilon = (_termCrit.type & TermCriteria::EPS) ? _termCrit.epsilon : 0; + termCrit = _termCrit; } -EM::~EM() +class CV_EXPORTS EMImpl : public EM { - //clear(); -} +public: + EMImpl(const Params& _params) + { + setParams(_params); + } -void EM::clear() -{ - trainSamples.release(); - trainProbs.release(); - trainLogLikelihoods.release(); - trainLabels.release(); + virtual ~EMImpl() {} + + void setParams(const Params& _params) + { + params = _params; + CV_Assert(params.nclusters > 1); + CV_Assert(params.covMatType == COV_MAT_SPHERICAL || + params.covMatType == COV_MAT_DIAGONAL || + params.covMatType == COV_MAT_GENERIC); + } + + Params getParams() const + { + return params; + } - weights.release(); - means.release(); - covs.clear(); + void clear() + { + trainSamples.release(); + trainProbs.release(); + trainLogLikelihoods.release(); + trainLabels.release(); - covsEigenValues.clear(); - invCovsEigenValues.clear(); - covsRotateMats.clear(); + weights.release(); + means.release(); + covs.clear(); - logWeightDivDet.release(); -} + covsEigenValues.clear(); + invCovsEigenValues.clear(); + covsRotateMats.clear(); + logWeightDivDet.release(); + } -bool EM::train(InputArray samples, + bool train(const Ptr& data, int) + { + Mat samples = data->getTrainSamples(), labels; + return train_(samples, labels, noArray(), noArray()); + } + + bool train_(InputArray samples, OutputArray logLikelihoods, OutputArray labels, OutputArray probs) -{ - Mat samplesMat = samples.getMat(); - setTrainData(START_AUTO_STEP, samplesMat, 0, 0, 0, 0); - return doTrain(START_AUTO_STEP, logLikelihoods, labels, probs); -} + { + Mat samplesMat = samples.getMat(); + setTrainData(START_AUTO_STEP, samplesMat, 0, 0, 0, 0); + return doTrain(START_AUTO_STEP, logLikelihoods, labels, probs); + } -bool EM::trainE(InputArray samples, + bool trainE(InputArray samples, InputArray _means0, InputArray _covs0, InputArray _weights0, OutputArray logLikelihoods, OutputArray labels, OutputArray probs) -{ - Mat samplesMat = samples.getMat(); - std::vector covs0; - _covs0.getMatVector(covs0); + { + Mat samplesMat = samples.getMat(); + std::vector covs0; + _covs0.getMatVector(covs0); - Mat means0 = _means0.getMat(), weights0 = _weights0.getMat(); + Mat means0 = _means0.getMat(), weights0 = _weights0.getMat(); - setTrainData(START_E_STEP, samplesMat, 0, !_means0.empty() ? &means0 : 0, - !_covs0.empty() ? &covs0 : 0, !_weights0.empty() ? &weights0 : 0); - return doTrain(START_E_STEP, logLikelihoods, labels, probs); -} + setTrainData(START_E_STEP, samplesMat, 0, !_means0.empty() ? &means0 : 0, + !_covs0.empty() ? &covs0 : 0, !_weights0.empty() ? &weights0 : 0); + return doTrain(START_E_STEP, logLikelihoods, labels, probs); + } -bool EM::trainM(InputArray samples, + bool trainM(InputArray samples, InputArray _probs0, OutputArray logLikelihoods, OutputArray labels, OutputArray probs) -{ - Mat samplesMat = samples.getMat(); - Mat probs0 = _probs0.getMat(); - - setTrainData(START_M_STEP, samplesMat, !_probs0.empty() ? &probs0 : 0, 0, 0, 0); - return doTrain(START_M_STEP, logLikelihoods, labels, probs); -} - - -Vec2d EM::predict(InputArray _sample, OutputArray _probs) const -{ - Mat sample = _sample.getMat(); - CV_Assert(isTrained()); - - CV_Assert(!sample.empty()); - if(sample.type() != CV_64FC1) { - Mat tmp; - sample.convertTo(tmp, CV_64FC1); - sample = tmp; - } - sample = sample.reshape(1, 1); + Mat samplesMat = samples.getMat(); + Mat probs0 = _probs0.getMat(); - Mat probs; - if( _probs.needed() ) - { - _probs.create(1, nclusters, CV_64FC1); - probs = _probs.getMat(); + setTrainData(START_M_STEP, samplesMat, !_probs0.empty() ? &probs0 : 0, 0, 0, 0); + return doTrain(START_M_STEP, logLikelihoods, labels, probs); } - return computeProbabilities(sample, !probs.empty() ? &probs : 0); -} - -bool EM::isTrained() const -{ - return !means.empty(); -} + float predict(InputArray _inputs, OutputArray _outputs, int) const + { + bool needprobs = _outputs.needed(); + Mat samples = _inputs.getMat(), probs, probsrow; + int ptype = CV_32F; + float firstres = 0.f; + int i, nsamples = samples.rows; + if( needprobs ) + { + if( _outputs.fixedType() ) + ptype = _outputs.type(); + _outputs.create(samples.rows, params.nclusters, ptype); + } + else + nsamples = std::min(nsamples, 1); -static -void checkTrainData(int startStep, const Mat& samples, - int nclusters, int covMatType, const Mat* probs, const Mat* means, - const std::vector* covs, const Mat* weights) -{ - // Check samples. - CV_Assert(!samples.empty()); - CV_Assert(samples.channels() == 1); - - int nsamples = samples.rows; - int dim = samples.cols; - - // Check training params. - CV_Assert(nclusters > 0); - CV_Assert(nclusters <= nsamples); - CV_Assert(startStep == EM::START_AUTO_STEP || - startStep == EM::START_E_STEP || - startStep == EM::START_M_STEP); - CV_Assert(covMatType == EM::COV_MAT_GENERIC || - covMatType == EM::COV_MAT_DIAGONAL || - covMatType == EM::COV_MAT_SPHERICAL); - - CV_Assert(!probs || - (!probs->empty() && - probs->rows == nsamples && probs->cols == nclusters && - (probs->type() == CV_32FC1 || probs->type() == CV_64FC1))); - - CV_Assert(!weights || - (!weights->empty() && - (weights->cols == 1 || weights->rows == 1) && static_cast(weights->total()) == nclusters && - (weights->type() == CV_32FC1 || weights->type() == CV_64FC1))); - - CV_Assert(!means || - (!means->empty() && - means->rows == nclusters && means->cols == dim && - means->channels() == 1)); - - CV_Assert(!covs || - (!covs->empty() && - static_cast(covs->size()) == nclusters)); - if(covs) - { - const Size covSize(dim, dim); - for(size_t i = 0; i < covs->size(); i++) + for( i = 0; i < nsamples; i++ ) { - const Mat& m = (*covs)[i]; - CV_Assert(!m.empty() && m.size() == covSize && (m.channels() == 1)); + if( needprobs ) + probsrow = probs.row(i); + Vec2d res = computeProbabilities(samples.row(i), needprobs ? &probsrow : 0, ptype); + if( i == 0 ) + firstres = (float)res[1]; } + return firstres; } - if(startStep == EM::START_E_STEP) - { - CV_Assert(means); - } - else if(startStep == EM::START_M_STEP) + Vec2d predict2(InputArray _sample, OutputArray _probs) const { - CV_Assert(probs); - } -} - -static -void preprocessSampleData(const Mat& src, Mat& dst, int dstType, bool isAlwaysClone) -{ - if(src.type() == dstType && !isAlwaysClone) - dst = src; - else - src.convertTo(dst, dstType); -} + int ptype = CV_32F; + Mat sample = _sample.getMat(); + CV_Assert(isTrained()); -static -void preprocessProbability(Mat& probs) -{ - max(probs, 0., probs); + CV_Assert(!sample.empty()); + if(sample.type() != CV_64FC1) + { + Mat tmp; + sample.convertTo(tmp, CV_64FC1); + sample = tmp; + } + sample.reshape(1, 1); - const double uniformProbability = (double)(1./probs.cols); - for(int y = 0; y < probs.rows; y++) - { - Mat sampleProbs = probs.row(y); + Mat probs; + if( _probs.needed() ) + { + if( _probs.fixedType() ) + ptype = _probs.type(); + _probs.create(1, params.nclusters, ptype); + probs = _probs.getMat(); + } - double maxVal = 0; - minMaxLoc(sampleProbs, 0, &maxVal); - if(maxVal < FLT_EPSILON) - sampleProbs.setTo(uniformProbability); - else - normalize(sampleProbs, sampleProbs, 1, 0, NORM_L1); + return computeProbabilities(sample, !probs.empty() ? &probs : 0, ptype); } -} -void EM::setTrainData(int startStep, const Mat& samples, - const Mat* probs0, - const Mat* means0, - const std::vector* covs0, - const Mat* weights0) -{ - clear(); - - checkTrainData(startStep, samples, nclusters, covMatType, probs0, means0, covs0, weights0); - - bool isKMeansInit = (startStep == EM::START_AUTO_STEP) || (startStep == EM::START_E_STEP && (covs0 == 0 || weights0 == 0)); - // Set checked data - preprocessSampleData(samples, trainSamples, isKMeansInit ? CV_32FC1 : CV_64FC1, false); - - // set probs - if(probs0 && startStep == EM::START_M_STEP) + bool isTrained() const { - preprocessSampleData(*probs0, trainProbs, CV_64FC1, true); - preprocessProbability(trainProbs); + return !means.empty(); } - // set weights - if(weights0 && (startStep == EM::START_E_STEP && covs0)) + bool isClassifier() const { - weights0->convertTo(weights, CV_64FC1); - weights = weights.reshape(1,1); - preprocessProbability(weights); + return true; } - // set means - if(means0 && (startStep == EM::START_E_STEP/* || startStep == EM::START_AUTO_STEP*/)) - means0->convertTo(means, isKMeansInit ? CV_32FC1 : CV_64FC1); - - // set covs - if(covs0 && (startStep == EM::START_E_STEP && weights0)) + int getVarCount() const { - covs.resize(nclusters); - for(size_t i = 0; i < covs0->size(); i++) - (*covs0)[i].convertTo(covs[i], CV_64FC1); + return means.cols; } -} -void EM::decomposeCovs() -{ - CV_Assert(!covs.empty()); - covsEigenValues.resize(nclusters); - if(covMatType == EM::COV_MAT_GENERIC) - covsRotateMats.resize(nclusters); - invCovsEigenValues.resize(nclusters); - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + String getDefaultModelName() const { - CV_Assert(!covs[clusterIndex].empty()); - - SVD svd(covs[clusterIndex], SVD::MODIFY_A + SVD::FULL_UV); + return "opencv_ml_em"; + } - if(covMatType == EM::COV_MAT_SPHERICAL) + static void checkTrainData(int startStep, const Mat& samples, + int nclusters, int covMatType, const Mat* probs, const Mat* means, + const std::vector* covs, const Mat* weights) + { + // Check samples. + CV_Assert(!samples.empty()); + CV_Assert(samples.channels() == 1); + + int nsamples = samples.rows; + int dim = samples.cols; + + // Check training params. + CV_Assert(nclusters > 0); + CV_Assert(nclusters <= nsamples); + CV_Assert(startStep == START_AUTO_STEP || + startStep == START_E_STEP || + startStep == START_M_STEP); + CV_Assert(covMatType == COV_MAT_GENERIC || + covMatType == COV_MAT_DIAGONAL || + covMatType == COV_MAT_SPHERICAL); + + CV_Assert(!probs || + (!probs->empty() && + probs->rows == nsamples && probs->cols == nclusters && + (probs->type() == CV_32FC1 || probs->type() == CV_64FC1))); + + CV_Assert(!weights || + (!weights->empty() && + (weights->cols == 1 || weights->rows == 1) && static_cast(weights->total()) == nclusters && + (weights->type() == CV_32FC1 || weights->type() == CV_64FC1))); + + CV_Assert(!means || + (!means->empty() && + means->rows == nclusters && means->cols == dim && + means->channels() == 1)); + + CV_Assert(!covs || + (!covs->empty() && + static_cast(covs->size()) == nclusters)); + if(covs) { - double maxSingularVal = svd.w.at(0); - covsEigenValues[clusterIndex] = Mat(1, 1, CV_64FC1, Scalar(maxSingularVal)); + const Size covSize(dim, dim); + for(size_t i = 0; i < covs->size(); i++) + { + const Mat& m = (*covs)[i]; + CV_Assert(!m.empty() && m.size() == covSize && (m.channels() == 1)); + } } - else if(covMatType == EM::COV_MAT_DIAGONAL) + + if(startStep == START_E_STEP) { - covsEigenValues[clusterIndex] = svd.w; + CV_Assert(means); } - else //EM::COV_MAT_GENERIC + else if(startStep == START_M_STEP) { - covsEigenValues[clusterIndex] = svd.w; - covsRotateMats[clusterIndex] = svd.u; + CV_Assert(probs); } - max(covsEigenValues[clusterIndex], minEigenValue, covsEigenValues[clusterIndex]); - invCovsEigenValues[clusterIndex] = 1./covsEigenValues[clusterIndex]; } -} - -void EM::clusterTrainSamples() -{ - int nsamples = trainSamples.rows; - - // Cluster samples, compute/update means - // Convert samples and means to 32F, because kmeans requires this type. - Mat trainSamplesFlt, meansFlt; - if(trainSamples.type() != CV_32FC1) - trainSamples.convertTo(trainSamplesFlt, CV_32FC1); - else - trainSamplesFlt = trainSamples; - if(!means.empty()) + static void preprocessSampleData(const Mat& src, Mat& dst, int dstType, bool isAlwaysClone) { - if(means.type() != CV_32FC1) - means.convertTo(meansFlt, CV_32FC1); + if(src.type() == dstType && !isAlwaysClone) + dst = src; else - meansFlt = means; + src.convertTo(dst, dstType); } - Mat labels; - kmeans(trainSamplesFlt, nclusters, labels, TermCriteria(TermCriteria::COUNT, means.empty() ? 10 : 1, 0.5), 10, KMEANS_PP_CENTERS, meansFlt); - - // Convert samples and means back to 64F. - CV_Assert(meansFlt.type() == CV_32FC1); - if(trainSamples.type() != CV_64FC1) + static void preprocessProbability(Mat& probs) { - Mat trainSamplesBuffer; - trainSamplesFlt.convertTo(trainSamplesBuffer, CV_64FC1); - trainSamples = trainSamplesBuffer; - } - meansFlt.convertTo(means, CV_64FC1); + max(probs, 0., probs); - // Compute weights and covs - weights = Mat(1, nclusters, CV_64FC1, Scalar(0)); - covs.resize(nclusters); - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) - { - Mat clusterSamples; - for(int sampleIndex = 0; sampleIndex < nsamples; sampleIndex++) + const double uniformProbability = (double)(1./probs.cols); + for(int y = 0; y < probs.rows; y++) { - if(labels.at(sampleIndex) == clusterIndex) - { - const Mat sample = trainSamples.row(sampleIndex); - clusterSamples.push_back(sample); - } - } - CV_Assert(!clusterSamples.empty()); + Mat sampleProbs = probs.row(y); - calcCovarMatrix(clusterSamples, covs[clusterIndex], means.row(clusterIndex), - CV_COVAR_NORMAL + CV_COVAR_ROWS + CV_COVAR_USE_AVG + CV_COVAR_SCALE, CV_64FC1); - weights.at(clusterIndex) = static_cast(clusterSamples.rows)/static_cast(nsamples); + double maxVal = 0; + minMaxLoc(sampleProbs, 0, &maxVal); + if(maxVal < FLT_EPSILON) + sampleProbs.setTo(uniformProbability); + else + normalize(sampleProbs, sampleProbs, 1, 0, NORM_L1); + } } - decomposeCovs(); -} + void setTrainData(int startStep, const Mat& samples, + const Mat* probs0, + const Mat* means0, + const std::vector* covs0, + const Mat* weights0) + { + int nclusters = params.nclusters, covMatType = params.covMatType; + clear(); -void EM::computeLogWeightDivDet() -{ - CV_Assert(!covsEigenValues.empty()); + checkTrainData(startStep, samples, nclusters, covMatType, probs0, means0, covs0, weights0); - Mat logWeights; - cv::max(weights, DBL_MIN, weights); - log(weights, logWeights); + bool isKMeansInit = (startStep == START_AUTO_STEP) || (startStep == START_E_STEP && (covs0 == 0 || weights0 == 0)); + // Set checked data + preprocessSampleData(samples, trainSamples, isKMeansInit ? CV_32FC1 : CV_64FC1, false); - logWeightDivDet.create(1, nclusters, CV_64FC1); - // note: logWeightDivDet = log(weight_k) - 0.5 * log(|det(cov_k)|) + // set probs + if(probs0 && startStep == START_M_STEP) + { + preprocessSampleData(*probs0, trainProbs, CV_64FC1, true); + preprocessProbability(trainProbs); + } - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) - { - double logDetCov = 0.; - const int evalCount = static_cast(covsEigenValues[clusterIndex].total()); - for(int di = 0; di < evalCount; di++) - logDetCov += std::log(covsEigenValues[clusterIndex].at(covMatType != EM::COV_MAT_SPHERICAL ? di : 0)); + // set weights + if(weights0 && (startStep == START_E_STEP && covs0)) + { + weights0->convertTo(weights, CV_64FC1); + weights.reshape(1,1); + preprocessProbability(weights); + } - logWeightDivDet.at(clusterIndex) = logWeights.at(clusterIndex) - 0.5 * logDetCov; + // set means + if(means0 && (startStep == START_E_STEP/* || startStep == START_AUTO_STEP*/)) + means0->convertTo(means, isKMeansInit ? CV_32FC1 : CV_64FC1); + + // set covs + if(covs0 && (startStep == START_E_STEP && weights0)) + { + covs.resize(nclusters); + for(size_t i = 0; i < covs0->size(); i++) + (*covs0)[i].convertTo(covs[i], CV_64FC1); + } } -} -bool EM::doTrain(int startStep, OutputArray logLikelihoods, OutputArray labels, OutputArray probs) -{ - int dim = trainSamples.cols; - // Precompute the empty initial train data in the cases of EM::START_E_STEP and START_AUTO_STEP - if(startStep != EM::START_M_STEP) + void decomposeCovs() { - if(covs.empty()) + int nclusters = params.nclusters, covMatType = params.covMatType; + CV_Assert(!covs.empty()); + covsEigenValues.resize(nclusters); + if(covMatType == COV_MAT_GENERIC) + covsRotateMats.resize(nclusters); + invCovsEigenValues.resize(nclusters); + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) { - CV_Assert(weights.empty()); - clusterTrainSamples(); + CV_Assert(!covs[clusterIndex].empty()); + + SVD svd(covs[clusterIndex], SVD::MODIFY_A + SVD::FULL_UV); + + if(covMatType == COV_MAT_SPHERICAL) + { + double maxSingularVal = svd.w.at(0); + covsEigenValues[clusterIndex] = Mat(1, 1, CV_64FC1, Scalar(maxSingularVal)); + } + else if(covMatType == COV_MAT_DIAGONAL) + { + covsEigenValues[clusterIndex] = svd.w; + } + else //COV_MAT_GENERIC + { + covsEigenValues[clusterIndex] = svd.w; + covsRotateMats[clusterIndex] = svd.u; + } + max(covsEigenValues[clusterIndex], minEigenValue, covsEigenValues[clusterIndex]); + invCovsEigenValues[clusterIndex] = 1./covsEigenValues[clusterIndex]; } } - if(!covs.empty() && covsEigenValues.empty() ) + void clusterTrainSamples() { - CV_Assert(invCovsEigenValues.empty()); - decomposeCovs(); - } + int nclusters = params.nclusters; + int nsamples = trainSamples.rows; - if(startStep == EM::START_M_STEP) - mStep(); + // Cluster samples, compute/update means - double trainLogLikelihood, prevTrainLogLikelihood = 0.; - for(int iter = 0; ; iter++) - { - eStep(); - trainLogLikelihood = sum(trainLogLikelihoods)[0]; + // Convert samples and means to 32F, because kmeans requires this type. + Mat trainSamplesFlt, meansFlt; + if(trainSamples.type() != CV_32FC1) + trainSamples.convertTo(trainSamplesFlt, CV_32FC1); + else + trainSamplesFlt = trainSamples; + if(!means.empty()) + { + if(means.type() != CV_32FC1) + means.convertTo(meansFlt, CV_32FC1); + else + meansFlt = means; + } + + Mat labels; + kmeans(trainSamplesFlt, nclusters, labels, + TermCriteria(TermCriteria::COUNT, means.empty() ? 10 : 1, 0.5), + 10, KMEANS_PP_CENTERS, meansFlt); - if(iter >= maxIters - 1) - break; + // Convert samples and means back to 64F. + CV_Assert(meansFlt.type() == CV_32FC1); + if(trainSamples.type() != CV_64FC1) + { + Mat trainSamplesBuffer; + trainSamplesFlt.convertTo(trainSamplesBuffer, CV_64FC1); + trainSamples = trainSamplesBuffer; + } + meansFlt.convertTo(means, CV_64FC1); - double trainLogLikelihoodDelta = trainLogLikelihood - prevTrainLogLikelihood; - if( iter != 0 && - (trainLogLikelihoodDelta < -DBL_EPSILON || - trainLogLikelihoodDelta < epsilon * std::fabs(trainLogLikelihood))) - break; + // Compute weights and covs + weights = Mat(1, nclusters, CV_64FC1, Scalar(0)); + covs.resize(nclusters); + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + { + Mat clusterSamples; + for(int sampleIndex = 0; sampleIndex < nsamples; sampleIndex++) + { + if(labels.at(sampleIndex) == clusterIndex) + { + const Mat sample = trainSamples.row(sampleIndex); + clusterSamples.push_back(sample); + } + } + CV_Assert(!clusterSamples.empty()); - mStep(); + calcCovarMatrix(clusterSamples, covs[clusterIndex], means.row(clusterIndex), + CV_COVAR_NORMAL + CV_COVAR_ROWS + CV_COVAR_USE_AVG + CV_COVAR_SCALE, CV_64FC1); + weights.at(clusterIndex) = static_cast(clusterSamples.rows)/static_cast(nsamples); + } - prevTrainLogLikelihood = trainLogLikelihood; + decomposeCovs(); } - if( trainLogLikelihood <= -DBL_MAX/10000. ) + void computeLogWeightDivDet() { - clear(); - return false; + int nclusters = params.nclusters; + CV_Assert(!covsEigenValues.empty()); + + Mat logWeights; + cv::max(weights, DBL_MIN, weights); + log(weights, logWeights); + + logWeightDivDet.create(1, nclusters, CV_64FC1); + // note: logWeightDivDet = log(weight_k) - 0.5 * log(|det(cov_k)|) + + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + { + double logDetCov = 0.; + const int evalCount = static_cast(covsEigenValues[clusterIndex].total()); + for(int di = 0; di < evalCount; di++) + logDetCov += std::log(covsEigenValues[clusterIndex].at(params.covMatType != COV_MAT_SPHERICAL ? di : 0)); + + logWeightDivDet.at(clusterIndex) = logWeights.at(clusterIndex) - 0.5 * logDetCov; + } } - // postprocess covs - covs.resize(nclusters); - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + bool doTrain(int startStep, OutputArray logLikelihoods, OutputArray labels, OutputArray probs) { - if(covMatType == EM::COV_MAT_SPHERICAL) + int nclusters = params.nclusters; + int dim = trainSamples.cols; + // Precompute the empty initial train data in the cases of START_E_STEP and START_AUTO_STEP + if(startStep != START_M_STEP) { - covs[clusterIndex].create(dim, dim, CV_64FC1); - setIdentity(covs[clusterIndex], Scalar(covsEigenValues[clusterIndex].at(0))); + if(covs.empty()) + { + CV_Assert(weights.empty()); + clusterTrainSamples(); + } } - else if(covMatType == EM::COV_MAT_DIAGONAL) + + if(!covs.empty() && covsEigenValues.empty() ) { - covs[clusterIndex] = Mat::diag(covsEigenValues[clusterIndex]); + CV_Assert(invCovsEigenValues.empty()); + decomposeCovs(); } - } - if(labels.needed()) - trainLabels.copyTo(labels); - if(probs.needed()) - trainProbs.copyTo(probs); - if(logLikelihoods.needed()) - trainLogLikelihoods.copyTo(logLikelihoods); + if(startStep == START_M_STEP) + mStep(); - trainSamples.release(); - trainProbs.release(); - trainLabels.release(); - trainLogLikelihoods.release(); + double trainLogLikelihood, prevTrainLogLikelihood = 0.; + int maxIters = (params.termCrit.type & TermCriteria::MAX_ITER) ? + params.termCrit.maxCount : DEFAULT_MAX_ITERS; + double epsilon = (params.termCrit.type & TermCriteria::EPS) ? params.termCrit.epsilon : 0.; - return true; -} + for(int iter = 0; ; iter++) + { + eStep(); + trainLogLikelihood = sum(trainLogLikelihoods)[0]; -Vec2d EM::computeProbabilities(const Mat& sample, Mat* probs) const -{ - // L_ik = log(weight_k) - 0.5 * log(|det(cov_k)|) - 0.5 *(x_i - mean_k)' cov_k^(-1) (x_i - mean_k)] - // q = arg(max_k(L_ik)) - // probs_ik = exp(L_ik - L_iq) / (1 + sum_j!=q (exp(L_ij - L_iq)) - // see Alex Smola's blog http://blog.smola.org/page/2 for - // details on the log-sum-exp trick - - CV_Assert(!means.empty()); - CV_Assert(sample.type() == CV_64FC1); - CV_Assert(sample.rows == 1); - CV_Assert(sample.cols == means.cols); - - int dim = sample.cols; - - Mat L(1, nclusters, CV_64FC1); - int label = 0; - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) - { - const Mat centeredSample = sample - means.row(clusterIndex); + if(iter >= maxIters - 1) + break; + + double trainLogLikelihoodDelta = trainLogLikelihood - prevTrainLogLikelihood; + if( iter != 0 && + (trainLogLikelihoodDelta < -DBL_EPSILON || + trainLogLikelihoodDelta < epsilon * std::fabs(trainLogLikelihood))) + break; - Mat rotatedCenteredSample = covMatType != EM::COV_MAT_GENERIC ? - centeredSample : centeredSample * covsRotateMats[clusterIndex]; + mStep(); - double Lval = 0; - for(int di = 0; di < dim; di++) + prevTrainLogLikelihood = trainLogLikelihood; + } + + if( trainLogLikelihood <= -DBL_MAX/10000. ) { - double w = invCovsEigenValues[clusterIndex].at(covMatType != EM::COV_MAT_SPHERICAL ? di : 0); - double val = rotatedCenteredSample.at(di); - Lval += w * val * val; + clear(); + return false; } - CV_DbgAssert(!logWeightDivDet.empty()); - L.at(clusterIndex) = logWeightDivDet.at(clusterIndex) - 0.5 * Lval; - if(L.at(clusterIndex) > L.at(label)) - label = clusterIndex; - } + // postprocess covs + covs.resize(nclusters); + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + { + if(params.covMatType == COV_MAT_SPHERICAL) + { + covs[clusterIndex].create(dim, dim, CV_64FC1); + setIdentity(covs[clusterIndex], Scalar(covsEigenValues[clusterIndex].at(0))); + } + else if(params.covMatType == COV_MAT_DIAGONAL) + { + covs[clusterIndex] = Mat::diag(covsEigenValues[clusterIndex]); + } + } - double maxLVal = L.at(label); - Mat expL_Lmax = L; // exp(L_ij - L_iq) - for(int i = 0; i < L.cols; i++) - expL_Lmax.at(i) = std::exp(L.at(i) - maxLVal); - double expDiffSum = sum(expL_Lmax)[0]; // sum_j(exp(L_ij - L_iq)) + if(labels.needed()) + trainLabels.copyTo(labels); + if(probs.needed()) + trainProbs.copyTo(probs); + if(logLikelihoods.needed()) + trainLogLikelihoods.copyTo(logLikelihoods); - if(probs) - { - probs->create(1, nclusters, CV_64FC1); - double factor = 1./expDiffSum; - expL_Lmax *= factor; - expL_Lmax.copyTo(*probs); - } + trainSamples.release(); + trainProbs.release(); + trainLabels.release(); + trainLogLikelihoods.release(); - Vec2d res; - res[0] = std::log(expDiffSum) + maxLVal - 0.5 * dim * CV_LOG2PI; - res[1] = label; + return true; + } - return res; -} + Vec2d computeProbabilities(const Mat& sample, Mat* probs, int ptype) const + { + // L_ik = log(weight_k) - 0.5 * log(|det(cov_k)|) - 0.5 *(x_i - mean_k)' cov_k^(-1) (x_i - mean_k)] + // q = arg(max_k(L_ik)) + // probs_ik = exp(L_ik - L_iq) / (1 + sum_j!=q (exp(L_ij - L_iq)) + // see Alex Smola's blog http://blog.smola.org/page/2 for + // details on the log-sum-exp trick + + int nclusters = params.nclusters, covMatType = params.covMatType; + int stype = sample.type(); + CV_Assert(!means.empty()); + CV_Assert((stype == CV_32F || stype == CV_64F) && (ptype == CV_32F || ptype == CV_64F)); + CV_Assert(sample.size() == Size(means.cols, 1)); + + int dim = sample.cols; + + Mat L(1, nclusters, CV_64FC1), centeredSample(1, dim, CV_64F); + int i, label = 0; + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + { + const double* mptr = means.ptr(clusterIndex); + double* dptr = centeredSample.ptr(); + if( stype == CV_32F ) + { + const float* sptr = sample.ptr(); + for( i = 0; i < dim; i++ ) + dptr[i] = sptr[i] - mptr[i]; + } + else + { + const double* sptr = sample.ptr(); + for( i = 0; i < dim; i++ ) + dptr[i] = sptr[i] - mptr[i]; + } -void EM::eStep() -{ - // Compute probs_ik from means_k, covs_k and weights_k. - trainProbs.create(trainSamples.rows, nclusters, CV_64FC1); - trainLabels.create(trainSamples.rows, 1, CV_32SC1); - trainLogLikelihoods.create(trainSamples.rows, 1, CV_64FC1); + Mat rotatedCenteredSample = covMatType != COV_MAT_GENERIC ? + centeredSample : centeredSample * covsRotateMats[clusterIndex]; - computeLogWeightDivDet(); + double Lval = 0; + for(int di = 0; di < dim; di++) + { + double w = invCovsEigenValues[clusterIndex].at(covMatType != COV_MAT_SPHERICAL ? di : 0); + double val = rotatedCenteredSample.at(di); + Lval += w * val * val; + } + CV_DbgAssert(!logWeightDivDet.empty()); + L.at(clusterIndex) = logWeightDivDet.at(clusterIndex) - 0.5 * Lval; - CV_DbgAssert(trainSamples.type() == CV_64FC1); - CV_DbgAssert(means.type() == CV_64FC1); + if(L.at(clusterIndex) > L.at(label)) + label = clusterIndex; + } - for(int sampleIndex = 0; sampleIndex < trainSamples.rows; sampleIndex++) - { - Mat sampleProbs = trainProbs.row(sampleIndex); - Vec2d res = computeProbabilities(trainSamples.row(sampleIndex), &sampleProbs); - trainLogLikelihoods.at(sampleIndex) = res[0]; - trainLabels.at(sampleIndex) = static_cast(res[1]); - } -} + double maxLVal = L.at(label); + double expDiffSum = 0; + for( i = 0; i < L.cols; i++ ) + { + double v = std::exp(L.at(i) - maxLVal); + L.at(i) = v; + expDiffSum += v; // sum_j(exp(L_ij - L_iq)) + } -void EM::mStep() -{ - // Update means_k, covs_k and weights_k from probs_ik - int dim = trainSamples.cols; + if(probs) + L.convertTo(*probs, ptype, 1./expDiffSum); - // Update weights - // not normalized first - reduce(trainProbs, weights, 0, CV_REDUCE_SUM); + Vec2d res; + res[0] = std::log(expDiffSum) + maxLVal - 0.5 * dim * CV_LOG2PI; + res[1] = label; - // Update means - means.create(nclusters, dim, CV_64FC1); - means = Scalar(0); + return res; + } - const double minPosWeight = trainSamples.rows * DBL_EPSILON; - double minWeight = DBL_MAX; - int minWeightClusterIndex = -1; - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + void eStep() { - if(weights.at(clusterIndex) <= minPosWeight) - continue; + // Compute probs_ik from means_k, covs_k and weights_k. + trainProbs.create(trainSamples.rows, params.nclusters, CV_64FC1); + trainLabels.create(trainSamples.rows, 1, CV_32SC1); + trainLogLikelihoods.create(trainSamples.rows, 1, CV_64FC1); - if(weights.at(clusterIndex) < minWeight) - { - minWeight = weights.at(clusterIndex); - minWeightClusterIndex = clusterIndex; - } + computeLogWeightDivDet(); + + CV_DbgAssert(trainSamples.type() == CV_64FC1); + CV_DbgAssert(means.type() == CV_64FC1); - Mat clusterMean = means.row(clusterIndex); for(int sampleIndex = 0; sampleIndex < trainSamples.rows; sampleIndex++) - clusterMean += trainProbs.at(sampleIndex, clusterIndex) * trainSamples.row(sampleIndex); - clusterMean /= weights.at(clusterIndex); + { + Mat sampleProbs = trainProbs.row(sampleIndex); + Vec2d res = computeProbabilities(trainSamples.row(sampleIndex), &sampleProbs, CV_64F); + trainLogLikelihoods.at(sampleIndex) = res[0]; + trainLabels.at(sampleIndex) = static_cast(res[1]); + } } - // Update covsEigenValues and invCovsEigenValues - covs.resize(nclusters); - covsEigenValues.resize(nclusters); - if(covMatType == EM::COV_MAT_GENERIC) - covsRotateMats.resize(nclusters); - invCovsEigenValues.resize(nclusters); - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + void mStep() { - if(weights.at(clusterIndex) <= minPosWeight) - continue; - - if(covMatType != EM::COV_MAT_SPHERICAL) - covsEigenValues[clusterIndex].create(1, dim, CV_64FC1); - else - covsEigenValues[clusterIndex].create(1, 1, CV_64FC1); - - if(covMatType == EM::COV_MAT_GENERIC) - covs[clusterIndex].create(dim, dim, CV_64FC1); + // Update means_k, covs_k and weights_k from probs_ik + int nclusters = params.nclusters; + int covMatType = params.covMatType; + int dim = trainSamples.cols; + + // Update weights + // not normalized first + reduce(trainProbs, weights, 0, CV_REDUCE_SUM); + + // Update means + means.create(nclusters, dim, CV_64FC1); + means = Scalar(0); + + const double minPosWeight = trainSamples.rows * DBL_EPSILON; + double minWeight = DBL_MAX; + int minWeightClusterIndex = -1; + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + { + if(weights.at(clusterIndex) <= minPosWeight) + continue; - Mat clusterCov = covMatType != EM::COV_MAT_GENERIC ? - covsEigenValues[clusterIndex] : covs[clusterIndex]; + if(weights.at(clusterIndex) < minWeight) + { + minWeight = weights.at(clusterIndex); + minWeightClusterIndex = clusterIndex; + } - clusterCov = Scalar(0); + Mat clusterMean = means.row(clusterIndex); + for(int sampleIndex = 0; sampleIndex < trainSamples.rows; sampleIndex++) + clusterMean += trainProbs.at(sampleIndex, clusterIndex) * trainSamples.row(sampleIndex); + clusterMean /= weights.at(clusterIndex); + } - Mat centeredSample; - for(int sampleIndex = 0; sampleIndex < trainSamples.rows; sampleIndex++) + // Update covsEigenValues and invCovsEigenValues + covs.resize(nclusters); + covsEigenValues.resize(nclusters); + if(covMatType == COV_MAT_GENERIC) + covsRotateMats.resize(nclusters); + invCovsEigenValues.resize(nclusters); + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) { - centeredSample = trainSamples.row(sampleIndex) - means.row(clusterIndex); + if(weights.at(clusterIndex) <= minPosWeight) + continue; - if(covMatType == EM::COV_MAT_GENERIC) - clusterCov += trainProbs.at(sampleIndex, clusterIndex) * centeredSample.t() * centeredSample; + if(covMatType != COV_MAT_SPHERICAL) + covsEigenValues[clusterIndex].create(1, dim, CV_64FC1); else + covsEigenValues[clusterIndex].create(1, 1, CV_64FC1); + + if(covMatType == COV_MAT_GENERIC) + covs[clusterIndex].create(dim, dim, CV_64FC1); + + Mat clusterCov = covMatType != COV_MAT_GENERIC ? + covsEigenValues[clusterIndex] : covs[clusterIndex]; + + clusterCov = Scalar(0); + + Mat centeredSample; + for(int sampleIndex = 0; sampleIndex < trainSamples.rows; sampleIndex++) { - double p = trainProbs.at(sampleIndex, clusterIndex); - for(int di = 0; di < dim; di++ ) + centeredSample = trainSamples.row(sampleIndex) - means.row(clusterIndex); + + if(covMatType == COV_MAT_GENERIC) + clusterCov += trainProbs.at(sampleIndex, clusterIndex) * centeredSample.t() * centeredSample; + else { - double val = centeredSample.at(di); - clusterCov.at(covMatType != EM::COV_MAT_SPHERICAL ? di : 0) += p*val*val; + double p = trainProbs.at(sampleIndex, clusterIndex); + for(int di = 0; di < dim; di++ ) + { + double val = centeredSample.at(di); + clusterCov.at(covMatType != COV_MAT_SPHERICAL ? di : 0) += p*val*val; + } } } - } - if(covMatType == EM::COV_MAT_SPHERICAL) - clusterCov /= dim; + if(covMatType == COV_MAT_SPHERICAL) + clusterCov /= dim; + + clusterCov /= weights.at(clusterIndex); + + // Update covsRotateMats for COV_MAT_GENERIC only + if(covMatType == COV_MAT_GENERIC) + { + SVD svd(covs[clusterIndex], SVD::MODIFY_A + SVD::FULL_UV); + covsEigenValues[clusterIndex] = svd.w; + covsRotateMats[clusterIndex] = svd.u; + } - clusterCov /= weights.at(clusterIndex); + max(covsEigenValues[clusterIndex], minEigenValue, covsEigenValues[clusterIndex]); - // Update covsRotateMats for EM::COV_MAT_GENERIC only - if(covMatType == EM::COV_MAT_GENERIC) + // update invCovsEigenValues + invCovsEigenValues[clusterIndex] = 1./covsEigenValues[clusterIndex]; + } + + for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) { - SVD svd(covs[clusterIndex], SVD::MODIFY_A + SVD::FULL_UV); - covsEigenValues[clusterIndex] = svd.w; - covsRotateMats[clusterIndex] = svd.u; + if(weights.at(clusterIndex) <= minPosWeight) + { + Mat clusterMean = means.row(clusterIndex); + means.row(minWeightClusterIndex).copyTo(clusterMean); + covs[minWeightClusterIndex].copyTo(covs[clusterIndex]); + covsEigenValues[minWeightClusterIndex].copyTo(covsEigenValues[clusterIndex]); + if(covMatType == COV_MAT_GENERIC) + covsRotateMats[minWeightClusterIndex].copyTo(covsRotateMats[clusterIndex]); + invCovsEigenValues[minWeightClusterIndex].copyTo(invCovsEigenValues[clusterIndex]); + } } - max(covsEigenValues[clusterIndex], minEigenValue, covsEigenValues[clusterIndex]); + // Normalize weights + weights /= trainSamples.rows; + } - // update invCovsEigenValues - invCovsEigenValues[clusterIndex] = 1./covsEigenValues[clusterIndex]; + void write_params(FileStorage& fs) const + { + fs << "nclusters" << params.nclusters; + fs << "cov_mat_type" << (params.covMatType == COV_MAT_SPHERICAL ? String("spherical") : + params.covMatType == COV_MAT_DIAGONAL ? String("diagonal") : + params.covMatType == COV_MAT_GENERIC ? String("generic") : + format("unknown_%d", params.covMatType)); + writeTermCrit(fs, params.termCrit); } - for(int clusterIndex = 0; clusterIndex < nclusters; clusterIndex++) + void write(FileStorage& fs) const { - if(weights.at(clusterIndex) <= minPosWeight) - { - Mat clusterMean = means.row(clusterIndex); - means.row(minWeightClusterIndex).copyTo(clusterMean); - covs[minWeightClusterIndex].copyTo(covs[clusterIndex]); - covsEigenValues[minWeightClusterIndex].copyTo(covsEigenValues[clusterIndex]); - if(covMatType == EM::COV_MAT_GENERIC) - covsRotateMats[minWeightClusterIndex].copyTo(covsRotateMats[clusterIndex]); - invCovsEigenValues[minWeightClusterIndex].copyTo(invCovsEigenValues[clusterIndex]); - } + fs << "training_params" << "{"; + write_params(fs); + fs << "}"; + fs << "weights" << weights; + fs << "means" << means; + + size_t i, n = covs.size(); + + fs << "covs" << "["; + for( i = 0; i < n; i++ ) + fs << covs[i]; + fs << "]"; + } + + void read_params(const FileNode& fn) + { + Params _params; + _params.nclusters = (int)fn["nclusters"]; + String s = (String)fn["cov_mat_type"]; + _params.covMatType = s == "spherical" ? COV_MAT_SPHERICAL : + s == "diagonal" ? COV_MAT_DIAGONAL : + s == "generic" ? COV_MAT_GENERIC : -1; + CV_Assert(_params.covMatType >= 0); + _params.termCrit = readTermCrit(fn); + setParams(_params); + } + + void read(const FileNode& fn) + { + clear(); + read_params(fn["training_params"]); + + fn["weights"] >> weights; + fn["means"] >> means; + + FileNode cfn = fn["covs"]; + FileNodeIterator cfn_it = cfn.begin(); + int i, n = (int)cfn.size(); + covs.resize(n); + + for( i = 0; i < n; i++, ++cfn_it ) + (*cfn_it) >> covs[i]; + + decomposeCovs(); + computeLogWeightDivDet(); } - // Normalize weights - weights /= trainSamples.rows; + Mat getWeights() const { return weights; } + Mat getMeans() const { return means; } + void getCovs(std::vector& _covs) const + { + _covs.resize(covs.size()); + std::copy(covs.begin(), covs.end(), _covs.begin()); + } + + Params params; + + // all inner matrices have type CV_64FC1 + Mat trainSamples; + Mat trainProbs; + Mat trainLogLikelihoods; + Mat trainLabels; + + Mat weights; + Mat means; + std::vector covs; + + std::vector covsEigenValues; + std::vector covsRotateMats; + std::vector invCovsEigenValues; + Mat logWeightDivDet; +}; + + +Ptr EM::train(InputArray samples, OutputArray logLikelihoods, + OutputArray labels, OutputArray probs, + const EM::Params& params) +{ + Ptr em = makePtr(params); + if(!em->train_(samples, logLikelihoods, labels, probs)) + em.release(); + return em; } -void EM::read(const FileNode& fn) +Ptr EM::train_startWithE(InputArray samples, InputArray means0, + InputArray covs0, InputArray weights0, + OutputArray logLikelihoods, OutputArray labels, + OutputArray probs, const EM::Params& params) { - Algorithm::read(fn); + Ptr em = makePtr(params); + if(!em->trainE(samples, means0, covs0, weights0, logLikelihoods, labels, probs)) + em.release(); + return em; +} - decomposeCovs(); - computeLogWeightDivDet(); +Ptr EM::train_startWithM(InputArray samples, InputArray probs0, + OutputArray logLikelihoods, OutputArray labels, + OutputArray probs, const EM::Params& params) +{ + Ptr em = makePtr(params); + if(!em->trainM(samples, probs0, logLikelihoods, labels, probs)) + em.release(); + return em; } +Ptr EM::create(const Params& params) +{ + return makePtr(params); +} + +} } // namespace cv /* End of file. */ diff --git a/modules/ml/src/ertrees.cpp b/modules/ml/src/ertrees.cpp deleted file mode 100644 index 0201deb0c6b262112230caaf546f41c214f744f8..0000000000000000000000000000000000000000 --- a/modules/ml/src/ertrees.cpp +++ /dev/null @@ -1,1859 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - - IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - - By downloading, copying, installing or using the software you agree to this license. - If you do not agree to this license, do not download, install, - copy or use the software. - - - Intel License Agreement - - Copyright (C) 2000, Intel Corporation, all rights reserved. - Third party copyrights are property of their respective owners. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - * Redistribution's of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistribution's in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * The name of Intel Corporation may not be used to endorse or promote products - derived from this software without specific prior written permission. - - This software is provided by the copyright holders and contributors "as is" and - any express or implied warranties, including, but not limited to, the implied - warranties of merchantability and fitness for a particular purpose are disclaimed. - In no event shall the Intel Corporation or contributors be liable for any direct, - indirect, incidental, special, exemplary, or consequential damages - (including, but not limited to, procurement of substitute goods or services; - loss of use, data, or profits; or business interruption) however caused - and on any theory of liability, whether in contract, strict liability, - or tort (including negligence or otherwise) arising in any way out of - the use of this software, even if advised of the possibility of such damage. - -M*/ - -#include "precomp.hpp" - -static const float ord_nan = FLT_MAX*0.5f; -static const int min_block_size = 1 << 16; -static const int block_size_delta = 1 << 10; - -template -class LessThanPtr -{ -public: - bool operator()(T* a, T* b) const { return *a < *b; } -}; - -class LessThanPairs -{ -public: - bool operator()(const CvPair16u32s& a, const CvPair16u32s& b) const { return *a.i < *b.i; } -}; - -void CvERTreeTrainData::set_data( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, const CvMat* _sample_idx, - const CvMat* _var_type, const CvMat* _missing_mask, const CvDTreeParams& _params, - bool _shared, bool _add_labels, bool _update_data ) -{ - CvMat* sample_indices = 0; - CvMat* var_type0 = 0; - CvMat* tmp_map = 0; - int** int_ptr = 0; - CvPair16u32s* pair16u32s_ptr = 0; - CvDTreeTrainData* data = 0; - float *_fdst = 0; - int *_idst = 0; - unsigned short* udst = 0; - int* idst = 0; - - CV_FUNCNAME( "CvERTreeTrainData::set_data" ); - - __BEGIN__; - - int sample_all = 0, r_type, cv_n; - int total_c_count = 0; - int tree_block_size, temp_block_size, max_split_size, nv_size, cv_size = 0; - int ds_step, dv_step, ms_step = 0, mv_step = 0; // {data|mask}{sample|var}_step - int vi, i, size; - char err[100]; - const int *sidx = 0, *vidx = 0; - - uint64 effective_buf_size = 0; - int effective_buf_height = 0, effective_buf_width = 0; - - if ( _params.use_surrogates ) - CV_ERROR(CV_StsBadArg, "CvERTrees do not support surrogate splits"); - - if( _update_data && data_root ) - { - CV_ERROR(CV_StsBadArg, "CvERTrees do not support data update"); - } - - clear(); - - var_all = 0; - rng = &cv::theRNG(); - - CV_CALL( set_params( _params )); - - // check parameter types and sizes - CV_CALL( cvCheckTrainData( _train_data, _tflag, _missing_mask, &var_all, &sample_all )); - - train_data = _train_data; - responses = _responses; - missing_mask = _missing_mask; - - if( _tflag == CV_ROW_SAMPLE ) - { - ds_step = _train_data->step/CV_ELEM_SIZE(_train_data->type); - dv_step = 1; - if( _missing_mask ) - ms_step = _missing_mask->step, mv_step = 1; - } - else - { - dv_step = _train_data->step/CV_ELEM_SIZE(_train_data->type); - ds_step = 1; - if( _missing_mask ) - mv_step = _missing_mask->step, ms_step = 1; - } - tflag = _tflag; - - sample_count = sample_all; - var_count = var_all; - - if( _sample_idx ) - { - CV_CALL( sample_indices = cvPreprocessIndexArray( _sample_idx, sample_all )); - sidx = sample_indices->data.i; - sample_count = sample_indices->rows + sample_indices->cols - 1; - } - - if( _var_idx ) - { - CV_CALL( var_idx = cvPreprocessIndexArray( _var_idx, var_all )); - vidx = var_idx->data.i; - var_count = var_idx->rows + var_idx->cols - 1; - } - - if( !CV_IS_MAT(_responses) || - (CV_MAT_TYPE(_responses->type) != CV_32SC1 && - CV_MAT_TYPE(_responses->type) != CV_32FC1) || - (_responses->rows != 1 && _responses->cols != 1) || - _responses->rows + _responses->cols - 1 != sample_all ) - CV_ERROR( CV_StsBadArg, "The array of _responses must be an integer or " - "floating-point vector containing as many elements as " - "the total number of samples in the training data matrix" ); - - is_buf_16u = false; - if ( sample_count < 65536 ) - is_buf_16u = true; - - r_type = CV_VAR_CATEGORICAL; - if( _var_type ) - CV_CALL( var_type0 = cvPreprocessVarType( _var_type, var_idx, var_count, &r_type )); - - CV_CALL( var_type = cvCreateMat( 1, var_count+2, CV_32SC1 )); - - cat_var_count = 0; - ord_var_count = -1; - - is_classifier = r_type == CV_VAR_CATEGORICAL; - - // step 0. calc the number of categorical vars - for( vi = 0; vi < var_count; vi++ ) - { - char vt = var_type0 ? var_type0->data.ptr[vi] : CV_VAR_ORDERED; - var_type->data.i[vi] = vt == CV_VAR_CATEGORICAL ? cat_var_count++ : ord_var_count--; - } - - ord_var_count = ~ord_var_count; - cv_n = params.cv_folds; - // set the two last elements of var_type array to be able - // to locate responses and cross-validation labels using - // the corresponding get_* functions. - var_type->data.i[var_count] = cat_var_count; - var_type->data.i[var_count+1] = cat_var_count+1; - - // in case of single ordered predictor we need dummy cv_labels - // for safe split_node_data() operation - have_labels = cv_n > 0 || (ord_var_count == 1 && cat_var_count == 0) || _add_labels; - - work_var_count = cat_var_count + (is_classifier ? 1 : 0) + (have_labels ? 1 : 0); - - shared = _shared; - buf_count = shared ? 2 : 1; - - buf_size = -1; // the member buf_size is obsolete - - effective_buf_size = (uint64)(work_var_count + 1)*(uint64)sample_count * buf_count; // this is the total size of "CvMat buf" to be allocated - effective_buf_width = sample_count; - effective_buf_height = work_var_count+1; - - if (effective_buf_width >= effective_buf_height) - effective_buf_height *= buf_count; - else - effective_buf_width *= buf_count; - - if ((uint64)effective_buf_width * (uint64)effective_buf_height != effective_buf_size) - { - CV_Error(CV_StsBadArg, "The memory buffer cannot be allocated since its size exceeds integer fields limit"); - } - - if ( is_buf_16u ) - { - CV_CALL( buf = cvCreateMat( effective_buf_height, effective_buf_width, CV_16UC1 )); - CV_CALL( pair16u32s_ptr = (CvPair16u32s*)cvAlloc( sample_count*sizeof(pair16u32s_ptr[0]) )); - } - else - { - CV_CALL( buf = cvCreateMat( effective_buf_height, effective_buf_width, CV_32SC1 )); - CV_CALL( int_ptr = (int**)cvAlloc( sample_count*sizeof(int_ptr[0]) )); - } - - size = is_classifier ? cat_var_count+1 : cat_var_count; - size = !size ? 1 : size; - CV_CALL( cat_count = cvCreateMat( 1, size, CV_32SC1 )); - CV_CALL( cat_ofs = cvCreateMat( 1, size, CV_32SC1 )); - - size = is_classifier ? (cat_var_count + 1)*params.max_categories : cat_var_count*params.max_categories; - size = !size ? 1 : size; - CV_CALL( cat_map = cvCreateMat( 1, size, CV_32SC1 )); - - // now calculate the maximum size of split, - // create memory storage that will keep nodes and splits of the decision tree - // allocate root node and the buffer for the whole training data - max_split_size = cvAlign(sizeof(CvDTreeSplit) + - (MAX(0,sample_count - 33)/32)*sizeof(int),sizeof(void*)); - tree_block_size = MAX((int)sizeof(CvDTreeNode)*8, max_split_size); - tree_block_size = MAX(tree_block_size + block_size_delta, min_block_size); - CV_CALL( tree_storage = cvCreateMemStorage( tree_block_size )); - CV_CALL( node_heap = cvCreateSet( 0, sizeof(*node_heap), sizeof(CvDTreeNode), tree_storage )); - - nv_size = var_count*sizeof(int); - nv_size = cvAlign(MAX( nv_size, (int)sizeof(CvSetElem) ), sizeof(void*)); - - temp_block_size = nv_size; - - if( cv_n ) - { - if( sample_count < cv_n*MAX(params.min_sample_count,10) ) - CV_ERROR( CV_StsOutOfRange, - "The many folds in cross-validation for such a small dataset" ); - - cv_size = cvAlign( cv_n*(sizeof(int) + sizeof(double)*2), sizeof(double) ); - temp_block_size = MAX(temp_block_size, cv_size); - } - - temp_block_size = MAX( temp_block_size + block_size_delta, min_block_size ); - CV_CALL( temp_storage = cvCreateMemStorage( temp_block_size )); - CV_CALL( nv_heap = cvCreateSet( 0, sizeof(*nv_heap), nv_size, temp_storage )); - if( cv_size ) - CV_CALL( cv_heap = cvCreateSet( 0, sizeof(*cv_heap), cv_size, temp_storage )); - - CV_CALL( data_root = new_node( 0, sample_count, 0, 0 )); - - max_c_count = 1; - - _fdst = 0; - _idst = 0; - if (ord_var_count) - _fdst = (float*)cvAlloc(sample_count*sizeof(_fdst[0])); - if (is_buf_16u && (cat_var_count || is_classifier)) - _idst = (int*)cvAlloc(sample_count*sizeof(_idst[0])); - - // transform the training data to convenient representation - for( vi = 0; vi <= var_count; vi++ ) - { - int ci; - const uchar* mask = 0; - int m_step = 0, step; - const int* idata = 0; - const float* fdata = 0; - int num_valid = 0; - - if( vi < var_count ) // analyze i-th input variable - { - int vi0 = vidx ? vidx[vi] : vi; - ci = get_var_type(vi); - step = ds_step; m_step = ms_step; - if( CV_MAT_TYPE(_train_data->type) == CV_32SC1 ) - idata = _train_data->data.i + vi0*dv_step; - else - fdata = _train_data->data.fl + vi0*dv_step; - if( _missing_mask ) - mask = _missing_mask->data.ptr + vi0*mv_step; - } - else // analyze _responses - { - ci = cat_var_count; - step = CV_IS_MAT_CONT(_responses->type) ? - 1 : _responses->step / CV_ELEM_SIZE(_responses->type); - if( CV_MAT_TYPE(_responses->type) == CV_32SC1 ) - idata = _responses->data.i; - else - fdata = _responses->data.fl; - } - - if( (vi < var_count && ci>=0) || - (vi == var_count && is_classifier) ) // process categorical variable or response - { - int c_count, prev_label; - int* c_map; - - if (is_buf_16u) - udst = (unsigned short*)(buf->data.s + ci*sample_count); - else - idst = buf->data.i + ci*sample_count; - - // copy data - for( i = 0; i < sample_count; i++ ) - { - int val = INT_MAX, si = sidx ? sidx[i] : i; - if( !mask || !mask[(size_t)si*m_step] ) - { - if( idata ) - val = idata[(size_t)si*step]; - else - { - float t = fdata[(size_t)si*step]; - val = cvRound(t); - if( val != t ) - { - sprintf( err, "%d-th value of %d-th (categorical) " - "variable is not an integer", i, vi ); - CV_ERROR( CV_StsBadArg, err ); - } - } - - if( val == INT_MAX ) - { - sprintf( err, "%d-th value of %d-th (categorical) " - "variable is too large", i, vi ); - CV_ERROR( CV_StsBadArg, err ); - } - num_valid++; - } - if (is_buf_16u) - { - _idst[i] = val; - pair16u32s_ptr[i].u = udst + i; - pair16u32s_ptr[i].i = _idst + i; - } - else - { - idst[i] = val; - int_ptr[i] = idst + i; - } - } - - c_count = num_valid > 0; - - if (is_buf_16u) - { - std::sort(pair16u32s_ptr, pair16u32s_ptr + sample_count, LessThanPairs()); - // count the categories - for( i = 1; i < num_valid; i++ ) - if (*pair16u32s_ptr[i].i != *pair16u32s_ptr[i-1].i) - c_count ++ ; - } - else - { - std::sort(int_ptr, int_ptr + sample_count, LessThanPtr()); - // count the categories - for( i = 1; i < num_valid; i++ ) - c_count += *int_ptr[i] != *int_ptr[i-1]; - } - - if( vi > 0 ) - max_c_count = MAX( max_c_count, c_count ); - cat_count->data.i[ci] = c_count; - cat_ofs->data.i[ci] = total_c_count; - - // resize cat_map, if need - if( cat_map->cols < total_c_count + c_count ) - { - tmp_map = cat_map; - CV_CALL( cat_map = cvCreateMat( 1, - MAX(cat_map->cols*3/2,total_c_count+c_count), CV_32SC1 )); - for( i = 0; i < total_c_count; i++ ) - cat_map->data.i[i] = tmp_map->data.i[i]; - cvReleaseMat( &tmp_map ); - } - - c_map = cat_map->data.i + total_c_count; - total_c_count += c_count; - - c_count = -1; - if (is_buf_16u) - { - // compact the class indices and build the map - prev_label = ~*pair16u32s_ptr[0].i; - for( i = 0; i < num_valid; i++ ) - { - int cur_label = *pair16u32s_ptr[i].i; - if( cur_label != prev_label ) - c_map[++c_count] = prev_label = cur_label; - *pair16u32s_ptr[i].u = (unsigned short)c_count; - } - // replace labels for missing values with 65535 - for( ; i < sample_count; i++ ) - *pair16u32s_ptr[i].u = 65535; - } - else - { - // compact the class indices and build the map - prev_label = ~*int_ptr[0]; - for( i = 0; i < num_valid; i++ ) - { - int cur_label = *int_ptr[i]; - if( cur_label != prev_label ) - c_map[++c_count] = prev_label = cur_label; - *int_ptr[i] = c_count; - } - // replace labels for missing values with -1 - for( ; i < sample_count; i++ ) - *int_ptr[i] = -1; - } - } - else if( ci < 0 ) // process ordered variable - { - for( i = 0; i < sample_count; i++ ) - { - float val = ord_nan; - int si = sidx ? sidx[i] : i; - if( !mask || !mask[(size_t)si*m_step] ) - { - if( idata ) - val = (float)idata[(size_t)si*step]; - else - val = fdata[(size_t)si*step]; - - if( fabs(val) >= ord_nan ) - { - sprintf( err, "%d-th value of %d-th (ordered) " - "variable (=%g) is too large", i, vi, val ); - CV_ERROR( CV_StsBadArg, err ); - } - num_valid++; - } - } - } - if( vi < var_count ) - data_root->set_num_valid(vi, num_valid); - } - - // set sample labels - if (is_buf_16u) - udst = (unsigned short*)(buf->data.s + get_work_var_count()*sample_count); - else - idst = buf->data.i + get_work_var_count()*sample_count; - - for (i = 0; i < sample_count; i++) - { - if (udst) - udst[i] = sidx ? (unsigned short)sidx[i] : (unsigned short)i; - else - idst[i] = sidx ? sidx[i] : i; - } - - if( cv_n ) - { - unsigned short* usdst = 0; - int* idst2 = 0; - - if (is_buf_16u) - { - usdst = (unsigned short*)(buf->data.s + (get_work_var_count()-1)*sample_count); - for( i = vi = 0; i < sample_count; i++ ) - { - usdst[i] = (unsigned short)vi++; - vi &= vi < cv_n ? -1 : 0; - } - - for( i = 0; i < sample_count; i++ ) - { - int a = (*rng)(sample_count); - int b = (*rng)(sample_count); - unsigned short unsh = (unsigned short)vi; - CV_SWAP( usdst[a], usdst[b], unsh ); - } - } - else - { - idst2 = buf->data.i + (get_work_var_count()-1)*sample_count; - for( i = vi = 0; i < sample_count; i++ ) - { - idst2[i] = vi++; - vi &= vi < cv_n ? -1 : 0; - } - - for( i = 0; i < sample_count; i++ ) - { - int a = (*rng)(sample_count); - int b = (*rng)(sample_count); - CV_SWAP( idst2[a], idst2[b], vi ); - } - } - } - - if ( cat_map ) - cat_map->cols = MAX( total_c_count, 1 ); - - max_split_size = cvAlign(sizeof(CvDTreeSplit) + - (MAX(0,max_c_count - 33)/32)*sizeof(int),sizeof(void*)); - CV_CALL( split_heap = cvCreateSet( 0, sizeof(*split_heap), max_split_size, tree_storage )); - - have_priors = is_classifier && params.priors; - if( is_classifier ) - { - int m = get_num_classes(); - double sum = 0; - CV_CALL( priors = cvCreateMat( 1, m, CV_64F )); - for( i = 0; i < m; i++ ) - { - double val = have_priors ? params.priors[i] : 1.; - if( val <= 0 ) - CV_ERROR( CV_StsOutOfRange, "Every class weight should be positive" ); - priors->data.db[i] = val; - sum += val; - } - - // normalize weights - if( have_priors ) - cvScale( priors, priors, 1./sum ); - - CV_CALL( priors_mult = cvCloneMat( priors )); - CV_CALL( counts = cvCreateMat( 1, m, CV_32SC1 )); - } - - CV_CALL( direction = cvCreateMat( 1, sample_count, CV_8UC1 )); - CV_CALL( split_buf = cvCreateMat( 1, sample_count, CV_32SC1 )); - - __END__; - - if( data ) - delete data; - - if (_fdst) - cvFree( &_fdst ); - if (_idst) - cvFree( &_idst ); - cvFree( &int_ptr ); - cvReleaseMat( &var_type0 ); - cvReleaseMat( &sample_indices ); - cvReleaseMat( &tmp_map ); -} - -void CvERTreeTrainData::get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* missing_buf, - const float** ord_values, const int** missing, int* sample_indices_buf ) -{ - int vidx = var_idx ? var_idx->data.i[vi] : vi; - int node_sample_count = n->sample_count; - // may use missing_buf as buffer for sample indices! - const int* sample_indices = get_sample_indices(n, sample_indices_buf ? sample_indices_buf : missing_buf); - - int td_step = train_data->step/CV_ELEM_SIZE(train_data->type); - int m_step = missing_mask ? missing_mask->step/CV_ELEM_SIZE(missing_mask->type) : 1; - if( tflag == CV_ROW_SAMPLE ) - { - for( int i = 0; i < node_sample_count; i++ ) - { - int idx = sample_indices[i]; - missing_buf[i] = missing_mask ? *(missing_mask->data.ptr + idx * m_step + vi) : 0; - ord_values_buf[i] = *(train_data->data.fl + idx * td_step + vidx); - } - } - else - for( int i = 0; i < node_sample_count; i++ ) - { - int idx = sample_indices[i]; - missing_buf[i] = missing_mask ? *(missing_mask->data.ptr + vi* m_step + idx) : 0; - ord_values_buf[i] = *(train_data->data.fl + vidx* td_step + idx); - } - *ord_values = ord_values_buf; - *missing = missing_buf; -} - - -const int* CvERTreeTrainData::get_sample_indices( CvDTreeNode* n, int* indices_buf ) -{ - return get_cat_var_data( n, var_count + (is_classifier ? 1 : 0) + (have_labels ? 1 : 0), indices_buf ); -} - - -const int* CvERTreeTrainData::get_cv_labels( CvDTreeNode* n, int* labels_buf ) -{ - if (have_labels) - return get_cat_var_data( n, var_count + (is_classifier ? 1 : 0), labels_buf ); - return 0; -} - - -const int* CvERTreeTrainData::get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf ) -{ - int ci = get_var_type( vi); - const int* cat_values = 0; - if( !is_buf_16u ) - cat_values = buf->data.i + n->buf_idx*get_length_subbuf() + ci*sample_count + n->offset; - else { - const unsigned short* short_values = (const unsigned short*)(buf->data.s + n->buf_idx*get_length_subbuf() + - ci*sample_count + n->offset); - for( int i = 0; i < n->sample_count; i++ ) - cat_values_buf[i] = short_values[i]; - cat_values = cat_values_buf; - } - return cat_values; -} - -void CvERTreeTrainData::get_vectors( const CvMat* _subsample_idx, - float* values, uchar* missing, - float* _responses, bool get_class_idx ) -{ - CvMat* subsample_idx = 0; - CvMat* subsample_co = 0; - - cv::AutoBuffer inn_buf(sample_count*(sizeof(float) + sizeof(int))); - - CV_FUNCNAME( "CvERTreeTrainData::get_vectors" ); - - __BEGIN__; - - int i, vi, total = sample_count, count = total, cur_ofs = 0; - int* sidx = 0; - int* co = 0; - - if( _subsample_idx ) - { - CV_CALL( subsample_idx = cvPreprocessIndexArray( _subsample_idx, sample_count )); - sidx = subsample_idx->data.i; - CV_CALL( subsample_co = cvCreateMat( 1, sample_count*2, CV_32SC1 )); - co = subsample_co->data.i; - cvZero( subsample_co ); - count = subsample_idx->cols + subsample_idx->rows - 1; - for( i = 0; i < count; i++ ) - co[sidx[i]*2]++; - for( i = 0; i < total; i++ ) - { - int count_i = co[i*2]; - if( count_i ) - { - co[i*2+1] = cur_ofs*var_count; - cur_ofs += count_i; - } - } - } - - if( missing ) - memset( missing, 1, count*var_count ); - - for( vi = 0; vi < var_count; vi++ ) - { - int ci = get_var_type(vi); - if( ci >= 0 ) // categorical - { - float* dst = values + vi; - uchar* m = missing ? missing + vi : 0; - int* lbls_buf = (int*)(uchar*)inn_buf; - const int* src = get_cat_var_data(data_root, vi, lbls_buf); - - for( i = 0; i < count; i++, dst += var_count ) - { - int idx = sidx ? sidx[i] : i; - int val = src[idx]; - *dst = (float)val; - if( m ) - { - *m = (!is_buf_16u && val < 0) || (is_buf_16u && (val == 65535)); - m += var_count; - } - } - } - else // ordered - { - int* mis_buf = (int*)(uchar*)inn_buf; - const float *dst = 0; - const int* mis = 0; - get_ord_var_data(data_root, vi, values + vi, mis_buf, &dst, &mis, 0); - for (int si = 0; si < total; si++) - *(missing + vi + si) = mis[si] == 0 ? 0 : 1; - } - } - - // copy responses - if( _responses ) - { - if( is_classifier ) - { - int* lbls_buf = (int*)(uchar*)inn_buf; - const int* src = get_class_labels(data_root, lbls_buf); - for( i = 0; i < count; i++ ) - { - int idx = sidx ? sidx[i] : i; - int val = get_class_idx ? src[idx] : - cat_map->data.i[cat_ofs->data.i[cat_var_count]+src[idx]]; - _responses[i] = (float)val; - } - } - else - { - float* _values_buf = (float*)(uchar*)inn_buf; - int* sample_idx_buf = (int*)(_values_buf + sample_count); - const float* _values = get_ord_responses(data_root, _values_buf, sample_idx_buf); - for( i = 0; i < count; i++ ) - { - int idx = sidx ? sidx[i] : i; - _responses[i] = _values[idx]; - } - } - } - - __END__; - - cvReleaseMat( &subsample_idx ); - cvReleaseMat( &subsample_co ); -} - -CvDTreeNode* CvERTreeTrainData::subsample_data( const CvMat* _subsample_idx ) -{ - CvDTreeNode* root = 0; - - CV_FUNCNAME( "CvERTreeTrainData::subsample_data" ); - - __BEGIN__; - - if( !data_root ) - CV_ERROR( CV_StsError, "No training data has been set" ); - - if( !_subsample_idx ) - { - // make a copy of the root node - CvDTreeNode temp; - int i; - root = new_node( 0, 1, 0, 0 ); - temp = *root; - *root = *data_root; - root->num_valid = temp.num_valid; - if( root->num_valid ) - { - for( i = 0; i < var_count; i++ ) - root->num_valid[i] = data_root->num_valid[i]; - } - root->cv_Tn = temp.cv_Tn; - root->cv_node_risk = temp.cv_node_risk; - root->cv_node_error = temp.cv_node_error; - } - else - CV_ERROR( CV_StsError, "_subsample_idx must be null for extra-trees" ); - __END__; - - return root; -} - -double CvForestERTree::calc_node_dir( CvDTreeNode* node ) -{ - char* dir = (char*)data->direction->data.ptr; - int i, n = node->sample_count, vi = node->split->var_idx; - double L, R; - - assert( !node->split->inversed ); - - if( data->get_var_type(vi) >= 0 ) // split on categorical var - { - cv::AutoBuffer inn_buf(n*sizeof(int)*(!data->have_priors ? 1 : 2)); - int* labels_buf = (int*)(uchar*)inn_buf; - const int* labels = data->get_cat_var_data( node, vi, labels_buf ); - const int* subset = node->split->subset; - if( !data->have_priors ) - { - int sum = 0, sum_abs = 0; - - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - int d = ( ((idx >= 0)&&(!data->is_buf_16u)) || ((idx != 65535)&&(data->is_buf_16u)) ) ? - CV_DTREE_CAT_DIR(idx,subset) : 0; - sum += d; sum_abs += d & 1; - dir[i] = (char)d; - } - - R = (sum_abs + sum) >> 1; - L = (sum_abs - sum) >> 1; - } - else - { - const double* priors = data->priors_mult->data.db; - double sum = 0, sum_abs = 0; - int *responses_buf = labels_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - double w = priors[responses[i]]; - int d = idx >= 0 ? CV_DTREE_CAT_DIR(idx,subset) : 0; - sum += d*w; sum_abs += (d & 1)*w; - dir[i] = (char)d; - } - - R = (sum_abs + sum) * 0.5; - L = (sum_abs - sum) * 0.5; - } - } - else // split on ordered var - { - float split_val = node->split->ord.c; - cv::AutoBuffer inn_buf(n*(sizeof(int)*(!data->have_priors ? 1 : 2) + sizeof(float))); - float* val_buf = (float*)(uchar*)inn_buf; - int* missing_buf = (int*)(val_buf + n); - const float* val = 0; - const int* missing = 0; - data->get_ord_var_data( node, vi, val_buf, missing_buf, &val, &missing, 0 ); - - if( !data->have_priors ) - { - L = R = 0; - for( i = 0; i < n; i++ ) - { - if ( missing[i] ) - dir[i] = (char)0; - else - { - if ( val[i] < split_val) - { - dir[i] = (char)-1; - L++; - } - else - { - dir[i] = (char)1; - R++; - } - } - } - } - else - { - const double* priors = data->priors_mult->data.db; - int* responses_buf = missing_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - L = R = 0; - for( i = 0; i < n; i++ ) - { - if ( missing[i] ) - dir[i] = (char)0; - else - { - double w = priors[responses[i]]; - if ( val[i] < split_val) - { - dir[i] = (char)-1; - L += w; - } - else - { - dir[i] = (char)1; - R += w; - } - } - } - } - } - - node->maxlr = MAX( L, R ); - return node->split->quality/(L + R); -} - -CvDTreeSplit* CvForestERTree::find_split_ord_class( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, - uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - const float split_delta = (1 + FLT_EPSILON) * FLT_EPSILON; - - int n = node->sample_count; - int m = data->get_num_classes(); - - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(n*(2*sizeof(int) + sizeof(float))); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - float* values_buf = (float*)ext_buf; - int* missing_buf = (int*)(values_buf + n); - const float* values = 0; - const int* missing = 0; - data->get_ord_var_data( node, vi, values_buf, missing_buf, &values, &missing, 0 ); - int* responses_buf = missing_buf + n; - const int* responses = data->get_class_labels( node, responses_buf ); - - double lbest_val = 0, rbest_val = 0, best_val = init_quality, split_val = 0; - const double* priors = data->have_priors ? data->priors_mult->data.db : 0; - bool is_find_split = false; - float pmin, pmax; - int smpi = 0; - while ( missing[smpi] && (smpi < n) ) - smpi++; - assert(smpi < n); - - pmin = values[smpi]; - pmax = pmin; - for (; smpi < n; smpi++) - { - float ptemp = values[smpi]; - int ms = missing[smpi]; - if (ms) continue; - if ( ptemp < pmin) - pmin = ptemp; - if ( ptemp > pmax) - pmax = ptemp; - } - float fdiff = pmax-pmin; - if (fdiff > epsilon) - { - is_find_split = true; - cv::RNG* rng = data->rng; - split_val = pmin + rng->uniform(0.f, 1.f) * fdiff ; - if (split_val - pmin <= FLT_EPSILON) - split_val = pmin + split_delta; - if (pmax - split_val <= FLT_EPSILON) - split_val = pmax - split_delta; - - // calculate Gini index - if ( !priors ) - { - cv::AutoBuffer lrc(m*2); - int *lc = lrc, *rc = lc + m; - int L = 0, R = 0; - - // init arrays of class instance counters on both sides of the split - for(int i = 0; i < m; i++ ) - { - lc[i] = 0; - rc[i] = 0; - } - for( int si = 0; si < n; si++ ) - { - int r = responses[si]; - float val = values[si]; - int ms = missing[si]; - if (ms) continue; - if ( val < split_val ) - { - lc[r]++; - L++; - } - else - { - rc[r]++; - R++; - } - } - for (int i = 0; i < m; i++) - { - lbest_val += lc[i]*lc[i]; - rbest_val += rc[i]*rc[i]; - } - best_val = (lbest_val*R + rbest_val*L) / ((double)(L*R)); - } - else - { - cv::AutoBuffer lrc(m*2); - double *lc = lrc, *rc = lc + m; - double L = 0, R = 0; - - // init arrays of class instance counters on both sides of the split - for(int i = 0; i < m; i++ ) - { - lc[i] = 0; - rc[i] = 0; - } - for( int si = 0; si < n; si++ ) - { - int r = responses[si]; - float val = values[si]; - int ms = missing[si]; - double p = priors[r]; - if (ms) continue; - if ( val < split_val ) - { - lc[r] += p; - L += p; - } - else - { - rc[r] += p; - R += p; - } - } - for (int i = 0; i < m; i++) - { - lbest_val += lc[i]*lc[i]; - rbest_val += rc[i]*rc[i]; - } - best_val = (lbest_val*R + rbest_val*L) / (L*R); - } - - } - - CvDTreeSplit* split = 0; - if( is_find_split ) - { - split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); - split->var_idx = vi; - split->ord.c = (float)split_val; - split->ord.split_point = -1; - split->inversed = 0; - split->quality = (float)best_val; - } - return split; -} - -CvDTreeSplit* CvForestERTree::find_split_cat_class( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, - uchar* _ext_buf ) -{ - int ci = data->get_var_type(vi); - int n = node->sample_count; - int cm = data->get_num_classes(); - int vm = data->cat_count->data.i[ci]; - double best_val = init_quality; - CvDTreeSplit *split = 0; - - if ( vm > 1 ) - { - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(2*n); - int* ext_buf = _ext_buf ? (int*)_ext_buf : (int*)inn_buf; - - const int* labels = data->get_cat_var_data( node, vi, ext_buf ); - const int* responses = data->get_class_labels( node, ext_buf + n ); - - const double* priors = data->have_priors ? data->priors_mult->data.db : 0; - - // create random class mask - cv::AutoBuffer valid_cidx(vm); - for (int i = 0; i < vm; i++) - { - valid_cidx[i] = -1; - } - for (int si = 0; si < n; si++) - { - int c = labels[si]; - if ( ((c == 65535) && data->is_buf_16u) || ((c<0) && (!data->is_buf_16u)) ) - continue; - valid_cidx[c]++; - } - - int valid_ccount = 0; - for (int i = 0; i < vm; i++) - if (valid_cidx[i] >= 0) - { - valid_cidx[i] = valid_ccount; - valid_ccount++; - } - if (valid_ccount > 1) - { - CvRNG* rng = forest->get_rng(); - int l_cval_count = 1 + cvRandInt(rng) % (valid_ccount-1); - - CvMat* var_class_mask = cvCreateMat( 1, valid_ccount, CV_8UC1 ); - CvMat submask; - memset(var_class_mask->data.ptr, 0, valid_ccount*CV_ELEM_SIZE(var_class_mask->type)); - cvGetCols( var_class_mask, &submask, 0, l_cval_count ); - cvSet( &submask, cvScalar(1) ); - for (int i = 0; i < valid_ccount; i++) - { - uchar temp; - int i1 = cvRandInt( rng ) % valid_ccount; - int i2 = cvRandInt( rng ) % valid_ccount; - CV_SWAP( var_class_mask->data.ptr[i1], var_class_mask->data.ptr[i2], temp ); - } - - split = _split ? _split : data->new_split_cat( 0, -1.0f ); - split->var_idx = vi; - memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); - - // calculate Gini index - double lbest_val = 0, rbest_val = 0; - if( !priors ) - { - cv::AutoBuffer lrc(cm*2); - int *lc = lrc, *rc = lc + cm; - int L = 0, R = 0; - // init arrays of class instance counters on both sides of the split - for(int i = 0; i < cm; i++ ) - { - lc[i] = 0; - rc[i] = 0; - } - for( int si = 0; si < n; si++ ) - { - int r = responses[si]; - int var_class_idx = labels[si]; - if ( ((var_class_idx == 65535) && data->is_buf_16u) || ((var_class_idx<0) && (!data->is_buf_16u)) ) - continue; - int mask_class_idx = valid_cidx[var_class_idx]; - if (var_class_mask->data.ptr[mask_class_idx]) - { - lc[r]++; - L++; - split->subset[var_class_idx >> 5] |= 1 << (var_class_idx & 31); - } - else - { - rc[r]++; - R++; - } - } - for (int i = 0; i < cm; i++) - { - lbest_val += lc[i]*lc[i]; - rbest_val += rc[i]*rc[i]; - } - best_val = (lbest_val*R + rbest_val*L) / ((double)(L*R)); - } - else - { - cv::AutoBuffer lrc(cm*2); - int *lc = lrc, *rc = lc + cm; - double L = 0, R = 0; - // init arrays of class instance counters on both sides of the split - for(int i = 0; i < cm; i++ ) - { - lc[i] = 0; - rc[i] = 0; - } - for( int si = 0; si < n; si++ ) - { - int r = responses[si]; - int var_class_idx = labels[si]; - if ( ((var_class_idx == 65535) && data->is_buf_16u) || ((var_class_idx<0) && (!data->is_buf_16u)) ) - continue; - double p = priors[si]; - int mask_class_idx = valid_cidx[var_class_idx]; - - if (var_class_mask->data.ptr[mask_class_idx]) - { - lc[r]+=(int)p; - L+=p; - split->subset[var_class_idx >> 5] |= 1 << (var_class_idx & 31); - } - else - { - rc[r]+=(int)p; - R+=p; - } - } - for (int i = 0; i < cm; i++) - { - lbest_val += lc[i]*lc[i]; - rbest_val += rc[i]*rc[i]; - } - best_val = (lbest_val*R + rbest_val*L) / (L*R); - } - split->quality = (float)best_val; - - cvReleaseMat(&var_class_mask); - } - } - - return split; -} - -CvDTreeSplit* CvForestERTree::find_split_ord_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, - uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - const float split_delta = (1 + FLT_EPSILON) * FLT_EPSILON; - int n = node->sample_count; - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(n*(2*sizeof(int) + 2*sizeof(float))); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - float* values_buf = (float*)ext_buf; - int* missing_buf = (int*)(values_buf + n); - const float* values = 0; - const int* missing = 0; - data->get_ord_var_data( node, vi, values_buf, missing_buf, &values, &missing, 0 ); - float* responses_buf = (float*)(missing_buf + n); - int* sample_indices_buf = (int*)(responses_buf + n); - const float* responses = data->get_ord_responses( node, responses_buf, sample_indices_buf ); - - double best_val = init_quality, split_val = 0, lsum = 0, rsum = 0; - int L = 0, R = 0; - - bool is_find_split = false; - float pmin, pmax; - int smpi = 0; - while ( missing[smpi] && (smpi < n) ) - smpi++; - - assert(smpi < n); - - pmin = values[smpi]; - pmax = pmin; - for (; smpi < n; smpi++) - { - float ptemp = values[smpi]; - int m = missing[smpi]; - if (m) continue; - if ( ptemp < pmin) - pmin = ptemp; - if ( ptemp > pmax) - pmax = ptemp; - } - float fdiff = pmax-pmin; - if (fdiff > epsilon) - { - is_find_split = true; - cv::RNG* rng = data->rng; - split_val = pmin + rng->uniform(0.f, 1.f) * fdiff ; - if (split_val - pmin <= FLT_EPSILON) - split_val = pmin + split_delta; - if (pmax - split_val <= FLT_EPSILON) - split_val = pmax - split_delta; - - for (int si = 0; si < n; si++) - { - float r = responses[si]; - float val = values[si]; - int m = missing[si]; - if (m) continue; - if (val < split_val) - { - lsum += r; - L++; - } - else - { - rsum += r; - R++; - } - } - best_val = (lsum*lsum*R + rsum*rsum*L)/((double)L*R); - } - - CvDTreeSplit* split = 0; - if( is_find_split ) - { - split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); - split->var_idx = vi; - split->ord.c = (float)split_val; - split->ord.split_point = -1; - split->inversed = 0; - split->quality = (float)best_val; - } - return split; -} - -CvDTreeSplit* CvForestERTree::find_split_cat_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, - uchar* _ext_buf ) -{ - int ci = data->get_var_type(vi); - int n = node->sample_count; - int vm = data->cat_count->data.i[ci]; - double best_val = init_quality; - CvDTreeSplit *split = 0; - float lsum = 0, rsum = 0; - - if ( vm > 1 ) - { - int base_size = vm*sizeof(int); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + n*(2*sizeof(int) + sizeof(float))); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - int* labels_buf = (int*)ext_buf; - const int* labels = data->get_cat_var_data( node, vi, labels_buf ); - float* responses_buf = (float*)(labels_buf + n); - int* sample_indices_buf = (int*)(responses_buf + n); - const float* responses = data->get_ord_responses( node, responses_buf, sample_indices_buf ); - - // create random class mask - int *valid_cidx = (int*)base_buf; - for (int i = 0; i < vm; i++) - { - valid_cidx[i] = -1; - } - for (int si = 0; si < n; si++) - { - int c = labels[si]; - if ( ((c == 65535) && data->is_buf_16u) || ((c<0) && (!data->is_buf_16u)) ) - continue; - valid_cidx[c]++; - } - - int valid_ccount = 0; - for (int i = 0; i < vm; i++) - if (valid_cidx[i] >= 0) - { - valid_cidx[i] = valid_ccount; - valid_ccount++; - } - if (valid_ccount > 1) - { - CvRNG* rng = forest->get_rng(); - int l_cval_count = 1 + cvRandInt(rng) % (valid_ccount-1); - - CvMat* var_class_mask = cvCreateMat( 1, valid_ccount, CV_8UC1 ); - CvMat submask; - memset(var_class_mask->data.ptr, 0, valid_ccount*CV_ELEM_SIZE(var_class_mask->type)); - cvGetCols( var_class_mask, &submask, 0, l_cval_count ); - cvSet( &submask, cvScalar(1) ); - for (int i = 0; i < valid_ccount; i++) - { - uchar temp; - int i1 = cvRandInt( rng ) % valid_ccount; - int i2 = cvRandInt( rng ) % valid_ccount; - CV_SWAP( var_class_mask->data.ptr[i1], var_class_mask->data.ptr[i2], temp ); - } - - split = _split ? _split : data->new_split_cat( 0, -1.0f); - split->var_idx = vi; - memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); - - int L = 0, R = 0; - for( int si = 0; si < n; si++ ) - { - float r = responses[si]; - int var_class_idx = labels[si]; - if ( ((var_class_idx == 65535) && data->is_buf_16u) || ((var_class_idx<0) && (!data->is_buf_16u)) ) - continue; - int mask_class_idx = valid_cidx[var_class_idx]; - if (var_class_mask->data.ptr[mask_class_idx]) - { - lsum += r; - L++; - split->subset[var_class_idx >> 5] |= 1 << (var_class_idx & 31); - } - else - { - rsum += r; - R++; - } - } - best_val = (lsum*lsum*R + rsum*rsum*L)/((double)L*R); - - split->quality = (float)best_val; - - cvReleaseMat(&var_class_mask); - } - } - - return split; -} - -void CvForestERTree::split_node_data( CvDTreeNode* node ) -{ - int vi, i, n = node->sample_count, nl, nr, scount = data->sample_count; - char* dir = (char*)data->direction->data.ptr; - CvDTreeNode *left = 0, *right = 0; - int new_buf_idx = data->get_child_buf_idx( node ); - CvMat* buf = data->buf; - size_t length_buf_row = data->get_length_subbuf(); - cv::AutoBuffer temp_buf(n); - - complete_node_dir(node); - - for( i = nl = nr = 0; i < n; i++ ) - { - int d = dir[i]; - nr += d; - nl += d^1; - } - - bool split_input_data; - node->left = left = data->new_node( node, nl, new_buf_idx, node->offset ); - node->right = right = data->new_node( node, nr, new_buf_idx, node->offset + nl ); - - split_input_data = node->depth + 1 < data->params.max_depth && - (node->left->sample_count > data->params.min_sample_count || - node->right->sample_count > data->params.min_sample_count); - - cv::AutoBuffer inn_buf(n*(sizeof(int)+sizeof(float))); - // split ordered vars - for( vi = 0; vi < data->var_count; vi++ ) - { - int ci = data->get_var_type(vi); - if (ci >= 0) continue; - - int n1 = node->get_num_valid(vi), nr1 = 0; - float* values_buf = (float*)(uchar*)inn_buf; - int* missing_buf = (int*)(values_buf + n); - const float* values = 0; - const int* missing = 0; - data->get_ord_var_data( node, vi, values_buf, missing_buf, &values, &missing, 0 ); - - for( i = 0; i < n; i++ ) - nr1 += ((!missing[i]) & dir[i]); - left->set_num_valid(vi, n1 - nr1); - right->set_num_valid(vi, nr1); - } - // split categorical vars, responses and cv_labels using new_idx relocation table - for( vi = 0; vi < data->get_work_var_count() + data->ord_var_count; vi++ ) - { - int ci = data->get_var_type(vi); - if (ci < 0) continue; - - int n1 = node->get_num_valid(vi), nr1 = 0; - const int* src_lbls = data->get_cat_var_data(node, vi, (int*)(uchar*)inn_buf); - - for(i = 0; i < n; i++) - temp_buf[i] = src_lbls[i]; - - if (data->is_buf_16u) - { - unsigned short *ldst = (unsigned short *)(buf->data.s + left->buf_idx*length_buf_row + - ci*scount + left->offset); - unsigned short *rdst = (unsigned short *)(buf->data.s + right->buf_idx*length_buf_row + - ci*scount + right->offset); - - for( i = 0; i < n; i++ ) - { - int d = dir[i]; - int idx = temp_buf[i]; - if (d) - { - *rdst = (unsigned short)idx; - rdst++; - nr1 += (idx != 65535); - } - else - { - *ldst = (unsigned short)idx; - ldst++; - } - } - - if( vi < data->var_count ) - { - left->set_num_valid(vi, n1 - nr1); - right->set_num_valid(vi, nr1); - } - } - else - { - int *ldst = buf->data.i + left->buf_idx*length_buf_row + - ci*scount + left->offset; - int *rdst = buf->data.i + right->buf_idx*length_buf_row + - ci*scount + right->offset; - - for( i = 0; i < n; i++ ) - { - int d = dir[i]; - int idx = temp_buf[i]; - if (d) - { - *rdst = idx; - rdst++; - nr1 += (idx >= 0); - } - else - { - *ldst = idx; - ldst++; - } - - } - - if( vi < data->var_count ) - { - left->set_num_valid(vi, n1 - nr1); - right->set_num_valid(vi, nr1); - } - } - } - - // split sample indices - int *sample_idx_src_buf = (int*)(uchar*)inn_buf; - const int* sample_idx_src = 0; - if (split_input_data) - { - sample_idx_src = data->get_sample_indices(node, sample_idx_src_buf); - - for(i = 0; i < n; i++) - temp_buf[i] = sample_idx_src[i]; - - int pos = data->get_work_var_count(); - - if (data->is_buf_16u) - { - unsigned short* ldst = (unsigned short*)(buf->data.s + left->buf_idx*length_buf_row + - pos*scount + left->offset); - unsigned short* rdst = (unsigned short*)(buf->data.s + right->buf_idx*length_buf_row + - pos*scount + right->offset); - - for (i = 0; i < n; i++) - { - int d = dir[i]; - unsigned short idx = (unsigned short)temp_buf[i]; - if (d) - { - *rdst = idx; - rdst++; - } - else - { - *ldst = idx; - ldst++; - } - } - } - else - { - int* ldst = buf->data.i + left->buf_idx*length_buf_row + - pos*scount + left->offset; - int* rdst = buf->data.i + right->buf_idx*length_buf_row + - pos*scount + right->offset; - for (i = 0; i < n; i++) - { - int d = dir[i]; - int idx = temp_buf[i]; - if (d) - { - *rdst = idx; - rdst++; - } - else - { - *ldst = idx; - ldst++; - } - } - } - } - - // deallocate the parent node data that is not needed anymore - data->free_node_data(node); -} - -CvERTrees::CvERTrees() -{ -} - -CvERTrees::~CvERTrees() -{ -} - -cv::String CvERTrees::getName() const -{ - return CV_TYPE_NAME_ML_ERTREES; -} - -bool CvERTrees::train( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, - const CvMat* _sample_idx, const CvMat* _var_type, - const CvMat* _missing_mask, CvRTParams params ) -{ - bool result = false; - - CV_FUNCNAME("CvERTrees::train"); - __BEGIN__ - int var_count = 0; - - clear(); - - CvDTreeParams tree_params( params.max_depth, params.min_sample_count, - params.regression_accuracy, params.use_surrogates, params.max_categories, - params.cv_folds, params.use_1se_rule, false, params.priors ); - - data = new CvERTreeTrainData(); - CV_CALL(data->set_data( _train_data, _tflag, _responses, _var_idx, - _sample_idx, _var_type, _missing_mask, tree_params, true)); - - var_count = data->var_count; - if( params.nactive_vars > var_count ) - params.nactive_vars = var_count; - else if( params.nactive_vars == 0 ) - params.nactive_vars = (int)sqrt((double)var_count); - else if( params.nactive_vars < 0 ) - CV_ERROR( CV_StsBadArg, " must be non-negative" ); - - // Create mask of active variables at the tree nodes - CV_CALL(active_var_mask = cvCreateMat( 1, var_count, CV_8UC1 )); - if( params.calc_var_importance ) - { - CV_CALL(var_importance = cvCreateMat( 1, var_count, CV_32FC1 )); - cvZero(var_importance); - } - { // initialize active variables mask - CvMat submask1, submask2; - CV_Assert( (active_var_mask->cols >= 1) && (params.nactive_vars > 0) && (params.nactive_vars <= active_var_mask->cols) ); - cvGetCols( active_var_mask, &submask1, 0, params.nactive_vars ); - cvSet( &submask1, cvScalar(1) ); - if( params.nactive_vars < active_var_mask->cols ) - { - cvGetCols( active_var_mask, &submask2, params.nactive_vars, var_count ); - cvZero( &submask2 ); - } - } - - CV_CALL(result = grow_forest( params.term_crit )); - - result = true; - - __END__ - return result; - -} - -bool CvERTrees::train( CvMLData* _data, CvRTParams params) -{ - bool result = false; - - CV_FUNCNAME( "CvERTrees::train" ); - - __BEGIN__; - - CV_CALL( result = CvRTrees::train( _data, params) ); - - __END__; - - return result; -} - -bool CvERTrees::grow_forest( const CvTermCriteria term_crit ) -{ - bool result = false; - - CvMat* sample_idx_for_tree = 0; - - CV_FUNCNAME("CvERTrees::grow_forest"); - __BEGIN__; - - const int max_ntrees = term_crit.max_iter; - const double max_oob_err = term_crit.epsilon; - - const int dims = data->var_count; - float maximal_response = 0; - - CvMat* oob_sample_votes = 0; - CvMat* oob_responses = 0; - - float* oob_samples_perm_ptr= 0; - - float* samples_ptr = 0; - uchar* missing_ptr = 0; - float* true_resp_ptr = 0; - bool is_oob_or_vimportance = ((max_oob_err > 0) && (term_crit.type != CV_TERMCRIT_ITER)) || var_importance; - - // oob_predictions_sum[i] = sum of predicted values for the i-th sample - // oob_num_of_predictions[i] = number of summands - // (number of predictions for the i-th sample) - // initialize these variable to avoid warning C4701 - CvMat oob_predictions_sum = cvMat( 1, 1, CV_32FC1 ); - CvMat oob_num_of_predictions = cvMat( 1, 1, CV_32FC1 ); - - nsamples = data->sample_count; - nclasses = data->get_num_classes(); - - if ( is_oob_or_vimportance ) - { - if( data->is_classifier ) - { - CV_CALL(oob_sample_votes = cvCreateMat( nsamples, nclasses, CV_32SC1 )); - cvZero(oob_sample_votes); - } - else - { - // oob_responses[0,i] = oob_predictions_sum[i] - // = sum of predicted values for the i-th sample - // oob_responses[1,i] = oob_num_of_predictions[i] - // = number of summands (number of predictions for the i-th sample) - CV_CALL(oob_responses = cvCreateMat( 2, nsamples, CV_32FC1 )); - cvZero(oob_responses); - cvGetRow( oob_responses, &oob_predictions_sum, 0 ); - cvGetRow( oob_responses, &oob_num_of_predictions, 1 ); - } - - CV_CALL(oob_samples_perm_ptr = (float*)cvAlloc( sizeof(float)*nsamples*dims )); - CV_CALL(samples_ptr = (float*)cvAlloc( sizeof(float)*nsamples*dims )); - CV_CALL(missing_ptr = (uchar*)cvAlloc( sizeof(uchar)*nsamples*dims )); - CV_CALL(true_resp_ptr = (float*)cvAlloc( sizeof(float)*nsamples )); - - CV_CALL(data->get_vectors( 0, samples_ptr, missing_ptr, true_resp_ptr )); - { - double minval, maxval; - CvMat responses = cvMat(1, nsamples, CV_32FC1, true_resp_ptr); - cvMinMaxLoc( &responses, &minval, &maxval ); - maximal_response = (float)MAX( MAX( fabs(minval), fabs(maxval) ), 0 ); - } - } - - trees = (CvForestTree**)cvAlloc( sizeof(trees[0])*max_ntrees ); - memset( trees, 0, sizeof(trees[0])*max_ntrees ); - - CV_CALL(sample_idx_for_tree = cvCreateMat( 1, nsamples, CV_32SC1 )); - - for (int i = 0; i < nsamples; i++) - sample_idx_for_tree->data.i[i] = i; - ntrees = 0; - while( ntrees < max_ntrees ) - { - int i, oob_samples_count = 0; - double ncorrect_responses = 0; // used for estimation of variable importance - CvForestTree* tree = 0; - - trees[ntrees] = new CvForestERTree(); - tree = (CvForestERTree*)trees[ntrees]; - CV_CALL(tree->train( data, 0, this )); - - if ( is_oob_or_vimportance ) - { - CvMat sample, missing; - // form array of OOB samples indices and get these samples - sample = cvMat( 1, dims, CV_32FC1, samples_ptr ); - missing = cvMat( 1, dims, CV_8UC1, missing_ptr ); - - oob_error = 0; - for( i = 0; i < nsamples; i++, - sample.data.fl += dims, missing.data.ptr += dims ) - { - CvDTreeNode* predicted_node = 0; - - // predict oob samples - if( !predicted_node ) - CV_CALL(predicted_node = tree->predict(&sample, &missing, true)); - - if( !data->is_classifier ) //regression - { - double avg_resp, resp = predicted_node->value; - oob_predictions_sum.data.fl[i] += (float)resp; - oob_num_of_predictions.data.fl[i] += 1; - - // compute oob error - avg_resp = oob_predictions_sum.data.fl[i]/oob_num_of_predictions.data.fl[i]; - avg_resp -= true_resp_ptr[i]; - oob_error += avg_resp*avg_resp; - resp = (resp - true_resp_ptr[i])/maximal_response; - ncorrect_responses += exp( -resp*resp ); - } - else //classification - { - double prdct_resp; - CvPoint max_loc; - CvMat votes; - - cvGetRow(oob_sample_votes, &votes, i); - votes.data.i[predicted_node->class_idx]++; - - // compute oob error - cvMinMaxLoc( &votes, 0, 0, 0, &max_loc ); - - prdct_resp = data->cat_map->data.i[max_loc.x]; - oob_error += (fabs(prdct_resp - true_resp_ptr[i]) < FLT_EPSILON) ? 0 : 1; - - ncorrect_responses += cvRound(predicted_node->value - true_resp_ptr[i]) == 0; - } - oob_samples_count++; - } - if( oob_samples_count > 0 ) - oob_error /= (double)oob_samples_count; - - // estimate variable importance - if( var_importance && oob_samples_count > 0 ) - { - int m; - - memcpy( oob_samples_perm_ptr, samples_ptr, dims*nsamples*sizeof(float)); - for( m = 0; m < dims; m++ ) - { - double ncorrect_responses_permuted = 0; - // randomly permute values of the m-th variable in the oob samples - float* mth_var_ptr = oob_samples_perm_ptr + m; - - for( i = 0; i < nsamples; i++ ) - { - int i1, i2; - float temp; - - i1 = (*rng)(nsamples); - i2 = (*rng)(nsamples); - CV_SWAP( mth_var_ptr[i1*dims], mth_var_ptr[i2*dims], temp ); - - // turn values of (m-1)-th variable, that were permuted - // at the previous iteration, untouched - if( m > 1 ) - oob_samples_perm_ptr[i*dims+m-1] = samples_ptr[i*dims+m-1]; - } - - // predict "permuted" cases and calculate the number of votes for the - // correct class in the variable-m-permuted oob data - sample = cvMat( 1, dims, CV_32FC1, oob_samples_perm_ptr ); - missing = cvMat( 1, dims, CV_8UC1, missing_ptr ); - for( i = 0; i < nsamples; i++, - sample.data.fl += dims, missing.data.ptr += dims ) - { - double predct_resp, true_resp; - - predct_resp = tree->predict(&sample, &missing, true)->value; - true_resp = true_resp_ptr[i]; - if( data->is_classifier ) - ncorrect_responses_permuted += cvRound(true_resp - predct_resp) == 0; - else - { - true_resp = (true_resp - predct_resp)/maximal_response; - ncorrect_responses_permuted += exp( -true_resp*true_resp ); - } - } - var_importance->data.fl[m] += (float)(ncorrect_responses - - ncorrect_responses_permuted); - } - } - } - ntrees++; - if( term_crit.type != CV_TERMCRIT_ITER && oob_error < max_oob_err ) - break; - } - if( var_importance ) - { - for ( int vi = 0; vi < var_importance->cols; vi++ ) - var_importance->data.fl[vi] = ( var_importance->data.fl[vi] > 0 ) ? - var_importance->data.fl[vi] : 0; - cvNormalize( var_importance, var_importance, 1., 0, CV_L1 ); - } - - result = true; - - cvFree( &oob_samples_perm_ptr ); - cvFree( &samples_ptr ); - cvFree( &missing_ptr ); - cvFree( &true_resp_ptr ); - - cvReleaseMat( &sample_idx_for_tree ); - - cvReleaseMat( &oob_sample_votes ); - cvReleaseMat( &oob_responses ); - - __END__; - - return result; -} - -using namespace cv; - -bool CvERTrees::train( const Mat& _train_data, int _tflag, - const Mat& _responses, const Mat& _var_idx, - const Mat& _sample_idx, const Mat& _var_type, - const Mat& _missing_mask, CvRTParams params ) -{ - train_data_hdr = _train_data; - train_data_mat = _train_data; - responses_hdr = _responses; - responses_mat = _responses; - - CvMat vidx = _var_idx, sidx = _sample_idx, vtype = _var_type, mmask = _missing_mask; - - return train(&train_data_hdr, _tflag, &responses_hdr, vidx.data.ptr ? &vidx : 0, - sidx.data.ptr ? &sidx : 0, vtype.data.ptr ? &vtype : 0, - mmask.data.ptr ? &mmask : 0, params); -} - -// End of file. diff --git a/modules/ml/src/estimate.cpp b/modules/ml/src/estimate.cpp deleted file mode 100644 index e9cab881e91c0e590afae5e29c89c047d18658c7..0000000000000000000000000000000000000000 --- a/modules/ml/src/estimate.cpp +++ /dev/null @@ -1,728 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// Intel License Agreement -// -// Copyright (C) 2000, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of Intel Corporation may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#include "precomp.hpp" - -#if 0 - -ML_IMPL int -icvCmpIntegers (const void* a, const void* b) {return *(const int*)a - *(const int*)b;} - -/****************************************************************************************\ -* Cross-validation algorithms realizations * -\****************************************************************************************/ - -// Return pointer to trainIdx. Function DOES NOT FILL this matrix! -ML_IMPL -const CvMat* cvCrossValGetTrainIdxMatrix (const CvStatModel* estimateModel) -{ - CvMat* result = NULL; - - CV_FUNCNAME ("cvCrossValGetTrainIdxMatrix"); - __BEGIN__ - - if (!CV_IS_CROSSVAL(estimateModel)) - { - CV_ERROR (CV_StsBadArg, "Pointer point to not CvCrossValidationModel"); - } - - result = ((CvCrossValidationModel*)estimateModel)->sampleIdxTrain; - - __END__ - - return result; -} // End of cvCrossValGetTrainIdxMatrix - -/****************************************************************************************/ -// Return pointer to checkIdx. Function DOES NOT FILL this matrix! -ML_IMPL -const CvMat* cvCrossValGetCheckIdxMatrix (const CvStatModel* estimateModel) -{ - CvMat* result = NULL; - - CV_FUNCNAME ("cvCrossValGetCheckIdxMatrix"); - __BEGIN__ - - if (!CV_IS_CROSSVAL (estimateModel)) - { - CV_ERROR (CV_StsBadArg, "Pointer point to not CvCrossValidationModel"); - } - - result = ((CvCrossValidationModel*)estimateModel)->sampleIdxEval; - - __END__ - - return result; -} // End of cvCrossValGetCheckIdxMatrix - -/****************************************************************************************/ -// Create new Idx-matrix for next classifiers training and return code of result. -// Result is 0 if function can't make next step (error input or folds are finished), -// it is 1 if all was correct, and it is 2 if current fold wasn't' checked. -ML_IMPL -int cvCrossValNextStep (CvStatModel* estimateModel) -{ - int result = 0; - - CV_FUNCNAME ("cvCrossValGetNextTrainIdx"); - __BEGIN__ - - CvCrossValidationModel* crVal = (CvCrossValidationModel*) estimateModel; - int k, fold; - - if (!CV_IS_CROSSVAL (estimateModel)) - { - CV_ERROR (CV_StsBadArg, "Pointer point to not CvCrossValidationModel"); - } - - fold = ++crVal->current_fold; - - if (fold >= crVal->folds_all) - { - if (fold == crVal->folds_all) - EXIT; - else - { - CV_ERROR (CV_StsInternal, "All iterations has end long ago"); - } - } - - k = crVal->folds[fold + 1] - crVal->folds[fold]; - crVal->sampleIdxTrain->data.i = crVal->sampleIdxAll + crVal->folds[fold + 1]; - crVal->sampleIdxTrain->cols = crVal->samples_all - k; - crVal->sampleIdxEval->data.i = crVal->sampleIdxAll + crVal->folds[fold]; - crVal->sampleIdxEval->cols = k; - - if (crVal->is_checked) - { - crVal->is_checked = 0; - result = 1; - } - else - { - result = 2; - } - - __END__ - - return result; -} - -/****************************************************************************************/ -// Do checking part of loop of cross-validations metod. -ML_IMPL -void cvCrossValCheckClassifier (CvStatModel* estimateModel, - const CvStatModel* model, - const CvMat* trainData, - int sample_t_flag, - const CvMat* trainClasses) -{ - CV_FUNCNAME ("cvCrossValCheckClassifier "); - __BEGIN__ - - CvCrossValidationModel* crVal = (CvCrossValidationModel*) estimateModel; - int i, j, k; - int* data; - float* responses_fl; - int step; - float* responses_result; - int* responses_i; - double te, te1; - double sum_c, sum_p, sum_pp, sum_cp, sum_cc, sq_err; - -// Check input data to correct values. - if (!CV_IS_CROSSVAL (estimateModel)) - { - CV_ERROR (CV_StsBadArg,"First parameter point to not CvCrossValidationModel"); - } - if (!CV_IS_STAT_MODEL (model)) - { - CV_ERROR (CV_StsBadArg, "Second parameter point to not CvStatModel"); - } - if (!CV_IS_MAT (trainData)) - { - CV_ERROR (CV_StsBadArg, "Third parameter point to not CvMat"); - } - if (!CV_IS_MAT (trainClasses)) - { - CV_ERROR (CV_StsBadArg, "Fifth parameter point to not CvMat"); - } - if (crVal->is_checked) - { - CV_ERROR (CV_StsInternal, "This iterations already was checked"); - } - -// Initialize. - k = crVal->sampleIdxEval->cols; - data = crVal->sampleIdxEval->data.i; - -// Eval tested feature vectors. - CV_CALL (cvStatModelMultiPredict (model, trainData, sample_t_flag, - crVal->predict_results, NULL, crVal->sampleIdxEval)); -// Count number if correct results. - responses_result = crVal->predict_results->data.fl; - if (crVal->is_regression) - { - sum_c = sum_p = sum_pp = sum_cp = sum_cc = sq_err = 0; - if (CV_MAT_TYPE (trainClasses->type) == CV_32FC1) - { - responses_fl = trainClasses->data.fl; - step = trainClasses->rows == 1 ? 1 : trainClasses->step / sizeof(float); - for (i = 0; i < k; i++) - { - te = responses_result[*data]; - te1 = responses_fl[*data * step]; - sum_c += te1; - sum_p += te; - sum_cc += te1 * te1; - sum_pp += te * te; - sum_cp += te1 * te; - te -= te1; - sq_err += te * te; - - data++; - } - } - else - { - responses_i = trainClasses->data.i; - step = trainClasses->rows == 1 ? 1 : trainClasses->step / sizeof(int); - for (i = 0; i < k; i++) - { - te = responses_result[*data]; - te1 = responses_i[*data * step]; - sum_c += te1; - sum_p += te; - sum_cc += te1 * te1; - sum_pp += te * te; - sum_cp += te1 * te; - te -= te1; - sq_err += te * te; - - data++; - } - } - // Fixing new internal values of accuracy. - crVal->sum_correct += sum_c; - crVal->sum_predict += sum_p; - crVal->sum_cc += sum_cc; - crVal->sum_pp += sum_pp; - crVal->sum_cp += sum_cp; - crVal->sq_error += sq_err; - } - else - { - if (CV_MAT_TYPE (trainClasses->type) == CV_32FC1) - { - responses_fl = trainClasses->data.fl; - step = trainClasses->rows == 1 ? 1 : trainClasses->step / sizeof(float); - for (i = 0, j = 0; i < k; i++) - { - if (cvRound (responses_result[*data]) == cvRound (responses_fl[*data * step])) - j++; - data++; - } - } - else - { - responses_i = trainClasses->data.i; - step = trainClasses->rows == 1 ? 1 : trainClasses->step / sizeof(int); - for (i = 0, j = 0; i < k; i++) - { - if (cvRound (responses_result[*data]) == responses_i[*data * step]) - j++; - data++; - } - } - // Fixing new internal values of accuracy. - crVal->correct_results += j; - } -// Fixing that this fold already checked. - crVal->all_results += k; - crVal->is_checked = 1; - - __END__ -} // End of cvCrossValCheckClassifier - -/****************************************************************************************/ -// Return current accuracy. -ML_IMPL -float cvCrossValGetResult (const CvStatModel* estimateModel, - float* correlation) -{ - float result = 0; - - CV_FUNCNAME ("cvCrossValGetResult"); - __BEGIN__ - - double te, te1; - CvCrossValidationModel* crVal = (CvCrossValidationModel*)estimateModel; - - if (!CV_IS_CROSSVAL (estimateModel)) - { - CV_ERROR (CV_StsBadArg, "Pointer point to not CvCrossValidationModel"); - } - - if (crVal->all_results) - { - if (crVal->is_regression) - { - result = ((float)crVal->sq_error) / crVal->all_results; - if (correlation) - { - te = crVal->all_results * crVal->sum_cp - - crVal->sum_correct * crVal->sum_predict; - te *= te; - te1 = (crVal->all_results * crVal->sum_cc - - crVal->sum_correct * crVal->sum_correct) * - (crVal->all_results * crVal->sum_pp - - crVal->sum_predict * crVal->sum_predict); - *correlation = (float)(te / te1); - - } - } - else - { - result = ((float)crVal->correct_results) / crVal->all_results; - } - } - - __END__ - - return result; -} - -/****************************************************************************************/ -// Reset cross-validation EstimateModel to state the same as it was immidiatly after -// its creating. -ML_IMPL -void cvCrossValReset (CvStatModel* estimateModel) -{ - CV_FUNCNAME ("cvCrossValReset"); - __BEGIN__ - - CvCrossValidationModel* crVal = (CvCrossValidationModel*)estimateModel; - - if (!CV_IS_CROSSVAL (estimateModel)) - { - CV_ERROR (CV_StsBadArg, "Pointer point to not CvCrossValidationModel"); - } - - crVal->current_fold = -1; - crVal->is_checked = 1; - crVal->all_results = 0; - crVal->correct_results = 0; - crVal->sq_error = 0; - crVal->sum_correct = 0; - crVal->sum_predict = 0; - crVal->sum_cc = 0; - crVal->sum_pp = 0; - crVal->sum_cp = 0; - - __END__ -} - -/****************************************************************************************/ -// This function is standart CvStatModel field to release cross-validation EstimateModel. -ML_IMPL -void cvReleaseCrossValidationModel (CvStatModel** model) -{ - CvCrossValidationModel* pModel; - - CV_FUNCNAME ("cvReleaseCrossValidationModel"); - __BEGIN__ - - if (!model) - { - CV_ERROR (CV_StsNullPtr, ""); - } - - pModel = (CvCrossValidationModel*)*model; - if (!pModel) - { - return; - } - if (!CV_IS_CROSSVAL (pModel)) - { - CV_ERROR (CV_StsBadArg, ""); - } - - cvFree (&pModel->sampleIdxAll); - cvFree (&pModel->folds); - cvReleaseMat (&pModel->sampleIdxEval); - cvReleaseMat (&pModel->sampleIdxTrain); - cvReleaseMat (&pModel->predict_results); - - cvFree (model); - - __END__ -} // End of cvReleaseCrossValidationModel. - -/****************************************************************************************/ -// This function create cross-validation EstimateModel. -ML_IMPL CvStatModel* -cvCreateCrossValidationEstimateModel( - int samples_all, - const CvStatModelParams* estimateParams, - const CvMat* sampleIdx) -{ - CvStatModel* model = NULL; - CvCrossValidationModel* crVal = NULL; - - CV_FUNCNAME ("cvCreateCrossValidationEstimateModel"); - __BEGIN__ - - int k_fold = 10; - - int i, j, k, s_len; - int samples_selected; - CvRNG rng; - CvRNG* prng; - int* res_s_data; - int* te_s_data; - int* folds; - - rng = cvRNG(cvGetTickCount()); - cvRandInt (&rng); cvRandInt (&rng); cvRandInt (&rng); cvRandInt (&rng); -// Check input parameters. - if (estimateParams) - k_fold = ((CvCrossValidationParams*)estimateParams)->k_fold; - if (!k_fold) - { - CV_ERROR (CV_StsBadArg, "Error in parameters of cross-validation (k_fold == 0)!"); - } - if (samples_all <= 0) - { - CV_ERROR (CV_StsBadArg, " should be positive!"); - } - -// Alloc memory and fill standart StatModel's fields. - CV_CALL (crVal = (CvCrossValidationModel*)cvCreateStatModel ( - CV_STAT_MODEL_MAGIC_VAL | CV_CROSSVAL_MAGIC_VAL, - sizeof(CvCrossValidationModel), - cvReleaseCrossValidationModel, - NULL, NULL)); - crVal->current_fold = -1; - crVal->folds_all = k_fold; - if (estimateParams && ((CvCrossValidationParams*)estimateParams)->is_regression) - crVal->is_regression = 1; - else - crVal->is_regression = 0; - if (estimateParams && ((CvCrossValidationParams*)estimateParams)->rng) - prng = ((CvCrossValidationParams*)estimateParams)->rng; - else - prng = &rng; - - // Check and preprocess sample indices. - if (sampleIdx) - { - int s_step; - int s_type = 0; - - if (!CV_IS_MAT (sampleIdx)) - CV_ERROR (CV_StsBadArg, "Invalid sampleIdx array"); - - if (sampleIdx->rows != 1 && sampleIdx->cols != 1) - CV_ERROR (CV_StsBadSize, "sampleIdx array must be 1-dimensional"); - - s_len = sampleIdx->rows + sampleIdx->cols - 1; - s_step = sampleIdx->rows == 1 ? - 1 : sampleIdx->step / CV_ELEM_SIZE(sampleIdx->type); - - s_type = CV_MAT_TYPE (sampleIdx->type); - - switch (s_type) - { - case CV_8UC1: - case CV_8SC1: - { - uchar* s_data = sampleIdx->data.ptr; - - // sampleIdx is array of 1's and 0's - - // i.e. it is a mask of the selected samples - if( s_len != samples_all ) - CV_ERROR (CV_StsUnmatchedSizes, - "Sample mask should contain as many elements as the total number of samples"); - - samples_selected = 0; - for (i = 0; i < s_len; i++) - samples_selected += s_data[i * s_step] != 0; - - if (samples_selected == 0) - CV_ERROR (CV_StsOutOfRange, "No samples is selected!"); - } - s_len = samples_selected; - break; - case CV_32SC1: - if (s_len > samples_all) - CV_ERROR (CV_StsOutOfRange, - "sampleIdx array may not contain more elements than the total number of samples"); - samples_selected = s_len; - break; - default: - CV_ERROR (CV_StsUnsupportedFormat, "Unsupported sampleIdx array data type " - "(it should be 8uC1, 8sC1 or 32sC1)"); - } - - // Alloc additional memory for internal Idx and fill it. -/*!!*/ CV_CALL (res_s_data = crVal->sampleIdxAll = - (int*)cvAlloc (2 * s_len * sizeof(int))); - - if (s_type < CV_32SC1) - { - uchar* s_data = sampleIdx->data.ptr; - for (i = 0; i < s_len; i++) - if (s_data[i * s_step]) - { - *res_s_data++ = i; - } - res_s_data = crVal->sampleIdxAll; - } - else - { - int* s_data = sampleIdx->data.i; - int out_of_order = 0; - - for (i = 0; i < s_len; i++) - { - res_s_data[i] = s_data[i * s_step]; - if (i > 0 && res_s_data[i] < res_s_data[i - 1]) - out_of_order = 1; - } - - if (out_of_order) - qsort (res_s_data, s_len, sizeof(res_s_data[0]), icvCmpIntegers); - - if (res_s_data[0] < 0 || - res_s_data[s_len - 1] >= samples_all) - CV_ERROR (CV_StsBadArg, "There are out-of-range sample indices"); - for (i = 1; i < s_len; i++) - if (res_s_data[i] <= res_s_data[i - 1]) - CV_ERROR (CV_StsBadArg, "There are duplicated"); - } - } - else // if (sampleIdx) - { - // Alloc additional memory for internal Idx and fill it. - s_len = samples_all; - CV_CALL (res_s_data = crVal->sampleIdxAll = (int*)cvAlloc (2 * s_len * sizeof(int))); - for (i = 0; i < s_len; i++) - { - *res_s_data++ = i; - } - res_s_data = crVal->sampleIdxAll; - } // if (sampleIdx) ... else - -// Resort internal Idx. - te_s_data = res_s_data + s_len; - for (i = s_len; i > 1; i--) - { - j = cvRandInt (prng) % i; - k = *(--te_s_data); - *te_s_data = res_s_data[j]; - res_s_data[j] = k; - } - -// Duplicate resorted internal Idx. -// It will be used to simplify operation of getting trainIdx. - te_s_data = res_s_data + s_len; - for (i = 0; i < s_len; i++) - { - *te_s_data++ = *res_s_data++; - } - -// Cut sampleIdxAll to parts. - if (k_fold > 0) - { - if (k_fold > s_len) - { - CV_ERROR (CV_StsBadArg, - "Error in parameters of cross-validation ('k_fold' > #samples)!"); - } - folds = crVal->folds = (int*) cvAlloc ((k_fold + 1) * sizeof (int)); - *folds++ = 0; - for (i = 1; i < k_fold; i++) - { - *folds++ = cvRound (i * s_len * 1. / k_fold); - } - *folds = s_len; - folds = crVal->folds; - - crVal->max_fold_size = (s_len - 1) / k_fold + 1; - } - else - { - k = -k_fold; - crVal->max_fold_size = k; - if (k >= s_len) - { - CV_ERROR (CV_StsBadArg, - "Error in parameters of cross-validation (-'k_fold' > #samples)!"); - } - crVal->folds_all = k = (s_len - 1) / k + 1; - - folds = crVal->folds = (int*) cvAlloc ((k + 1) * sizeof (int)); - for (i = 0; i < k; i++) - { - *folds++ = -i * k_fold; - } - *folds = s_len; - folds = crVal->folds; - } - -// Prepare other internal fields to working. - CV_CALL (crVal->predict_results = cvCreateMat (1, samples_all, CV_32FC1)); - CV_CALL (crVal->sampleIdxEval = cvCreateMatHeader (1, 1, CV_32SC1)); - CV_CALL (crVal->sampleIdxTrain = cvCreateMatHeader (1, 1, CV_32SC1)); - crVal->sampleIdxEval->cols = 0; - crVal->sampleIdxTrain->cols = 0; - crVal->samples_all = s_len; - crVal->is_checked = 1; - - crVal->getTrainIdxMat = cvCrossValGetTrainIdxMatrix; - crVal->getCheckIdxMat = cvCrossValGetCheckIdxMatrix; - crVal->nextStep = cvCrossValNextStep; - crVal->check = cvCrossValCheckClassifier; - crVal->getResult = cvCrossValGetResult; - crVal->reset = cvCrossValReset; - - model = (CvStatModel*)crVal; - - __END__ - - if (!model) - { - cvReleaseCrossValidationModel ((CvStatModel**)&crVal); - } - - return model; -} // End of cvCreateCrossValidationEstimateModel - - -/****************************************************************************************\ -* Extended interface with backcalls for models * -\****************************************************************************************/ -ML_IMPL float -cvCrossValidation (const CvMat* trueData, - int tflag, - const CvMat* trueClasses, - CvStatModel* (*createClassifier) (const CvMat*, - int, - const CvMat*, - const CvClassifierTrainParams*, - const CvMat*, - const CvMat*, - const CvMat*, - const CvMat*), - const CvClassifierTrainParams* estimateParams, - const CvClassifierTrainParams* trainParams, - const CvMat* compIdx, - const CvMat* sampleIdx, - CvStatModel** pCrValModel, - const CvMat* typeMask, - const CvMat* missedMeasurementMask) -{ - CvCrossValidationModel* crVal = NULL; - float result = 0; - CvStatModel* pClassifier = NULL; - - CV_FUNCNAME ("cvCrossValidation"); - __BEGIN__ - - const CvMat* trainDataIdx; - int samples_all; - -// checking input data - if ((createClassifier) == NULL) - { - CV_ERROR (CV_StsNullPtr, "Null pointer to functiion which create classifier"); - } - if (pCrValModel && *pCrValModel && !CV_IS_CROSSVAL(*pCrValModel)) - { - CV_ERROR (CV_StsBadArg, - " point to not cross-validation model"); - } - -// initialization - if (pCrValModel && *pCrValModel) - { - crVal = (CvCrossValidationModel*)*pCrValModel; - crVal->reset ((CvStatModel*)crVal); - } - else - { - samples_all = ((tflag) ? trueData->rows : trueData->cols); - CV_CALL (crVal = (CvCrossValidationModel*) - cvCreateCrossValidationEstimateModel (samples_all, estimateParams, sampleIdx)); - } - - CV_CALL (trainDataIdx = crVal->getTrainIdxMat ((CvStatModel*)crVal)); - -// operation loop - for (; crVal->nextStep((CvStatModel*)crVal) != 0; ) - { - CV_CALL (pClassifier = createClassifier (trueData, tflag, trueClasses, - trainParams, compIdx, trainDataIdx, typeMask, missedMeasurementMask)); - CV_CALL (crVal->check ((CvStatModel*)crVal, pClassifier, - trueData, tflag, trueClasses)); - - pClassifier->release (&pClassifier); - } - -// Get result and fill output field. - CV_CALL (result = crVal->getResult ((CvStatModel*)crVal, 0)); - - if (pCrValModel && !*pCrValModel) - *pCrValModel = (CvStatModel*)crVal; - - __END__ - -// Free all memory that should be freed. - if (pClassifier) - pClassifier->release (&pClassifier); - if (crVal && (!pCrValModel || !*pCrValModel)) - crVal->release ((CvStatModel**)&crVal); - - return result; -} // End of cvCrossValidation - -#endif - -/* End of file */ diff --git a/modules/ml/src/gbt.cpp b/modules/ml/src/gbt.cpp index 42d0d4f3af874ecc105edf3e03eb38b11267d04e..b186abf672b067bdafde8256a6cda3767f078717 100644 --- a/modules/ml/src/gbt.cpp +++ b/modules/ml/src/gbt.cpp @@ -2,6 +2,8 @@ #include "precomp.hpp" #include +#if 0 + #define pCvSeq CvSeq* #define pCvDTreeNode CvDTreeNode* @@ -1359,3 +1361,5 @@ float CvGBTrees::predict( const cv::Mat& sample, const cv::Mat& _missing, return predict(&_sample, _missing.empty() ? 0 : &miss, 0, slice==cv::Range::all() ? CV_WHOLE_SEQ : cvSlice(slice.start, slice.end), k); } + +#endif diff --git a/modules/ml/src/inner_functions.cpp b/modules/ml/src/inner_functions.cpp index f0e085da6c3a13d1ba2eea56c4feb62e6c75fd93..3d5f335201e1618e762935145ceeabf1c5ca1a5f 100644 --- a/modules/ml/src/inner_functions.cpp +++ b/modules/ml/src/inner_functions.cpp @@ -40,1840 +40,143 @@ #include "precomp.hpp" +namespace cv { namespace ml { -CvStatModel::CvStatModel() +ParamGrid::ParamGrid() { minVal = maxVal = 0.; logStep = 1; } +ParamGrid::ParamGrid(double _minVal, double _maxVal, double _logStep) { - default_model_name = "my_stat_model"; + minVal = std::min(_minVal, _maxVal); + maxVal = std::max(_minVal, _maxVal); + logStep = std::max(_logStep, 1.); } +void StatModel::clear() {} -CvStatModel::~CvStatModel() -{ - clear(); -} +int StatModel::getVarCount() const { return 0; } - -void CvStatModel::clear() +bool StatModel::train( const Ptr&, int ) { + CV_Error(CV_StsNotImplemented, ""); + return false; } - -void CvStatModel::save( const char* filename, const char* name ) const +bool StatModel::train( InputArray samples, int layout, InputArray responses ) { - CvFileStorage* fs = 0; - - CV_FUNCNAME( "CvStatModel::save" ); - - __BEGIN__; - - CV_CALL( fs = cvOpenFileStorage( filename, 0, CV_STORAGE_WRITE )); - if( !fs ) - CV_ERROR( CV_StsError, "Could not open the file storage. Check the path and permissions" ); - - write( fs, name ? name : default_model_name ); - - __END__; - - cvReleaseFileStorage( &fs ); + return train(TrainData::create(samples, layout, responses)); } - -void CvStatModel::load( const char* filename, const char* name ) +float StatModel::calcError( const Ptr& data, bool testerr, OutputArray _resp ) const { - CvFileStorage* fs = 0; - - CV_FUNCNAME( "CvStatModel::load" ); + Mat samples = data->getSamples(); + int layout = data->getLayout(); + Mat sidx = testerr ? data->getTestSampleIdx() : data->getTrainSampleIdx(); + const int* sidx_ptr = sidx.ptr(); + int i, n = (int)sidx.total(); + bool isclassifier = isClassifier(); + Mat responses = data->getResponses(); - __BEGIN__; + if( n == 0 ) + n = data->getNSamples(); - CvFileNode* model_node = 0; + if( n == 0 ) + return -FLT_MAX; - CV_CALL( fs = cvOpenFileStorage( filename, 0, CV_STORAGE_READ )); - if( !fs ) - EXIT; + Mat resp; + if( _resp.needed() ) + resp.create(n, 1, CV_32F); - if( name ) - model_node = cvGetFileNodeByName( fs, 0, name ); - else + double err = 0; + for( i = 0; i < n; i++ ) { - CvFileNode* root = cvGetRootFileNode( fs ); - if( root->data.seq->total > 0 ) - model_node = (CvFileNode*)cvGetSeqElem( root->data.seq, 0 ); - } - - read( fs, model_node ); + int si = sidx_ptr ? sidx_ptr[i] : i; + Mat sample = layout == ROW_SAMPLE ? samples.row(si) : samples.col(si); + float val = predict(sample); + float val0 = responses.at(si); - __END__; - - cvReleaseFileStorage( &fs ); -} + if( isclassifier ) + err += fabs(val - val0) > FLT_EPSILON; + else + err += (val - val0)*(val - val0); + if( resp.data ) + resp.at(i) = val; + /*if( i < 100 ) + { + printf("%d. ref %.1f vs pred %.1f\n", i, val0, val); + }*/ + } + if( _resp.needed() ) + resp.copyTo(_resp); -void CvStatModel::write( CvFileStorage*, const char* ) const -{ - OPENCV_ERROR( CV_StsNotImplemented, "CvStatModel::write", "" ); + return (float)(err / n * (isclassifier ? 100 : 1)); } - -void CvStatModel::read( CvFileStorage*, CvFileNode* ) +void StatModel::save(const String& filename) const { - OPENCV_ERROR( CV_StsNotImplemented, "CvStatModel::read", "" ); + FileStorage fs(filename, FileStorage::WRITE); + fs << getDefaultModelName() << "{"; + write(fs); + fs << "}"; } - /* Calculates upper triangular matrix S, where A is a symmetrical matrix A=S'*S */ -static void cvChol( CvMat* A, CvMat* S ) +static void Cholesky( const Mat& A, Mat& S ) { - int dim = A->rows; + CV_Assert(A.type() == CV_32F); + + int dim = A.rows; + S.create(dim, dim, CV_32F); int i, j, k; - float sum; for( i = 0; i < dim; i++ ) { for( j = 0; j < i; j++ ) - CV_MAT_ELEM(*S, float, i, j) = 0; + S.at(i,j) = 0.f; - sum = 0; + float sum = 0.f; for( k = 0; k < i; k++ ) - sum += CV_MAT_ELEM(*S, float, k, i) * CV_MAT_ELEM(*S, float, k, i); + { + float val = S.at(k,i); + sum += val*val; + } - CV_MAT_ELEM(*S, float, i, i) = (float)sqrt(CV_MAT_ELEM(*A, float, i, i) - sum); + S.at(i,i) = std::sqrt(std::max(A.at(i,i) - sum, 0.f)); + float ival = 1.f/S.at(i, i); for( j = i + 1; j < dim; j++ ) { sum = 0; for( k = 0; k < i; k++ ) - sum += CV_MAT_ELEM(*S, float, k, i) * CV_MAT_ELEM(*S, float, k, j); - - CV_MAT_ELEM(*S, float, i, j) = - (CV_MAT_ELEM(*A, float, i, j) - sum) / CV_MAT_ELEM(*S, float, i, i); + sum += S.at(k, i) * S.at(k, j); + S.at(i, j) = (A.at(i, j) - sum)*ival; } } } /* Generates from multivariate normal distribution, where - is an average row vector, - symmetric covariation matrix */ -CV_IMPL void cvRandMVNormal( CvMat* mean, CvMat* cov, CvMat* sample, CvRNG* rng ) -{ - int dim = sample->cols; - int amount = sample->rows; - - CvRNG state = rng ? *rng : cvRNG( cvGetTickCount() ); - cvRandArr(&state, sample, CV_RAND_NORMAL, cvScalarAll(0), cvScalarAll(1) ); - - CvMat* utmat = cvCreateMat(dim, dim, sample->type); - CvMat* vect = cvCreateMatHeader(1, dim, sample->type); - - cvChol(cov, utmat); - - int i; - for( i = 0; i < amount; i++ ) - { - cvGetRow(sample, vect, i); - cvMatMulAdd(vect, utmat, mean, vect); - } - - cvReleaseMat(&vect); - cvReleaseMat(&utmat); -} - - -/* Generates of points from a discrete variate xi, - where Pr{xi = k} == probs[k], 0 < k < len - 1. */ -static void cvRandSeries( float probs[], int len, int sample[], int amount ) -{ - CvMat* univals = cvCreateMat(1, amount, CV_32FC1); - float* knots = (float*)cvAlloc( len * sizeof(float) ); - - int i, j; - - CvRNG state = cvRNG(-1); - cvRandArr(&state, univals, CV_RAND_UNI, cvScalarAll(0), cvScalarAll(1) ); - - knots[0] = probs[0]; - for( i = 1; i < len; i++ ) - knots[i] = knots[i - 1] + probs[i]; - - for( i = 0; i < amount; i++ ) - for( j = 0; j < len; j++ ) - { - if ( CV_MAT_ELEM(*univals, float, 0, i) <= knots[j] ) - { - sample[i] = j; - break; - } - } - - cvFree(&knots); -} - -/* Generates from gaussian mixture distribution */ -CV_IMPL void cvRandGaussMixture( CvMat* means[], - CvMat* covs[], - float weights[], - int clsnum, - CvMat* sample, - CvMat* sampClasses ) -{ - int dim = sample->cols; - int amount = sample->rows; - - int i, clss; - - int* sample_clsnum = (int*)cvAlloc( amount * sizeof(int) ); - CvMat** utmats = (CvMat**)cvAlloc( clsnum * sizeof(CvMat*) ); - CvMat* vect = cvCreateMatHeader(1, dim, CV_32FC1); - - CvMat* classes; - if( sampClasses ) - classes = sampClasses; - else - classes = cvCreateMat(1, amount, CV_32FC1); - - CvRNG state = cvRNG(-1); - cvRandArr(&state, sample, CV_RAND_NORMAL, cvScalarAll(0), cvScalarAll(1)); - - cvRandSeries(weights, clsnum, sample_clsnum, amount); - - for( i = 0; i < clsnum; i++ ) - { - utmats[i] = cvCreateMat(dim, dim, CV_32FC1); - cvChol(covs[i], utmats[i]); - } - - for( i = 0; i < amount; i++ ) - { - CV_MAT_ELEM(*classes, float, 0, i) = (float)sample_clsnum[i]; - cvGetRow(sample, vect, i); - clss = sample_clsnum[i]; - cvMatMulAdd(vect, utmats[clss], means[clss], vect); - } - - if( !sampClasses ) - cvReleaseMat(&classes); - for( i = 0; i < clsnum; i++ ) - cvReleaseMat(&utmats[i]); - cvFree(&utmats); - cvFree(&sample_clsnum); - cvReleaseMat(&vect); -} - - -CvMat* icvGenerateRandomClusterCenters ( int seed, const CvMat* data, - int num_of_clusters, CvMat* _centers ) -{ - CvMat* centers = _centers; - - CV_FUNCNAME("icvGenerateRandomClusterCenters"); - __BEGIN__; - - CvRNG rng; - CvMat data_comp, centers_comp; - CvPoint minLoc, maxLoc; // Not used, just for function "cvMinMaxLoc" - double minVal, maxVal; - int i; - int dim = data ? data->cols : 0; - - if( ICV_IS_MAT_OF_TYPE(data, CV_32FC1) ) - { - if( _centers && !ICV_IS_MAT_OF_TYPE (_centers, CV_32FC1) ) - { - CV_ERROR(CV_StsBadArg,""); - } - else if( !_centers ) - CV_CALL(centers = cvCreateMat (num_of_clusters, dim, CV_32FC1)); - } - else if( ICV_IS_MAT_OF_TYPE(data, CV_64FC1) ) - { - if( _centers && !ICV_IS_MAT_OF_TYPE (_centers, CV_64FC1) ) - { - CV_ERROR(CV_StsBadArg,""); - } - else if( !_centers ) - CV_CALL(centers = cvCreateMat (num_of_clusters, dim, CV_64FC1)); - } - else - CV_ERROR (CV_StsBadArg,""); - - if( num_of_clusters < 1 ) - CV_ERROR (CV_StsBadArg,""); - - rng = cvRNG(seed); - for (i = 0; i < dim; i++) - { - CV_CALL(cvGetCol (data, &data_comp, i)); - CV_CALL(cvMinMaxLoc (&data_comp, &minVal, &maxVal, &minLoc, &maxLoc)); - CV_CALL(cvGetCol (centers, ¢ers_comp, i)); - CV_CALL(cvRandArr (&rng, ¢ers_comp, CV_RAND_UNI, cvScalarAll(minVal), cvScalarAll(maxVal))); - } - - __END__; - - if( (cvGetErrStatus () < 0) || (centers != _centers) ) - cvReleaseMat (¢ers); - - return _centers ? _centers : centers; -} // end of icvGenerateRandomClusterCenters - -// By S. Dilman - begin - - -#define ICV_RAND_MAX 4294967296 // == 2^32 - -// static void cvRandRoundUni (CvMat* center, -// float radius_small, -// float radius_large, -// CvMat* desired_matrix, -// CvRNG* rng_state_ptr) -// { -// float rad, norm, coefficient; -// int dim, size, i, j; -// CvMat *cov, sample; -// CvRNG rng_local; - -// CV_FUNCNAME("cvRandRoundUni"); -// __BEGIN__ - -// rng_local = *rng_state_ptr; - -// CV_ASSERT ((radius_small >= 0) && -// (radius_large > 0) && -// (radius_small <= radius_large)); -// CV_ASSERT (center && desired_matrix && rng_state_ptr); -// CV_ASSERT (center->rows == 1); -// CV_ASSERT (center->cols == desired_matrix->cols); - -// dim = desired_matrix->cols; -// size = desired_matrix->rows; -// cov = cvCreateMat (dim, dim, CV_32FC1); -// cvSetIdentity (cov); -// cvRandMVNormal (center, cov, desired_matrix, &rng_local); - -// for (i = 0; i < size; i++) -// { -// rad = (float)(cvRandReal(&rng_local)*(radius_large - radius_small) + radius_small); -// cvGetRow (desired_matrix, &sample, i); -// norm = (float) cvNorm (&sample, 0, CV_L2); -// coefficient = rad / norm; -// for (j = 0; j < dim; j++) -// CV_MAT_ELEM (sample, float, 0, j) *= coefficient; -// } - -// __END__ - -// } - -// By S. Dilman - end - - -static int CV_CDECL -icvCmpIntegers( const void* a, const void* b ) -{ - return *(const int*)a - *(const int*)b; -} - - -static int CV_CDECL -icvCmpIntegersPtr( const void* _a, const void* _b ) -{ - int a = **(const int**)_a; - int b = **(const int**)_b; - return (a < b ? -1 : 0)|(a > b); -} - - -static int icvCmpSparseVecElems( const void* a, const void* b ) -{ - return ((CvSparseVecElem32f*)a)->idx - ((CvSparseVecElem32f*)b)->idx; -} - - -CvMat* -cvPreprocessIndexArray( const CvMat* idx_arr, int data_arr_size, bool check_for_duplicates ) -{ - CvMat* idx = 0; - - CV_FUNCNAME( "cvPreprocessIndexArray" ); - - __BEGIN__; - - int i, idx_total, idx_selected = 0, step, type, prev = INT_MIN, is_sorted = 1; - uchar* srcb = 0; - int* srci = 0; - int* dsti; - - if( !CV_IS_MAT(idx_arr) ) - CV_ERROR( CV_StsBadArg, "Invalid index array" ); - - if( idx_arr->rows != 1 && idx_arr->cols != 1 ) - CV_ERROR( CV_StsBadSize, "the index array must be 1-dimensional" ); - - idx_total = idx_arr->rows + idx_arr->cols - 1; - srcb = idx_arr->data.ptr; - srci = idx_arr->data.i; - - type = CV_MAT_TYPE(idx_arr->type); - step = CV_IS_MAT_CONT(idx_arr->type) ? 1 : idx_arr->step/CV_ELEM_SIZE(type); - - switch( type ) - { - case CV_8UC1: - case CV_8SC1: - // idx_arr is array of 1's and 0's - - // i.e. it is a mask of the selected components - if( idx_total != data_arr_size ) - CV_ERROR( CV_StsUnmatchedSizes, - "Component mask should contain as many elements as the total number of input variables" ); - - for( i = 0; i < idx_total; i++ ) - idx_selected += srcb[i*step] != 0; - - if( idx_selected == 0 ) - CV_ERROR( CV_StsOutOfRange, "No components/input_variables is selected!" ); - - break; - case CV_32SC1: - // idx_arr is array of integer indices of selected components - if( idx_total > data_arr_size ) - CV_ERROR( CV_StsOutOfRange, - "index array may not contain more elements than the total number of input variables" ); - idx_selected = idx_total; - // check if sorted already - for( i = 0; i < idx_total; i++ ) - { - int val = srci[i*step]; - if( val >= prev ) - { - is_sorted = 0; - break; - } - prev = val; - } - break; - default: - CV_ERROR( CV_StsUnsupportedFormat, "Unsupported index array data type " - "(it should be 8uC1, 8sC1 or 32sC1)" ); - } - - CV_CALL( idx = cvCreateMat( 1, idx_selected, CV_32SC1 )); - dsti = idx->data.i; - - if( type < CV_32SC1 ) - { - for( i = 0; i < idx_total; i++ ) - if( srcb[i*step] ) - *dsti++ = i; - } - else - { - for( i = 0; i < idx_total; i++ ) - dsti[i] = srci[i*step]; - - if( !is_sorted ) - qsort( dsti, idx_total, sizeof(dsti[0]), icvCmpIntegers ); - - if( dsti[0] < 0 || dsti[idx_total-1] >= data_arr_size ) - CV_ERROR( CV_StsOutOfRange, "the index array elements are out of range" ); - - if( check_for_duplicates ) - { - for( i = 1; i < idx_total; i++ ) - if( dsti[i] <= dsti[i-1] ) - CV_ERROR( CV_StsBadArg, "There are duplicated index array elements" ); - } - } - - __END__; - - if( cvGetErrStatus() < 0 ) - cvReleaseMat( &idx ); - - return idx; -} - - -CvMat* -cvPreprocessVarType( const CvMat* var_type, const CvMat* var_idx, - int var_count, int* response_type ) -{ - CvMat* out_var_type = 0; - CV_FUNCNAME( "cvPreprocessVarType" ); - - if( response_type ) - *response_type = -1; - - __BEGIN__; - - int i, tm_size, tm_step; - //int* map = 0; - const uchar* src; - uchar* dst; - - if( !CV_IS_MAT(var_type) ) - CV_ERROR( var_type ? CV_StsBadArg : CV_StsNullPtr, "Invalid or absent var_type array" ); - - if( var_type->rows != 1 && var_type->cols != 1 ) - CV_ERROR( CV_StsBadSize, "var_type array must be 1-dimensional" ); - - if( !CV_IS_MASK_ARR(var_type)) - CV_ERROR( CV_StsUnsupportedFormat, "type mask must be 8uC1 or 8sC1 array" ); - - tm_size = var_type->rows + var_type->cols - 1; - tm_step = var_type->rows == 1 ? 1 : var_type->step/CV_ELEM_SIZE(var_type->type); - - if( /*tm_size != var_count &&*/ tm_size != var_count + 1 ) - CV_ERROR( CV_StsBadArg, - "type mask must be of + 1 size" ); - - if( response_type && tm_size > var_count ) - *response_type = var_type->data.ptr[var_count*tm_step] != 0; - - if( var_idx ) - { - if( !CV_IS_MAT(var_idx) || CV_MAT_TYPE(var_idx->type) != CV_32SC1 || - (var_idx->rows != 1 && var_idx->cols != 1) || !CV_IS_MAT_CONT(var_idx->type) ) - CV_ERROR( CV_StsBadArg, "var index array should be continuous 1-dimensional integer vector" ); - if( var_idx->rows + var_idx->cols - 1 > var_count ) - CV_ERROR( CV_StsBadSize, "var index array is too large" ); - //map = var_idx->data.i; - var_count = var_idx->rows + var_idx->cols - 1; - } - - CV_CALL( out_var_type = cvCreateMat( 1, var_count, CV_8UC1 )); - src = var_type->data.ptr; - dst = out_var_type->data.ptr; - - for( i = 0; i < var_count; i++ ) - { - //int idx = map ? map[i] : i; - assert( (unsigned)/*idx*/i < (unsigned)tm_size ); - dst[i] = (uchar)(src[/*idx*/i*tm_step] != 0); - } - - __END__; - - return out_var_type; -} - - -CvMat* -cvPreprocessOrderedResponses( const CvMat* responses, const CvMat* sample_idx, int sample_all ) -{ - CvMat* out_responses = 0; - - CV_FUNCNAME( "cvPreprocessOrderedResponses" ); - - __BEGIN__; - - int i, r_type, r_step; - const int* map = 0; - float* dst; - int sample_count = sample_all; - - if( !CV_IS_MAT(responses) ) - CV_ERROR( CV_StsBadArg, "Invalid response array" ); - - if( responses->rows != 1 && responses->cols != 1 ) - CV_ERROR( CV_StsBadSize, "Response array must be 1-dimensional" ); - - if( responses->rows + responses->cols - 1 != sample_count ) - CV_ERROR( CV_StsUnmatchedSizes, - "Response array must contain as many elements as the total number of samples" ); - - r_type = CV_MAT_TYPE(responses->type); - if( r_type != CV_32FC1 && r_type != CV_32SC1 ) - CV_ERROR( CV_StsUnsupportedFormat, "Unsupported response type" ); - - r_step = responses->step ? responses->step / CV_ELEM_SIZE(responses->type) : 1; - - if( r_type == CV_32FC1 && CV_IS_MAT_CONT(responses->type) && !sample_idx ) - { - out_responses = cvCloneMat( responses ); - EXIT; - } - - if( sample_idx ) - { - if( !CV_IS_MAT(sample_idx) || CV_MAT_TYPE(sample_idx->type) != CV_32SC1 || - (sample_idx->rows != 1 && sample_idx->cols != 1) || !CV_IS_MAT_CONT(sample_idx->type) ) - CV_ERROR( CV_StsBadArg, "sample index array should be continuous 1-dimensional integer vector" ); - if( sample_idx->rows + sample_idx->cols - 1 > sample_count ) - CV_ERROR( CV_StsBadSize, "sample index array is too large" ); - map = sample_idx->data.i; - sample_count = sample_idx->rows + sample_idx->cols - 1; - } - - CV_CALL( out_responses = cvCreateMat( 1, sample_count, CV_32FC1 )); - - dst = out_responses->data.fl; - if( r_type == CV_32FC1 ) - { - const float* src = responses->data.fl; - for( i = 0; i < sample_count; i++ ) - { - int idx = map ? map[i] : i; - assert( (unsigned)idx < (unsigned)sample_all ); - dst[i] = src[idx*r_step]; - } - } - else - { - const int* src = responses->data.i; - for( i = 0; i < sample_count; i++ ) - { - int idx = map ? map[i] : i; - assert( (unsigned)idx < (unsigned)sample_all ); - dst[i] = (float)src[idx*r_step]; - } - } - - __END__; - - return out_responses; -} - -CvMat* -cvPreprocessCategoricalResponses( const CvMat* responses, - const CvMat* sample_idx, int sample_all, - CvMat** out_response_map, CvMat** class_counts ) -{ - CvMat* out_responses = 0; - int** response_ptr = 0; - - CV_FUNCNAME( "cvPreprocessCategoricalResponses" ); - - if( out_response_map ) - *out_response_map = 0; - - if( class_counts ) - *class_counts = 0; - - __BEGIN__; - - int i, r_type, r_step; - int cls_count = 1, prev_cls, prev_i; - const int* map = 0; - const int* srci; - const float* srcfl; - int* dst; - int* cls_map; - int* cls_counts = 0; - int sample_count = sample_all; - - if( !CV_IS_MAT(responses) ) - CV_ERROR( CV_StsBadArg, "Invalid response array" ); - - if( responses->rows != 1 && responses->cols != 1 ) - CV_ERROR( CV_StsBadSize, "Response array must be 1-dimensional" ); - - if( responses->rows + responses->cols - 1 != sample_count ) - CV_ERROR( CV_StsUnmatchedSizes, - "Response array must contain as many elements as the total number of samples" ); - - r_type = CV_MAT_TYPE(responses->type); - if( r_type != CV_32FC1 && r_type != CV_32SC1 ) - CV_ERROR( CV_StsUnsupportedFormat, "Unsupported response type" ); - - r_step = responses->rows == 1 ? 1 : responses->step / CV_ELEM_SIZE(responses->type); - - if( sample_idx ) - { - if( !CV_IS_MAT(sample_idx) || CV_MAT_TYPE(sample_idx->type) != CV_32SC1 || - (sample_idx->rows != 1 && sample_idx->cols != 1) || !CV_IS_MAT_CONT(sample_idx->type) ) - CV_ERROR( CV_StsBadArg, "sample index array should be continuous 1-dimensional integer vector" ); - if( sample_idx->rows + sample_idx->cols - 1 > sample_count ) - CV_ERROR( CV_StsBadSize, "sample index array is too large" ); - map = sample_idx->data.i; - sample_count = sample_idx->rows + sample_idx->cols - 1; - } - - CV_CALL( out_responses = cvCreateMat( 1, sample_count, CV_32SC1 )); - - if( !out_response_map ) - CV_ERROR( CV_StsNullPtr, "out_response_map pointer is NULL" ); - - CV_CALL( response_ptr = (int**)cvAlloc( sample_count*sizeof(response_ptr[0]))); - - srci = responses->data.i; - srcfl = responses->data.fl; - dst = out_responses->data.i; - - for( i = 0; i < sample_count; i++ ) - { - int idx = map ? map[i] : i; - assert( (unsigned)idx < (unsigned)sample_all ); - if( r_type == CV_32SC1 ) - dst[i] = srci[idx*r_step]; - else - { - float rf = srcfl[idx*r_step]; - int ri = cvRound(rf); - if( ri != rf ) - { - char buf[100]; - sprintf( buf, "response #%d is not integral", idx ); - CV_ERROR( CV_StsBadArg, buf ); - } - dst[i] = ri; - } - response_ptr[i] = dst + i; - } - - qsort( response_ptr, sample_count, sizeof(int*), icvCmpIntegersPtr ); - - // count the classes - for( i = 1; i < sample_count; i++ ) - cls_count += *response_ptr[i] != *response_ptr[i-1]; - - if( cls_count < 2 ) - CV_ERROR( CV_StsBadArg, "There is only a single class" ); - - CV_CALL( *out_response_map = cvCreateMat( 1, cls_count, CV_32SC1 )); - - if( class_counts ) - { - CV_CALL( *class_counts = cvCreateMat( 1, cls_count, CV_32SC1 )); - cls_counts = (*class_counts)->data.i; - } - - // compact the class indices and build the map - prev_cls = ~*response_ptr[0]; - cls_count = -1; - cls_map = (*out_response_map)->data.i; - - for( i = 0, prev_i = -1; i < sample_count; i++ ) - { - int cur_cls = *response_ptr[i]; - if( cur_cls != prev_cls ) - { - if( cls_counts && cls_count >= 0 ) - cls_counts[cls_count] = i - prev_i; - cls_map[++cls_count] = prev_cls = cur_cls; - prev_i = i; - } - *response_ptr[i] = cls_count; - } - - if( cls_counts ) - cls_counts[cls_count] = i - prev_i; - - __END__; - - cvFree( &response_ptr ); - - return out_responses; -} - - -const float** -cvGetTrainSamples( const CvMat* train_data, int tflag, - const CvMat* var_idx, const CvMat* sample_idx, - int* _var_count, int* _sample_count, - bool always_copy_data ) +void randMVNormal( InputArray _mean, InputArray _cov, int nsamples, OutputArray _samples ) { - float** samples = 0; - - CV_FUNCNAME( "cvGetTrainSamples" ); - - __BEGIN__; - - int i, j, var_count, sample_count, s_step, v_step; - bool copy_data; - const float* data; - const int *s_idx, *v_idx; - - if( !CV_IS_MAT(train_data) ) - CV_ERROR( CV_StsBadArg, "Invalid or NULL training data matrix" ); - - var_count = var_idx ? var_idx->cols + var_idx->rows - 1 : - tflag == CV_ROW_SAMPLE ? train_data->cols : train_data->rows; - sample_count = sample_idx ? sample_idx->cols + sample_idx->rows - 1 : - tflag == CV_ROW_SAMPLE ? train_data->rows : train_data->cols; - - if( _var_count ) - *_var_count = var_count; + Mat mean = _mean.getMat(), cov = _cov.getMat(); + int dim = (int)mean.total(); - if( _sample_count ) - *_sample_count = sample_count; + _samples.create(nsamples, dim, CV_32F); + Mat samples = _samples.getMat(); + randu(samples, 0., 1.); - copy_data = tflag != CV_ROW_SAMPLE || var_idx || always_copy_data; + Mat utmat; + Cholesky(cov, utmat); + int flags = mean.cols == 1 ? 0 : GEMM_3_T; - CV_CALL( samples = (float**)cvAlloc(sample_count*sizeof(samples[0]) + - (copy_data ? 1 : 0)*var_count*sample_count*sizeof(samples[0][0])) ); - data = train_data->data.fl; - s_step = train_data->step / sizeof(samples[0][0]); - v_step = 1; - s_idx = sample_idx ? sample_idx->data.i : 0; - v_idx = var_idx ? var_idx->data.i : 0; - - if( !copy_data ) - { - for( i = 0; i < sample_count; i++ ) - samples[i] = (float*)(data + (s_idx ? s_idx[i] : i)*s_step); - } - else + for( int i = 0; i < nsamples; i++ ) { - samples[0] = (float*)(samples + sample_count); - if( tflag != CV_ROW_SAMPLE ) - CV_SWAP( s_step, v_step, i ); - - for( i = 0; i < sample_count; i++ ) - { - float* dst = samples[i] = samples[0] + i*var_count; - const float* src = data + (s_idx ? s_idx[i] : i)*s_step; - - if( !v_idx ) - for( j = 0; j < var_count; j++ ) - dst[j] = src[j*v_step]; - else - for( j = 0; j < var_count; j++ ) - dst[j] = src[v_idx[j]*v_step]; - } + Mat sample = samples.row(i); + gemm(sample, utmat, 1, mean, 1, sample, flags); } - - __END__; - - return (const float**)samples; } - -void -cvCheckTrainData( const CvMat* train_data, int tflag, - const CvMat* missing_mask, - int* var_all, int* sample_all ) -{ - CV_FUNCNAME( "cvCheckTrainData" ); - - if( var_all ) - *var_all = 0; - - if( sample_all ) - *sample_all = 0; - - __BEGIN__; - - // check parameter types and sizes - if( !CV_IS_MAT(train_data) || CV_MAT_TYPE(train_data->type) != CV_32FC1 ) - CV_ERROR( CV_StsBadArg, "train data must be floating-point matrix" ); - - if( missing_mask ) - { - if( !CV_IS_MAT(missing_mask) || !CV_IS_MASK_ARR(missing_mask) || - !CV_ARE_SIZES_EQ(train_data, missing_mask) ) - CV_ERROR( CV_StsBadArg, - "missing value mask must be 8-bit matrix of the same size as training data" ); - } - - if( tflag != CV_ROW_SAMPLE && tflag != CV_COL_SAMPLE ) - CV_ERROR( CV_StsBadArg, - "Unknown training data layout (must be CV_ROW_SAMPLE or CV_COL_SAMPLE)" ); - - if( var_all ) - *var_all = tflag == CV_ROW_SAMPLE ? train_data->cols : train_data->rows; - - if( sample_all ) - *sample_all = tflag == CV_ROW_SAMPLE ? train_data->rows : train_data->cols; - - __END__; -} - - -int -cvPrepareTrainData( const char* /*funcname*/, - const CvMat* train_data, int tflag, - const CvMat* responses, int response_type, - const CvMat* var_idx, - const CvMat* sample_idx, - bool always_copy_data, - const float*** out_train_samples, - int* _sample_count, - int* _var_count, - int* _var_all, - CvMat** out_responses, - CvMat** out_response_map, - CvMat** out_var_idx, - CvMat** out_sample_idx ) -{ - int ok = 0; - CvMat* _var_idx = 0; - CvMat* _sample_idx = 0; - CvMat* _responses = 0; - int sample_all = 0, sample_count = 0, var_all = 0, var_count = 0; - - CV_FUNCNAME( "cvPrepareTrainData" ); - - // step 0. clear all the output pointers to ensure we do not try - // to call free() with uninitialized pointers - if( out_responses ) - *out_responses = 0; - - if( out_response_map ) - *out_response_map = 0; - - if( out_var_idx ) - *out_var_idx = 0; - - if( out_sample_idx ) - *out_sample_idx = 0; - - if( out_train_samples ) - *out_train_samples = 0; - - if( _sample_count ) - *_sample_count = 0; - - if( _var_count ) - *_var_count = 0; - - if( _var_all ) - *_var_all = 0; - - __BEGIN__; - - if( !out_train_samples ) - CV_ERROR( CV_StsBadArg, "output pointer to train samples is NULL" ); - - CV_CALL( cvCheckTrainData( train_data, tflag, 0, &var_all, &sample_all )); - - if( sample_idx ) - CV_CALL( _sample_idx = cvPreprocessIndexArray( sample_idx, sample_all )); - if( var_idx ) - CV_CALL( _var_idx = cvPreprocessIndexArray( var_idx, var_all )); - - if( responses ) - { - if( !out_responses ) - CV_ERROR( CV_StsNullPtr, "output response pointer is NULL" ); - - if( response_type == CV_VAR_NUMERICAL ) - { - CV_CALL( _responses = cvPreprocessOrderedResponses( responses, - _sample_idx, sample_all )); - } - else - { - CV_CALL( _responses = cvPreprocessCategoricalResponses( responses, - _sample_idx, sample_all, out_response_map, 0 )); - } - } - - CV_CALL( *out_train_samples = - cvGetTrainSamples( train_data, tflag, _var_idx, _sample_idx, - &var_count, &sample_count, always_copy_data )); - - ok = 1; - - __END__; - - if( ok ) - { - if( out_responses ) - *out_responses = _responses, _responses = 0; - - if( out_var_idx ) - *out_var_idx = _var_idx, _var_idx = 0; - - if( out_sample_idx ) - *out_sample_idx = _sample_idx, _sample_idx = 0; - - if( _sample_count ) - *_sample_count = sample_count; - - if( _var_count ) - *_var_count = var_count; - - if( _var_all ) - *_var_all = var_all; - } - else - { - if( out_response_map ) - cvReleaseMat( out_response_map ); - cvFree( out_train_samples ); - } - - if( _responses != responses ) - cvReleaseMat( &_responses ); - cvReleaseMat( &_var_idx ); - cvReleaseMat( &_sample_idx ); - - return ok; -} - - -typedef struct CvSampleResponsePair -{ - const float* sample; - const uchar* mask; - int response; - int index; -} -CvSampleResponsePair; - - -static int -CV_CDECL icvCmpSampleResponsePairs( const void* a, const void* b ) -{ - int ra = ((const CvSampleResponsePair*)a)->response; - int rb = ((const CvSampleResponsePair*)b)->response; - int ia = ((const CvSampleResponsePair*)a)->index; - int ib = ((const CvSampleResponsePair*)b)->index; - - return ra < rb ? -1 : ra > rb ? 1 : ia - ib; - //return (ra > rb ? -1 : 0)|(ra < rb); -} - - -void -cvSortSamplesByClasses( const float** samples, const CvMat* classes, - int* class_ranges, const uchar** mask ) -{ - CvSampleResponsePair* pairs = 0; - CV_FUNCNAME( "cvSortSamplesByClasses" ); - - __BEGIN__; - - int i, k = 0, sample_count; - - if( !samples || !classes || !class_ranges ) - CV_ERROR( CV_StsNullPtr, "INTERNAL ERROR: some of the args are NULL pointers" ); - - if( classes->rows != 1 || CV_MAT_TYPE(classes->type) != CV_32SC1 ) - CV_ERROR( CV_StsBadArg, "classes array must be a single row of integers" ); - - sample_count = classes->cols; - CV_CALL( pairs = (CvSampleResponsePair*)cvAlloc( (sample_count+1)*sizeof(pairs[0]))); - - for( i = 0; i < sample_count; i++ ) - { - pairs[i].sample = samples[i]; - pairs[i].mask = (mask) ? (mask[i]) : 0; - pairs[i].response = classes->data.i[i]; - pairs[i].index = i; - assert( classes->data.i[i] >= 0 ); - } - - qsort( pairs, sample_count, sizeof(pairs[0]), icvCmpSampleResponsePairs ); - pairs[sample_count].response = -1; - class_ranges[0] = 0; - - for( i = 0; i < sample_count; i++ ) - { - samples[i] = pairs[i].sample; - if (mask) - mask[i] = pairs[i].mask; - classes->data.i[i] = pairs[i].response; - - if( pairs[i].response != pairs[i+1].response ) - class_ranges[++k] = i+1; - } - - __END__; - - cvFree( &pairs ); -} - - -void -cvPreparePredictData( const CvArr* _sample, int dims_all, - const CvMat* comp_idx, int class_count, - const CvMat* prob, float** _row_sample, - int as_sparse ) -{ - float* row_sample = 0; - int* inverse_comp_idx = 0; - - CV_FUNCNAME( "cvPreparePredictData" ); - - __BEGIN__; - - const CvMat* sample = (const CvMat*)_sample; - float* sample_data; - int sample_step; - int is_sparse = CV_IS_SPARSE_MAT(sample); - int d, sizes[CV_MAX_DIM]; - int i, dims_selected; - int vec_size; - - if( !is_sparse && !CV_IS_MAT(sample) ) - CV_ERROR( !sample ? CV_StsNullPtr : CV_StsBadArg, "The sample is not a valid vector" ); - - if( cvGetElemType( sample ) != CV_32FC1 ) - CV_ERROR( CV_StsUnsupportedFormat, "Input sample must have 32fC1 type" ); - - CV_CALL( d = cvGetDims( sample, sizes )); - - if( !((is_sparse && d == 1) || (!is_sparse && d == 2 && (sample->rows == 1 || sample->cols == 1))) ) - CV_ERROR( CV_StsBadSize, "Input sample must be 1-dimensional vector" ); - - if( d == 1 ) - sizes[1] = 1; - - if( sizes[0] + sizes[1] - 1 != dims_all ) - CV_ERROR( CV_StsUnmatchedSizes, - "The sample size is different from what has been used for training" ); - - if( !_row_sample ) - CV_ERROR( CV_StsNullPtr, "INTERNAL ERROR: The row_sample pointer is NULL" ); - - if( comp_idx && (!CV_IS_MAT(comp_idx) || comp_idx->rows != 1 || - CV_MAT_TYPE(comp_idx->type) != CV_32SC1) ) - CV_ERROR( CV_StsBadArg, "INTERNAL ERROR: invalid comp_idx" ); - - dims_selected = comp_idx ? comp_idx->cols : dims_all; - - if( prob ) - { - if( !CV_IS_MAT(prob) ) - CV_ERROR( CV_StsBadArg, "The output matrix of probabilities is invalid" ); - - if( (prob->rows != 1 && prob->cols != 1) || - (CV_MAT_TYPE(prob->type) != CV_32FC1 && - CV_MAT_TYPE(prob->type) != CV_64FC1) ) - CV_ERROR( CV_StsBadSize, - "The matrix of probabilities must be 1-dimensional vector of 32fC1 type" ); - - if( prob->rows + prob->cols - 1 != class_count ) - CV_ERROR( CV_StsUnmatchedSizes, - "The vector of probabilities must contain as many elements as " - "the number of classes in the training set" ); - } - - vec_size = !as_sparse ? dims_selected*sizeof(row_sample[0]) : - (dims_selected + 1)*sizeof(CvSparseVecElem32f); - - if( CV_IS_MAT(sample) ) - { - sample_data = sample->data.fl; - sample_step = CV_IS_MAT_CONT(sample->type) ? 1 : sample->step/sizeof(row_sample[0]); - - if( !comp_idx && CV_IS_MAT_CONT(sample->type) && !as_sparse ) - *_row_sample = sample_data; - else - { - CV_CALL( row_sample = (float*)cvAlloc( vec_size )); - - if( !comp_idx ) - for( i = 0; i < dims_selected; i++ ) - row_sample[i] = sample_data[sample_step*i]; - else - { - int* comp = comp_idx->data.i; - for( i = 0; i < dims_selected; i++ ) - row_sample[i] = sample_data[sample_step*comp[i]]; - } - - *_row_sample = row_sample; - } - - if( as_sparse ) - { - const float* src = (const float*)row_sample; - CvSparseVecElem32f* dst = (CvSparseVecElem32f*)row_sample; - - dst[dims_selected].idx = -1; - for( i = dims_selected - 1; i >= 0; i-- ) - { - dst[i].idx = i; - dst[i].val = src[i]; - } - } - } - else - { - CvSparseNode* node; - CvSparseMatIterator mat_iterator; - const CvSparseMat* sparse = (const CvSparseMat*)sample; - assert( is_sparse ); - - node = cvInitSparseMatIterator( sparse, &mat_iterator ); - CV_CALL( row_sample = (float*)cvAlloc( vec_size )); - - if( comp_idx ) - { - CV_CALL( inverse_comp_idx = (int*)cvAlloc( dims_all*sizeof(int) )); - memset( inverse_comp_idx, -1, dims_all*sizeof(int) ); - for( i = 0; i < dims_selected; i++ ) - inverse_comp_idx[comp_idx->data.i[i]] = i; - } - - if( !as_sparse ) - { - memset( row_sample, 0, vec_size ); - - for( ; node != 0; node = cvGetNextSparseNode(&mat_iterator) ) - { - int idx = *CV_NODE_IDX( sparse, node ); - if( inverse_comp_idx ) - { - idx = inverse_comp_idx[idx]; - if( idx < 0 ) - continue; - } - row_sample[idx] = *(float*)CV_NODE_VAL( sparse, node ); - } - } - else - { - CvSparseVecElem32f* ptr = (CvSparseVecElem32f*)row_sample; - - for( ; node != 0; node = cvGetNextSparseNode(&mat_iterator) ) - { - int idx = *CV_NODE_IDX( sparse, node ); - if( inverse_comp_idx ) - { - idx = inverse_comp_idx[idx]; - if( idx < 0 ) - continue; - } - ptr->idx = idx; - ptr->val = *(float*)CV_NODE_VAL( sparse, node ); - ptr++; - } - - qsort( row_sample, ptr - (CvSparseVecElem32f*)row_sample, - sizeof(ptr[0]), icvCmpSparseVecElems ); - ptr->idx = -1; - } - - *_row_sample = row_sample; - } - - __END__; - - if( inverse_comp_idx ) - cvFree( &inverse_comp_idx ); - - if( cvGetErrStatus() < 0 && _row_sample ) - { - cvFree( &row_sample ); - *_row_sample = 0; - } -} - - -static void -icvConvertDataToSparse( const uchar* src, int src_step, int src_type, - uchar* dst, int dst_step, int dst_type, - CvSize size, int* idx ) -{ - CV_FUNCNAME( "icvConvertDataToSparse" ); - - __BEGIN__; - - int i, j; - src_type = CV_MAT_TYPE(src_type); - dst_type = CV_MAT_TYPE(dst_type); - - if( CV_MAT_CN(src_type) != 1 || CV_MAT_CN(dst_type) != 1 ) - CV_ERROR( CV_StsUnsupportedFormat, "The function supports only single-channel arrays" ); - - if( src_step == 0 ) - src_step = CV_ELEM_SIZE(src_type); - - if( dst_step == 0 ) - dst_step = CV_ELEM_SIZE(dst_type); - - // if there is no "idx" and if both arrays are continuous, - // do the whole processing (copying or conversion) in a single loop - if( !idx && CV_ELEM_SIZE(src_type)*size.width == src_step && - CV_ELEM_SIZE(dst_type)*size.width == dst_step ) - { - size.width *= size.height; - size.height = 1; - } - - if( src_type == dst_type ) - { - int full_width = CV_ELEM_SIZE(dst_type)*size.width; - - if( full_width == sizeof(int) ) // another common case: copy int's or float's - for( i = 0; i < size.height; i++, src += src_step ) - *(int*)(dst + dst_step*(idx ? idx[i] : i)) = *(int*)src; - else - for( i = 0; i < size.height; i++, src += src_step ) - memcpy( dst + dst_step*(idx ? idx[i] : i), src, full_width ); - } - else if( src_type == CV_32SC1 && (dst_type == CV_32FC1 || dst_type == CV_64FC1) ) - for( i = 0; i < size.height; i++, src += src_step ) - { - uchar* _dst = dst + dst_step*(idx ? idx[i] : i); - if( dst_type == CV_32FC1 ) - for( j = 0; j < size.width; j++ ) - ((float*)_dst)[j] = (float)((int*)src)[j]; - else - for( j = 0; j < size.width; j++ ) - ((double*)_dst)[j] = ((int*)src)[j]; - } - else if( (src_type == CV_32FC1 || src_type == CV_64FC1) && dst_type == CV_32SC1 ) - for( i = 0; i < size.height; i++, src += src_step ) - { - uchar* _dst = dst + dst_step*(idx ? idx[i] : i); - if( src_type == CV_32FC1 ) - for( j = 0; j < size.width; j++ ) - ((int*)_dst)[j] = cvRound(((float*)src)[j]); - else - for( j = 0; j < size.width; j++ ) - ((int*)_dst)[j] = cvRound(((double*)src)[j]); - } - else if( (src_type == CV_32FC1 && dst_type == CV_64FC1) || - (src_type == CV_64FC1 && dst_type == CV_32FC1) ) - for( i = 0; i < size.height; i++, src += src_step ) - { - uchar* _dst = dst + dst_step*(idx ? idx[i] : i); - if( src_type == CV_32FC1 ) - for( j = 0; j < size.width; j++ ) - ((double*)_dst)[j] = ((float*)src)[j]; - else - for( j = 0; j < size.width; j++ ) - ((float*)_dst)[j] = (float)((double*)src)[j]; - } - else - CV_ERROR( CV_StsUnsupportedFormat, "Unsupported combination of input and output vectors" ); - - __END__; -} - - -void -cvWritebackLabels( const CvMat* labels, CvMat* dst_labels, - const CvMat* centers, CvMat* dst_centers, - const CvMat* probs, CvMat* dst_probs, - const CvMat* sample_idx, int samples_all, - const CvMat* comp_idx, int dims_all ) -{ - CV_FUNCNAME( "cvWritebackLabels" ); - - __BEGIN__; - - int samples_selected = samples_all, dims_selected = dims_all; - - if( dst_labels && !CV_IS_MAT(dst_labels) ) - CV_ERROR( CV_StsBadArg, "Array of output labels is not a valid matrix" ); - - if( dst_centers ) - if( !ICV_IS_MAT_OF_TYPE(dst_centers, CV_32FC1) && - !ICV_IS_MAT_OF_TYPE(dst_centers, CV_64FC1) ) - CV_ERROR( CV_StsBadArg, "Array of cluster centers is not a valid matrix" ); - - if( dst_probs && !CV_IS_MAT(dst_probs) ) - CV_ERROR( CV_StsBadArg, "Probability matrix is not valid" ); - - if( sample_idx ) - { - CV_ASSERT( sample_idx->rows == 1 && CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ); - samples_selected = sample_idx->cols; - } - - if( comp_idx ) - { - CV_ASSERT( comp_idx->rows == 1 && CV_MAT_TYPE(comp_idx->type) == CV_32SC1 ); - dims_selected = comp_idx->cols; - } - - if( dst_labels && (!labels || labels->data.ptr != dst_labels->data.ptr) ) - { - if( !labels ) - CV_ERROR( CV_StsNullPtr, "NULL labels" ); - - CV_ASSERT( labels->rows == 1 ); - - if( dst_labels->rows != 1 && dst_labels->cols != 1 ) - CV_ERROR( CV_StsBadSize, "Array of output labels should be 1d vector" ); - - if( dst_labels->rows + dst_labels->cols - 1 != samples_all ) - CV_ERROR( CV_StsUnmatchedSizes, - "Size of vector of output labels is not equal to the total number of input samples" ); - - CV_ASSERT( labels->cols == samples_selected ); - - CV_CALL( icvConvertDataToSparse( labels->data.ptr, labels->step, labels->type, - dst_labels->data.ptr, dst_labels->step, dst_labels->type, - cvSize( 1, samples_selected ), sample_idx ? sample_idx->data.i : 0 )); - } - - if( dst_centers && (!centers || centers->data.ptr != dst_centers->data.ptr) ) - { - int i; - - if( !centers ) - CV_ERROR( CV_StsNullPtr, "NULL centers" ); - - if( centers->rows != dst_centers->rows ) - CV_ERROR( CV_StsUnmatchedSizes, "Invalid number of rows in matrix of output centers" ); - - if( dst_centers->cols != dims_all ) - CV_ERROR( CV_StsUnmatchedSizes, - "Number of columns in matrix of output centers is " - "not equal to the total number of components in the input samples" ); - - CV_ASSERT( centers->cols == dims_selected ); - - for( i = 0; i < centers->rows; i++ ) - CV_CALL( icvConvertDataToSparse( centers->data.ptr + i*centers->step, 0, centers->type, - dst_centers->data.ptr + i*dst_centers->step, 0, dst_centers->type, - cvSize( 1, dims_selected ), comp_idx ? comp_idx->data.i : 0 )); - } - - if( dst_probs && (!probs || probs->data.ptr != dst_probs->data.ptr) ) - { - if( !probs ) - CV_ERROR( CV_StsNullPtr, "NULL probs" ); - - if( probs->cols != dst_probs->cols ) - CV_ERROR( CV_StsUnmatchedSizes, "Invalid number of columns in output probability matrix" ); - - if( dst_probs->rows != samples_all ) - CV_ERROR( CV_StsUnmatchedSizes, - "Number of rows in output probability matrix is " - "not equal to the total number of input samples" ); - - CV_ASSERT( probs->rows == samples_selected ); - - CV_CALL( icvConvertDataToSparse( probs->data.ptr, probs->step, probs->type, - dst_probs->data.ptr, dst_probs->step, dst_probs->type, - cvSize( probs->cols, samples_selected ), - sample_idx ? sample_idx->data.i : 0 )); - } - - __END__; -} - -#if 0 -CV_IMPL void -cvStatModelMultiPredict( const CvStatModel* stat_model, - const CvArr* predict_input, - int flags, CvMat* predict_output, - CvMat* probs, const CvMat* sample_idx ) -{ - CvMemStorage* storage = 0; - CvMat* sample_idx_buffer = 0; - CvSparseMat** sparse_rows = 0; - int samples_selected = 0; - - CV_FUNCNAME( "cvStatModelMultiPredict" ); - - __BEGIN__; - - int i; - int predict_output_step = 1, sample_idx_step = 1; - int type; - int d, sizes[CV_MAX_DIM]; - int tflag = flags == CV_COL_SAMPLE; - int samples_all, dims_all; - int is_sparse = CV_IS_SPARSE_MAT(predict_input); - CvMat predict_input_part; - CvArr* sample = &predict_input_part; - CvMat probs_part; - CvMat* probs1 = probs ? &probs_part : 0; - - if( !CV_IS_STAT_MODEL(stat_model) ) - CV_ERROR( !stat_model ? CV_StsNullPtr : CV_StsBadArg, "Invalid statistical model" ); - - if( !stat_model->predict ) - CV_ERROR( CV_StsNotImplemented, "There is no \"predict\" method" ); - - if( !predict_input || !predict_output ) - CV_ERROR( CV_StsNullPtr, "NULL input or output matrices" ); - - if( !is_sparse && !CV_IS_MAT(predict_input) ) - CV_ERROR( CV_StsBadArg, "predict_input should be a matrix or a sparse matrix" ); - - if( !CV_IS_MAT(predict_output) ) - CV_ERROR( CV_StsBadArg, "predict_output should be a matrix" ); - - type = cvGetElemType( predict_input ); - if( type != CV_32FC1 || - (CV_MAT_TYPE(predict_output->type) != CV_32FC1 && - CV_MAT_TYPE(predict_output->type) != CV_32SC1 )) - CV_ERROR( CV_StsUnsupportedFormat, "The input or output matrix has unsupported format" ); - - CV_CALL( d = cvGetDims( predict_input, sizes )); - if( d > 2 ) - CV_ERROR( CV_StsBadSize, "The input matrix should be 1- or 2-dimensional" ); - - if( !tflag ) - { - samples_all = samples_selected = sizes[0]; - dims_all = sizes[1]; - } - else - { - samples_all = samples_selected = sizes[1]; - dims_all = sizes[0]; - } - - if( sample_idx ) - { - if( !CV_IS_MAT(sample_idx) ) - CV_ERROR( CV_StsBadArg, "Invalid sample_idx matrix" ); - - if( sample_idx->cols != 1 && sample_idx->rows != 1 ) - CV_ERROR( CV_StsBadSize, "sample_idx must be 1-dimensional matrix" ); - - samples_selected = sample_idx->rows + sample_idx->cols - 1; - - if( CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ) - { - if( samples_selected > samples_all ) - CV_ERROR( CV_StsBadSize, "sample_idx is too large vector" ); - } - else if( samples_selected != samples_all ) - CV_ERROR( CV_StsUnmatchedSizes, "sample_idx has incorrect size" ); - - sample_idx_step = sample_idx->step ? - sample_idx->step / CV_ELEM_SIZE(sample_idx->type) : 1; - } - - if( predict_output->rows != 1 && predict_output->cols != 1 ) - CV_ERROR( CV_StsBadSize, "predict_output should be a 1-dimensional matrix" ); - - if( predict_output->rows + predict_output->cols - 1 != samples_all ) - CV_ERROR( CV_StsUnmatchedSizes, "predict_output and predict_input have uncoordinated sizes" ); - - predict_output_step = predict_output->step ? - predict_output->step / CV_ELEM_SIZE(predict_output->type) : 1; - - if( probs ) - { - if( !CV_IS_MAT(probs) ) - CV_ERROR( CV_StsBadArg, "Invalid matrix of probabilities" ); - - if( probs->rows != samples_all ) - CV_ERROR( CV_StsUnmatchedSizes, - "matrix of probabilities must have as many rows as the total number of samples" ); - - if( CV_MAT_TYPE(probs->type) != CV_32FC1 ) - CV_ERROR( CV_StsUnsupportedFormat, "matrix of probabilities must have 32fC1 type" ); - } - - if( is_sparse ) - { - CvSparseNode* node; - CvSparseMatIterator mat_iterator; - CvSparseMat* sparse = (CvSparseMat*)predict_input; - - if( sample_idx && CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ) - { - CV_CALL( sample_idx_buffer = cvCreateMat( 1, samples_all, CV_8UC1 )); - cvZero( sample_idx_buffer ); - for( i = 0; i < samples_selected; i++ ) - sample_idx_buffer->data.ptr[sample_idx->data.i[i*sample_idx_step]] = 1; - samples_selected = samples_all; - sample_idx = sample_idx_buffer; - sample_idx_step = 1; - } - - CV_CALL( sparse_rows = (CvSparseMat**)cvAlloc( samples_selected*sizeof(sparse_rows[0]))); - for( i = 0; i < samples_selected; i++ ) - { - if( sample_idx && sample_idx->data.ptr[i*sample_idx_step] == 0 ) - continue; - CV_CALL( sparse_rows[i] = cvCreateSparseMat( 1, &dims_all, type )); - if( !storage ) - storage = sparse_rows[i]->heap->storage; - else - { - // hack: to decrease memory footprint, make all the sparse matrices - // reside in the same storage - int elem_size = sparse_rows[i]->heap->elem_size; - cvReleaseMemStorage( &sparse_rows[i]->heap->storage ); - sparse_rows[i]->heap = cvCreateSet( 0, sizeof(CvSet), elem_size, storage ); - } - } - - // put each row (or column) of predict_input into separate sparse matrix. - node = cvInitSparseMatIterator( sparse, &mat_iterator ); - for( ; node != 0; node = cvGetNextSparseNode( &mat_iterator )) - { - int* idx = CV_NODE_IDX( sparse, node ); - int idx0 = idx[tflag ^ 1]; - int idx1 = idx[tflag]; - - if( sample_idx && sample_idx->data.ptr[idx0*sample_idx_step] == 0 ) - continue; - - assert( sparse_rows[idx0] != 0 ); - *(float*)cvPtrND( sparse, &idx1, 0, 1, 0 ) = *(float*)CV_NODE_VAL( sparse, node ); - } - } - - for( i = 0; i < samples_selected; i++ ) - { - int idx = i; - float response; - - if( sample_idx ) - { - if( CV_MAT_TYPE(sample_idx->type) == CV_32SC1 ) - { - idx = sample_idx->data.i[i*sample_idx_step]; - if( (unsigned)idx >= (unsigned)samples_all ) - CV_ERROR( CV_StsOutOfRange, "Some of sample_idx elements are out of range" ); - } - else if( CV_MAT_TYPE(sample_idx->type) == CV_8UC1 && - sample_idx->data.ptr[i*sample_idx_step] == 0 ) - continue; - } - - if( !is_sparse ) - { - if( !tflag ) - cvGetRow( predict_input, &predict_input_part, idx ); - else - { - cvGetCol( predict_input, &predict_input_part, idx ); - } - } - else - sample = sparse_rows[idx]; - - if( probs ) - cvGetRow( probs, probs1, idx ); - - CV_CALL( response = stat_model->predict( stat_model, (CvMat*)sample, probs1 )); - - if( CV_MAT_TYPE(predict_output->type) == CV_32FC1 ) - predict_output->data.fl[idx*predict_output_step] = response; - else - { - CV_ASSERT( cvRound(response) == response ); - predict_output->data.i[idx*predict_output_step] = cvRound(response); - } - } - - __END__; - - if( sparse_rows ) - { - int i; - for( i = 0; i < samples_selected; i++ ) - if( sparse_rows[i] ) - { - sparse_rows[i]->heap->storage = 0; - cvReleaseSparseMat( &sparse_rows[i] ); - } - cvFree( &sparse_rows ); - } - - cvReleaseMat( &sample_idx_buffer ); - cvReleaseMemStorage( &storage ); -} -#endif - -// By P. Yarykin - begin - - -void cvCombineResponseMaps (CvMat* _responses, - const CvMat* old_response_map, - CvMat* new_response_map, - CvMat** out_response_map) -{ - int** old_data = NULL; - int** new_data = NULL; - - CV_FUNCNAME ("cvCombineResponseMaps"); - __BEGIN__ - - int i,j; - int old_n, new_n, out_n; - int samples, free_response; - int* first; - int* responses; - int* out_data; - - if( out_response_map ) - *out_response_map = 0; - -// Check input data. - if ((!ICV_IS_MAT_OF_TYPE (_responses, CV_32SC1)) || - (!ICV_IS_MAT_OF_TYPE (old_response_map, CV_32SC1)) || - (!ICV_IS_MAT_OF_TYPE (new_response_map, CV_32SC1))) - { - CV_ERROR (CV_StsBadArg, "Some of input arguments is not the CvMat") - } - -// Prepare sorted responses. - first = new_response_map->data.i; - new_n = new_response_map->cols; - CV_CALL (new_data = (int**)cvAlloc (new_n * sizeof (new_data[0]))); - for (i = 0; i < new_n; i++) - new_data[i] = first + i; - qsort (new_data, new_n, sizeof(int*), icvCmpIntegersPtr); - - first = old_response_map->data.i; - old_n = old_response_map->cols; - CV_CALL (old_data = (int**)cvAlloc (old_n * sizeof (old_data[0]))); - for (i = 0; i < old_n; i++) - old_data[i] = first + i; - qsort (old_data, old_n, sizeof(int*), icvCmpIntegersPtr); - -// Count the number of different responses. - for (i = 0, j = 0, out_n = 0; i < old_n && j < new_n; out_n++) - { - if (*old_data[i] == *new_data[j]) - { - i++; - j++; - } - else if (*old_data[i] < *new_data[j]) - i++; - else - j++; - } - out_n += old_n - i + new_n - j; - -// Create and fill the result response maps. - CV_CALL (*out_response_map = cvCreateMat (1, out_n, CV_32SC1)); - out_data = (*out_response_map)->data.i; - memcpy (out_data, first, old_n * sizeof (int)); - - free_response = old_n; - for (i = 0, j = 0; i < old_n && j < new_n; ) - { - if (*old_data[i] == *new_data[j]) - { - *new_data[j] = (int)(old_data[i] - first); - i++; - j++; - } - else if (*old_data[i] < *new_data[j]) - i++; - else - { - out_data[free_response] = *new_data[j]; - *new_data[j] = free_response++; - j++; - } - } - for (; j < new_n; j++) - { - out_data[free_response] = *new_data[j]; - *new_data[j] = free_response++; - } - CV_ASSERT (free_response == out_n); - -// Change according to out response map. - samples = _responses->cols + _responses->rows - 1; - responses = _responses->data.i; - first = new_response_map->data.i; - for (i = 0; i < samples; i++) - { - responses[i] = first[responses[i]]; - } - - __END__ - - cvFree(&old_data); - cvFree(&new_data); - -} - - -static int icvGetNumberOfCluster( double* prob_vector, int num_of_clusters, float r, - float outlier_thresh, int normalize_probs ) -{ - int max_prob_loc = 0; - - CV_FUNCNAME("icvGetNumberOfCluster"); - __BEGIN__; - - double prob, maxprob, sum; - int i; - - CV_ASSERT(prob_vector); - CV_ASSERT(num_of_clusters >= 0); - - maxprob = prob_vector[0]; - max_prob_loc = 0; - sum = maxprob; - for( i = 1; i < num_of_clusters; i++ ) - { - prob = prob_vector[i]; - sum += prob; - if( prob > maxprob ) - { - max_prob_loc = i; - maxprob = prob; - } - } - if( normalize_probs && fabs(sum - 1.) > FLT_EPSILON ) - { - for( i = 0; i < num_of_clusters; i++ ) - prob_vector[i] /= sum; - } - if( fabs(r - 1.) > FLT_EPSILON && fabs(sum - 1.) < outlier_thresh ) - max_prob_loc = -1; - - __END__; - - return max_prob_loc; - -} // End of icvGetNumberOfCluster - - -void icvFindClusterLabels( const CvMat* probs, float outlier_thresh, float r, - const CvMat* labels ) -{ - CvMat* counts = 0; - - CV_FUNCNAME("icvFindClusterLabels"); - __BEGIN__; - - int nclusters, nsamples; - int i, j; - double* probs_data; - - CV_ASSERT( ICV_IS_MAT_OF_TYPE(probs, CV_64FC1) ); - CV_ASSERT( ICV_IS_MAT_OF_TYPE(labels, CV_32SC1) ); - - nclusters = probs->cols; - nsamples = probs->rows; - CV_ASSERT( nsamples == labels->cols ); - - CV_CALL( counts = cvCreateMat( 1, nclusters + 1, CV_32SC1 ) ); - CV_CALL( cvSetZero( counts )); - for( i = 0; i < nsamples; i++ ) - { - labels->data.i[i] = icvGetNumberOfCluster( probs->data.db + i*probs->cols, - nclusters, r, outlier_thresh, 1 ); - counts->data.i[labels->data.i[i] + 1]++; - } - CV_ASSERT((int)cvSum(counts).val[0] == nsamples); - // Filling empty clusters with the vector, that has the maximal probability - for( j = 0; j < nclusters; j++ ) // outliers are ignored - { - int maxprob_loc = -1; - double maxprob = 0; - - if( counts->data.i[j+1] ) // j-th class is not empty - continue; - // look for the presentative, which is not lonely in it's cluster - // and that has a maximal probability among all these vectors - probs_data = probs->data.db; - for( i = 0; i < nsamples; i++, probs_data++ ) - { - int label = labels->data.i[i]; - double prob; - if( counts->data.i[label+1] == 0 || - (counts->data.i[label+1] <= 1 && label != -1) ) - continue; - prob = *probs_data; - if( prob >= maxprob ) - { - maxprob = prob; - maxprob_loc = i; - } - } - // maxprob_loc == 0 <=> number of vectors less then number of clusters - CV_ASSERT( maxprob_loc >= 0 ); - counts->data.i[labels->data.i[maxprob_loc] + 1]--; - labels->data.i[maxprob_loc] = j; - counts->data.i[j + 1]++; - } - - __END__; - - cvReleaseMat( &counts ); -} // End of icvFindClusterLabels +}} /* End of file */ diff --git a/modules/ml/src/knearest.cpp b/modules/ml/src/knearest.cpp index a05a30da86acc15e2a06df573f67e419c21bb936..3ead3228f54eb8b80817af9d791677953cad4e1a 100644 --- a/modules/ml/src/knearest.cpp +++ b/modules/ml/src/knearest.cpp @@ -7,9 +7,11 @@ // copy or use the software. // // -// Intel License Agreement +// License Agreement +// For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -22,7 +24,7 @@ // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // -// * The name of Intel Corporation may not be used to endorse or promote products +// * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and @@ -41,442 +43,321 @@ #include "precomp.hpp" /****************************************************************************************\ -* K-Nearest Neighbors Classifier * +* K-Nearest Neighbors Classifier * \****************************************************************************************/ -// k Nearest Neighbors -CvKNearest::CvKNearest() -{ - samples = 0; - clear(); -} - - -CvKNearest::~CvKNearest() -{ - clear(); -} - +namespace cv { +namespace ml { -CvKNearest::CvKNearest( const CvMat* _train_data, const CvMat* _responses, - const CvMat* _sample_idx, bool _is_regression, int _max_k ) +KNearest::Params::Params(int k, bool isclassifier_) { - samples = 0; - train( _train_data, _responses, _sample_idx, _is_regression, _max_k, false ); + defaultK = k; + isclassifier = isclassifier_; } -void CvKNearest::clear() +class KNearestImpl : public KNearest { - while( samples ) +public: + KNearestImpl(const Params& p) { - CvVectors* next_samples = samples->next; - cvFree( &samples->data.fl ); - cvFree( &samples ); - samples = next_samples; + params = p; } - var_count = 0; - total = 0; - max_k = 0; -} - - -int CvKNearest::get_max_k() const { return max_k; } - -int CvKNearest::get_var_count() const { return var_count; } -bool CvKNearest::is_regression() const { return regression; } - -int CvKNearest::get_sample_count() const { return total; } - -bool CvKNearest::train( const CvMat* _train_data, const CvMat* _responses, - const CvMat* _sample_idx, bool _is_regression, - int _max_k, bool _update_base ) -{ - bool ok = false; - CvMat* responses = 0; + virtual ~KNearestImpl() {} - CV_FUNCNAME( "CvKNearest::train" ); + Params getParams() const { return params; } + void setParams(const Params& p) { params = p; } - __BEGIN__; + bool isClassifier() const { return params.isclassifier; } + bool isTrained() const { return !samples.empty(); } - CvVectors* _samples = 0; - float** _data = 0; - int _count = 0, _dims = 0, _dims_all = 0, _rsize = 0; + String getDefaultModelName() const { return "opencv_ml_knn"; } - if( !_update_base ) - clear(); - - // Prepare training data and related parameters. - // Treat categorical responses as ordered - to prevent class label compression and - // to enable entering new classes in the updates - CV_CALL( cvPrepareTrainData( "CvKNearest::train", _train_data, CV_ROW_SAMPLE, - _responses, CV_VAR_ORDERED, 0, _sample_idx, true, (const float***)&_data, - &_count, &_dims, &_dims_all, &responses, 0, 0 )); - - if( !responses ) - CV_ERROR( CV_StsNoMem, "Could not allocate memory for responses" ); - - if( _update_base && _dims != var_count ) - CV_ERROR( CV_StsBadArg, "The newly added data have different dimensionality" ); - - if( !_update_base ) + void clear() { - if( _max_k < 1 ) - CV_ERROR( CV_StsOutOfRange, "max_k must be a positive number" ); - - regression = _is_regression; - var_count = _dims; - max_k = _max_k; + samples.release(); + responses.release(); } - _rsize = _count*sizeof(float); - CV_CALL( _samples = (CvVectors*)cvAlloc( sizeof(*_samples) + _rsize )); - _samples->next = samples; - _samples->type = CV_32F; - _samples->data.fl = _data; - _samples->count = _count; - total += _count; + int getVarCount() const { return samples.cols; } - samples = _samples; - memcpy( _samples + 1, responses->data.fl, _rsize ); + bool train( const Ptr& data, int flags ) + { + Mat new_samples = data->getTrainSamples(ROW_SAMPLE); + Mat new_responses; + data->getTrainResponses().convertTo(new_responses, CV_32F); + bool update = (flags & UPDATE_MODEL) != 0 && !samples.empty(); - ok = true; + CV_Assert( new_samples.type() == CV_32F ); - __END__; + if( !update ) + { + clear(); + } + else + { + CV_Assert( new_samples.cols == samples.cols && + new_responses.cols == responses.cols ); + } - if( responses && responses->data.ptr != _responses->data.ptr ) - cvReleaseMat(&responses); + samples.push_back(new_samples); + responses.push_back(new_responses); - return ok; -} + return true; + } + void findNearestCore( const Mat& _samples, int k0, const Range& range, + Mat* results, Mat* neighbor_responses, + Mat* dists, float* presult ) const + { + int testidx, baseidx, i, j, d = samples.cols, nsamples = samples.rows; + int testcount = range.end - range.start; + int k = std::min(k0, nsamples); + AutoBuffer buf(testcount*k*2); + float* dbuf = buf; + float* rbuf = dbuf + testcount*k; -void CvKNearest::find_neighbors_direct( const CvMat* _samples, int k, int start, int end, - float* neighbor_responses, const float** neighbors, float* dist ) const -{ - int i, j, count = end - start, k1 = 0, k2 = 0, d = var_count; - CvVectors* s = samples; + const float* rptr = responses.ptr(); - for( ; s != 0; s = s->next ) - { - int n = s->count; - for( j = 0; j < n; j++ ) + for( testidx = 0; testidx < testcount; testidx++ ) { - for( i = 0; i < count; i++ ) + for( i = 0; i < k; i++ ) { - double sum = 0; - Cv32suf si; - const float* v = s->data.fl[j]; - const float* u = (float*)(_samples->data.ptr + _samples->step*(start + i)); - Cv32suf* dd = (Cv32suf*)(dist + i*k); - float* nr; - const float** nn; - int t, ii, ii1; - - for( t = 0; t <= d - 4; t += 4 ) + dbuf[testidx*k + i] = FLT_MAX; + rbuf[testidx*k + i] = 0.f; + } + } + + for( baseidx = 0; baseidx < nsamples; baseidx++ ) + { + for( testidx = 0; testidx < testcount; testidx++ ) + { + const float* v = samples.ptr(baseidx); + const float* u = _samples.ptr(testidx + range.start); + + float s = 0; + for( i = 0; i <= d - 4; i += 4 ) { - double t0 = u[t] - v[t], t1 = u[t+1] - v[t+1]; - double t2 = u[t+2] - v[t+2], t3 = u[t+3] - v[t+3]; - sum += t0*t0 + t1*t1 + t2*t2 + t3*t3; + float t0 = u[i] - v[i], t1 = u[i+1] - v[i+1]; + float t2 = u[i+2] - v[i+2], t3 = u[i+3] - v[i+3]; + s += t0*t0 + t1*t1 + t2*t2 + t3*t3; } - for( ; t < d; t++ ) + for( ; i < d; i++ ) { - double t0 = u[t] - v[t]; - sum += t0*t0; + float t0 = u[i] - v[i]; + s += t0*t0; } - si.f = (float)sum; - for( ii = k1-1; ii >= 0; ii-- ) - if( si.i > dd[ii].i ) + Cv32suf si; + si.f = (float)s; + Cv32suf* dd = (Cv32suf*)(&dbuf[testidx*k]); + float* nr = &rbuf[testidx*k]; + + for( i = k; i > 0; i-- ) + if( si.i >= dd[i-1].i ) break; - if( ii >= k-1 ) + if( i >= k ) continue; - nr = neighbor_responses + i*k; - nn = neighbors ? neighbors + (start + i)*k : 0; - for( ii1 = k2 - 1; ii1 > ii; ii1-- ) + for( j = k-2; j >= i; j-- ) { - dd[ii1+1].i = dd[ii1].i; - nr[ii1+1] = nr[ii1]; - if( nn ) nn[ii1+1] = nn[ii1]; + dd[j+1].i = dd[j].i; + nr[j+1] = nr[j]; } - dd[ii+1].i = si.i; - nr[ii+1] = ((float*)(s + 1))[j]; - if( nn ) - nn[ii+1] = v; + dd[i].i = si.i; + nr[i] = rptr[baseidx]; } - k1 = MIN( k1+1, k ); - k2 = MIN( k1, k-1 ); } - } -} + float result = 0.f; + float inv_scale = 1.f/k; -float CvKNearest::write_results( int k, int k1, int start, int end, - const float* neighbor_responses, const float* dist, - CvMat* _results, CvMat* _neighbor_responses, - CvMat* _dist, Cv32suf* sort_buf ) const -{ - float result = 0.f; - int i, j, j1, count = end - start; - double inv_scale = 1./k1; - int rstep = _results && !CV_IS_MAT_CONT(_results->type) ? _results->step/sizeof(result) : 1; - - for( i = 0; i < count; i++ ) - { - const Cv32suf* nr = (const Cv32suf*)(neighbor_responses + i*k); - float* dst; - float r; - if( _results || start+i == 0 ) + for( testidx = 0; testidx < testcount; testidx++ ) { - if( regression ) + if( neighbor_responses ) { - double s = 0; - for( j = 0; j < k1; j++ ) - s += nr[j].f; - r = (float)(s*inv_scale); + float* nr = neighbor_responses->ptr(testidx + range.start); + for( j = 0; j < k; j++ ) + nr[j] = rbuf[testidx*k + j]; + for( ; j < k0; j++ ) + nr[j] = 0.f; } - else - { - int prev_start = 0, best_count = 0, cur_count; - Cv32suf best_val; - for( j = 0; j < k1; j++ ) - sort_buf[j].i = nr[j].i; + if( dists ) + { + float* dptr = dists->ptr(testidx + range.start); + for( j = 0; j < k; j++ ) + dptr[j] = dbuf[testidx*k + j]; + for( ; j < k0; j++ ) + dptr[j] = 0.f; + } - for( j = k1-1; j > 0; j-- ) + if( results || testidx+range.start == 0 ) + { + if( !params.isclassifier || k == 1 ) { - bool swap_fl = false; - for( j1 = 0; j1 < j; j1++ ) - if( sort_buf[j1].i > sort_buf[j1+1].i ) + float s = 0.f; + for( j = 0; j < k; j++ ) + s += rbuf[testidx*k + j]; + result = (float)(s*inv_scale); + } + else + { + float* rp = rbuf + testidx*k; + for( j = k-1; j > 0; j-- ) + { + bool swap_fl = false; + for( i = 0; i < j; i++ ) { - int t; - CV_SWAP( sort_buf[j1].i, sort_buf[j1+1].i, t ); - swap_fl = true; + if( rp[i] > rp[i+1] ) + { + std::swap(rp[i], rp[i+1]); + swap_fl = true; + } } - if( !swap_fl ) - break; - } + if( !swap_fl ) + break; + } - best_val.i = 0; - for( j = 1; j <= k1; j++ ) - if( j == k1 || sort_buf[j].i != sort_buf[j-1].i ) + result = rp[0]; + int prev_start = 0; + int best_count = 0; + for( j = 1; j <= k; j++ ) { - cur_count = j - prev_start; - if( best_count < cur_count ) + if( j == k || rp[j] != rp[j-1] ) { - best_count = cur_count; - best_val.i = sort_buf[j-1].i; + int count = j - prev_start; + if( best_count < count ) + { + best_count = count; + result = rp[j-1]; + } + prev_start = j; } - prev_start = j; } - r = best_val.f; + } + if( results ) + results->at(testidx + range.start) = result; + if( presult && testidx+range.start == 0 ) + *presult = result; } - - if( start+i == 0 ) - result = r; - - if( _results ) - _results->data.fl[(start + i)*rstep] = r; } + } - if( _neighbor_responses ) + struct findKNearestInvoker : public ParallelLoopBody + { + findKNearestInvoker(const KNearestImpl* _p, int _k, const Mat& __samples, + Mat* __results, Mat* __neighbor_responses, Mat* __dists, float* _presult) { - dst = (float*)(_neighbor_responses->data.ptr + - (start + i)*_neighbor_responses->step); - for( j = 0; j < k1; j++ ) - dst[j] = nr[j].f; - for( ; j < k; j++ ) - dst[j] = 0.f; + p = _p; + k = _k; + _samples = &__samples; + _results = __results; + _neighbor_responses = __neighbor_responses; + _dists = __dists; + presult = _presult; } - if( _dist ) + void operator()( const Range& range ) const { - dst = (float*)(_dist->data.ptr + (start + i)*_dist->step); - for( j = 0; j < k1; j++ ) - dst[j] = dist[j + i*k]; - for( ; j < k; j++ ) - dst[j] = 0.f; + int delta = std::min(range.end - range.start, 256); + for( int start = range.start; start < range.end; start += delta ) + { + p->findNearestCore( *_samples, k, Range(start, std::min(start + delta, range.end)), + _results, _neighbor_responses, _dists, presult ); + } } - } - - return result; -} -struct P1 : cv::ParallelLoopBody { - P1(const CvKNearest* _pointer, int _buf_sz, int _k, const CvMat* __samples, const float** __neighbors, - int _k1, CvMat* __results, CvMat* __neighbor_responses, CvMat* __dist, float* _result) - { - pointer = _pointer; - k = _k; - _samples = __samples; - _neighbors = __neighbors; - k1 = _k1; - _results = __results; - _neighbor_responses = __neighbor_responses; - _dist = __dist; - result = _result; - buf_sz = _buf_sz; - } - - const CvKNearest* pointer; - int k; - const CvMat* _samples; - const float** _neighbors; - int k1; - CvMat* _results; - CvMat* _neighbor_responses; - CvMat* _dist; - float* result; - int buf_sz; - - void operator()( const cv::Range& range ) const - { - cv::AutoBuffer buf(buf_sz); - for(int i = range.start; i < range.end; i += 1 ) + const KNearestImpl* p; + int k; + const Mat* _samples; + Mat* _results; + Mat* _neighbor_responses; + Mat* _dists; + float* presult; + }; + + float findNearest( InputArray _samples, int k, + OutputArray _results, + OutputArray _neighborResponses, + OutputArray _dists ) const { - float* neighbor_responses = &buf[0]; - float* dist = neighbor_responses + 1*k; - Cv32suf* sort_buf = (Cv32suf*)(dist + 1*k); - - pointer->find_neighbors_direct( _samples, k, i, i + 1, - neighbor_responses, _neighbors, dist ); + float result = 0.f; + CV_Assert( 0 < k ); - float r = pointer->write_results( k, k1, i, i + 1, neighbor_responses, dist, - _results, _neighbor_responses, _dist, sort_buf ); + Mat test_samples = _samples.getMat(); + CV_Assert( test_samples.type() == CV_32F && test_samples.cols == samples.cols ); + int testcount = test_samples.rows; - if( i == 0 ) - *result = r; - } - } - -}; - -float CvKNearest::find_nearest( const CvMat* _samples, int k, CvMat* _results, - const float** _neighbors, CvMat* _neighbor_responses, CvMat* _dist ) const -{ - float result = 0.f; - const int max_blk_count = 128, max_buf_sz = 1 << 12; - - if( !samples ) - CV_Error( CV_StsError, "The search tree must be constructed first using train method" ); - - if( !CV_IS_MAT(_samples) || - CV_MAT_TYPE(_samples->type) != CV_32FC1 || - _samples->cols != var_count ) - CV_Error( CV_StsBadArg, "Input samples must be floating-point matrix (x)" ); - - if( _results && (!CV_IS_MAT(_results) || - (_results->cols != 1 && _results->rows != 1) || - _results->cols + _results->rows - 1 != _samples->rows) ) - CV_Error( CV_StsBadArg, - "The results must be 1d vector containing as much elements as the number of samples" ); - - if( _results && CV_MAT_TYPE(_results->type) != CV_32FC1 && - (CV_MAT_TYPE(_results->type) != CV_32SC1 || regression)) - CV_Error( CV_StsUnsupportedFormat, - "The results must be floating-point or integer (in case of classification) vector" ); + if( testcount == 0 ) + { + _results.release(); + _neighborResponses.release(); + _dists.release(); + return 0.f; + } - if( k < 1 || k > max_k ) - CV_Error( CV_StsOutOfRange, "k must be within 1..max_k range" ); + Mat res, nr, d, *pres = 0, *pnr = 0, *pd = 0; + if( _results.needed() ) + { + _results.create(testcount, 1, CV_32F); + pres = &(res = _results.getMat()); + } + if( _neighborResponses.needed() ) + { + _neighborResponses.create(testcount, k, CV_32F); + pnr = &(nr = _neighborResponses.getMat()); + } + if( _dists.needed() ) + { + _dists.create(testcount, k, CV_32F); + pd = &(d = _dists.getMat()); + } - if( _neighbor_responses ) - { - if( !CV_IS_MAT(_neighbor_responses) || CV_MAT_TYPE(_neighbor_responses->type) != CV_32FC1 || - _neighbor_responses->rows != _samples->rows || _neighbor_responses->cols != k ) - CV_Error( CV_StsBadArg, - "The neighbor responses (if present) must be floating-point matrix of x size" ); + findKNearestInvoker invoker(this, k, test_samples, pres, pnr, pd, &result); + parallel_for_(Range(0, testcount), invoker); + //invoker(Range(0, testcount)); + return result; } - if( _dist ) + float predict(InputArray inputs, OutputArray outputs, int) const { - if( !CV_IS_MAT(_dist) || CV_MAT_TYPE(_dist->type) != CV_32FC1 || - _dist->rows != _samples->rows || _dist->cols != k ) - CV_Error( CV_StsBadArg, - "The distances from the neighbors (if present) must be floating-point matrix of x size" ); + return findNearest( inputs, params.defaultK, outputs, noArray(), noArray() ); } - int count = _samples->rows; - int count_scale = k*2; - int blk_count0 = MIN( count, max_blk_count ); - int buf_sz = MIN( blk_count0 * count_scale, max_buf_sz ); - blk_count0 = MAX( buf_sz/count_scale, 1 ); - blk_count0 += blk_count0 % 2; - blk_count0 = MIN( blk_count0, count ); - buf_sz = blk_count0 * count_scale + k; - int k1 = get_sample_count(); - k1 = MIN( k1, k ); - - cv::parallel_for_(cv::Range(0, count), P1(this, buf_sz, k, _samples, _neighbors, k1, - _results, _neighbor_responses, _dist, &result) - ); - - return result; -} - - -using namespace cv; - -CvKNearest::CvKNearest( const Mat& _train_data, const Mat& _responses, - const Mat& _sample_idx, bool _is_regression, int _max_k ) -{ - samples = 0; - train(_train_data, _responses, _sample_idx, _is_regression, _max_k, false ); -} - -bool CvKNearest::train( const Mat& _train_data, const Mat& _responses, - const Mat& _sample_idx, bool _is_regression, - int _max_k, bool _update_base ) -{ - CvMat tdata = _train_data, responses = _responses, sidx = _sample_idx; - - return train(&tdata, &responses, sidx.data.ptr ? &sidx : 0, _is_regression, _max_k, _update_base ); -} - - -float CvKNearest::find_nearest( const Mat& _samples, int k, Mat* _results, - const float** _neighbors, Mat* _neighbor_responses, - Mat* _dist ) const -{ - CvMat s = _samples, results, *presults = 0, nresponses, *pnresponses = 0, dist, *pdist = 0; - - if( _results ) + void write( FileStorage& fs ) const { - if(!(_results->data && (_results->type() == CV_32F || - (_results->type() == CV_32S && regression)) && - (_results->cols == 1 || _results->rows == 1) && - _results->cols + _results->rows - 1 == _samples.rows) ) - _results->create(_samples.rows, 1, CV_32F); - presults = &(results = *_results); - } + fs << "is_classifier" << (int)params.isclassifier; + fs << "default_k" << params.defaultK; - if( _neighbor_responses ) - { - if(!(_neighbor_responses->data && _neighbor_responses->type() == CV_32F && - _neighbor_responses->cols == k && _neighbor_responses->rows == _samples.rows) ) - _neighbor_responses->create(_samples.rows, k, CV_32F); - pnresponses = &(nresponses = *_neighbor_responses); + fs << "samples" << samples; + fs << "responses" << responses; } - if( _dist ) + void read( const FileNode& fn ) { - if(!(_dist->data && _dist->type() == CV_32F && - _dist->cols == k && _dist->rows == _samples.rows) ) - _dist->create(_samples.rows, k, CV_32F); - pdist = &(dist = *_dist); - } + clear(); + params.isclassifier = (int)fn["is_classifier"] != 0; + params.defaultK = (int)fn["default_k"]; - return find_nearest(&s, k, presults, _neighbors, pnresponses, pdist ); -} + fn["samples"] >> samples; + fn["responses"] >> responses; + } + Mat samples; + Mat responses; + Params params; +}; -float CvKNearest::find_nearest( const cv::Mat& _samples, int k, CV_OUT cv::Mat& results, - CV_OUT cv::Mat& neighborResponses, CV_OUT cv::Mat& dists) const +Ptr KNearest::create(const Params& p) { - return find_nearest(_samples, k, &results, 0, &neighborResponses, &dists); + return makePtr(p); +} + +} } /* End of file */ diff --git a/modules/ml/src/ml_init.cpp b/modules/ml/src/ml_init.cpp deleted file mode 100644 index fcf9e1c892416b32675a90f54a2ec1f7cc6063f9..0000000000000000000000000000000000000000 --- a/modules/ml/src/ml_init.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009, Willow Garage Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ - -#include "precomp.hpp" - -namespace cv -{ - -CV_INIT_ALGORITHM(EM, "StatModel.EM", - obj.info()->addParam(obj, "nclusters", obj.nclusters); - obj.info()->addParam(obj, "covMatType", obj.covMatType); - obj.info()->addParam(obj, "maxIters", obj.maxIters); - obj.info()->addParam(obj, "epsilon", obj.epsilon); - obj.info()->addParam(obj, "weights", obj.weights, true); - obj.info()->addParam(obj, "means", obj.means, true); - obj.info()->addParam(obj, "covs", obj.covs, true)) - -bool initModule_ml(void) -{ - Ptr em = createEM_ptr_hidden(); - return em->info() != 0; -} - -} diff --git a/modules/ml/src/nbayes.cpp b/modules/ml/src/nbayes.cpp index 938f3fbd8dc62efdf74d54ecd028456344e81c90..2dbbcdf69012ba9ef10f3ff8c1e4e4a9e66c67a5 100644 --- a/modules/ml/src/nbayes.cpp +++ b/modules/ml/src/nbayes.cpp @@ -40,622 +40,428 @@ #include "precomp.hpp" -CvNormalBayesClassifier::CvNormalBayesClassifier() -{ - var_count = var_all = 0; - var_idx = 0; - cls_labels = 0; - count = 0; - sum = 0; - productsum = 0; - avg = 0; - inv_eigen_values = 0; - cov_rotate_mats = 0; - c = 0; - default_model_name = "my_nb"; -} +namespace cv { +namespace ml { +NormalBayesClassifier::Params::Params() {} -void CvNormalBayesClassifier::clear() +class NormalBayesClassifierImpl : public NormalBayesClassifier { - if( cls_labels ) +public: + NormalBayesClassifierImpl() { - for( int cls = 0; cls < cls_labels->cols; cls++ ) - { - cvReleaseMat( &count[cls] ); - cvReleaseMat( &sum[cls] ); - cvReleaseMat( &productsum[cls] ); - cvReleaseMat( &avg[cls] ); - cvReleaseMat( &inv_eigen_values[cls] ); - cvReleaseMat( &cov_rotate_mats[cls] ); - } + nallvars = 0; } - cvReleaseMat( &cls_labels ); - cvReleaseMat( &var_idx ); - cvReleaseMat( &c ); - cvFree( &count ); -} - + void setParams(const Params&) {} + Params getParams() const { return Params(); } -CvNormalBayesClassifier::~CvNormalBayesClassifier() -{ - clear(); -} - - -CvNormalBayesClassifier::CvNormalBayesClassifier( - const CvMat* _train_data, const CvMat* _responses, - const CvMat* _var_idx, const CvMat* _sample_idx ) -{ - var_count = var_all = 0; - var_idx = 0; - cls_labels = 0; - count = 0; - sum = 0; - productsum = 0; - avg = 0; - inv_eigen_values = 0; - cov_rotate_mats = 0; - c = 0; - default_model_name = "my_nb"; - - train( _train_data, _responses, _var_idx, _sample_idx ); -} - - -bool CvNormalBayesClassifier::train( const CvMat* _train_data, const CvMat* _responses, - const CvMat* _var_idx, const CvMat* _sample_idx, bool update ) -{ - const float min_variation = FLT_EPSILON; - bool result = false; - CvMat* responses = 0; - const float** train_data = 0; - CvMat* __cls_labels = 0; - CvMat* __var_idx = 0; - CvMat* cov = 0; - - CV_FUNCNAME( "CvNormalBayesClassifier::train" ); - - __BEGIN__; - - int cls, nsamples = 0, _var_count = 0, _var_all = 0, nclasses = 0; - int s, c1, c2; - const int* responses_data; - - CV_CALL( cvPrepareTrainData( 0, - _train_data, CV_ROW_SAMPLE, _responses, CV_VAR_CATEGORICAL, - _var_idx, _sample_idx, false, &train_data, - &nsamples, &_var_count, &_var_all, &responses, - &__cls_labels, &__var_idx )); - - if( !update ) + bool train( const Ptr& trainData, int flags ) { - const size_t mat_size = sizeof(CvMat*); - size_t data_size; - - clear(); + const float min_variation = FLT_EPSILON; + Mat responses = trainData->getNormCatResponses(); + Mat __cls_labels = trainData->getClassLabels(); + Mat __var_idx = trainData->getVarIdx(); + Mat samples = trainData->getTrainSamples(); + int nclasses = (int)__cls_labels.total(); - var_idx = __var_idx; - cls_labels = __cls_labels; - __var_idx = __cls_labels = 0; - var_count = _var_count; - var_all = _var_all; + int nvars = trainData->getNVars(); + int s, c1, c2, cls; - nclasses = cls_labels->cols; - data_size = nclasses*6*mat_size; + int __nallvars = trainData->getNAllVars(); + bool update = (flags & UPDATE_MODEL) != 0; - CV_CALL( count = (CvMat**)cvAlloc( data_size )); - memset( count, 0, data_size ); - - sum = count + nclasses; - productsum = sum + nclasses; - avg = productsum + nclasses; - inv_eigen_values= avg + nclasses; - cov_rotate_mats = inv_eigen_values + nclasses; + if( !update ) + { + nallvars = __nallvars; + count.resize(nclasses); + sum.resize(nclasses); + productsum.resize(nclasses); + avg.resize(nclasses); + inv_eigen_values.resize(nclasses); + cov_rotate_mats.resize(nclasses); + + for( cls = 0; cls < nclasses; cls++ ) + { + count[cls] = Mat::zeros( 1, nvars, CV_32SC1 ); + sum[cls] = Mat::zeros( 1, nvars, CV_64FC1 ); + productsum[cls] = Mat::zeros( nvars, nvars, CV_64FC1 ); + avg[cls] = Mat::zeros( 1, nvars, CV_64FC1 ); + inv_eigen_values[cls] = Mat::zeros( 1, nvars, CV_64FC1 ); + cov_rotate_mats[cls] = Mat::zeros( nvars, nvars, CV_64FC1 ); + } - CV_CALL( c = cvCreateMat( 1, nclasses, CV_64FC1 )); + var_idx = __var_idx; + cls_labels = __cls_labels; - for( cls = 0; cls < nclasses; cls++ ) + c.create(1, nclasses, CV_64FC1); + } + else { - CV_CALL(count[cls] = cvCreateMat( 1, var_count, CV_32SC1 )); - CV_CALL(sum[cls] = cvCreateMat( 1, var_count, CV_64FC1 )); - CV_CALL(productsum[cls] = cvCreateMat( var_count, var_count, CV_64FC1 )); - CV_CALL(avg[cls] = cvCreateMat( 1, var_count, CV_64FC1 )); - CV_CALL(inv_eigen_values[cls] = cvCreateMat( 1, var_count, CV_64FC1 )); - CV_CALL(cov_rotate_mats[cls] = cvCreateMat( var_count, var_count, CV_64FC1 )); - CV_CALL(cvZero( count[cls] )); - CV_CALL(cvZero( sum[cls] )); - CV_CALL(cvZero( productsum[cls] )); - CV_CALL(cvZero( avg[cls] )); - CV_CALL(cvZero( inv_eigen_values[cls] )); - CV_CALL(cvZero( cov_rotate_mats[cls] )); + // check that the new training data has the same dimensionality etc. + if( nallvars != __nallvars || + var_idx.size() != __var_idx.size() || + norm(var_idx, __var_idx, NORM_INF) != 0 || + cls_labels.size() != __cls_labels.size() || + norm(cls_labels, __cls_labels, NORM_INF) != 0 ) + CV_Error( CV_StsBadArg, + "The new training data is inconsistent with the original training data; varIdx and the class labels should be the same" ); } - } - else - { - // check that the new training data has the same dimensionality etc. - if( _var_count != var_count || _var_all != var_all || !((!_var_idx && !var_idx) || - (_var_idx && var_idx && cvNorm(_var_idx,var_idx,CV_C) < DBL_EPSILON)) ) - CV_ERROR( CV_StsBadArg, - "The new training data is inconsistent with the original training data" ); - - if( cls_labels->cols != __cls_labels->cols || - cvNorm(cls_labels, __cls_labels, CV_C) > DBL_EPSILON ) - CV_ERROR( CV_StsNotImplemented, - "In the current implementation the new training data must have absolutely " - "the same set of class labels as used in the original training data" ); - - nclasses = cls_labels->cols; - } - responses_data = responses->data.i; - CV_CALL( cov = cvCreateMat( _var_count, _var_count, CV_64FC1 )); - - /* process train data (count, sum , productsum) */ - for( s = 0; s < nsamples; s++ ) - { - cls = responses_data[s]; - int* count_data = count[cls]->data.i; - double* sum_data = sum[cls]->data.db; - double* prod_data = productsum[cls]->data.db; - const float* train_vec = train_data[s]; + Mat cov( nvars, nvars, CV_64FC1 ); + int nsamples = samples.rows; - for( c1 = 0; c1 < _var_count; c1++, prod_data += _var_count ) + // process train data (count, sum , productsum) + for( s = 0; s < nsamples; s++ ) { - double val1 = train_vec[c1]; - sum_data[c1] += val1; - count_data[c1]++; - for( c2 = c1; c2 < _var_count; c2++ ) - prod_data[c2] += train_vec[c2]*val1; - } - } - cvReleaseMat( &responses ); - responses = 0; + cls = responses.at(s); + int* count_data = count[cls].ptr(); + double* sum_data = sum[cls].ptr(); + double* prod_data = productsum[cls].ptr(); + const float* train_vec = samples.ptr(s); - /* calculate avg, covariance matrix, c */ - for( cls = 0; cls < nclasses; cls++ ) - { - double det = 1; - int i, j; - CvMat* w = inv_eigen_values[cls]; - int* count_data = count[cls]->data.i; - double* avg_data = avg[cls]->data.db; - double* sum1 = sum[cls]->data.db; + for( c1 = 0; c1 < nvars; c1++, prod_data += nvars ) + { + double val1 = train_vec[c1]; + sum_data[c1] += val1; + count_data[c1]++; + for( c2 = c1; c2 < nvars; c2++ ) + prod_data[c2] += train_vec[c2]*val1; + } + } - cvCompleteSymm( productsum[cls], 0 ); + Mat vt; - for( j = 0; j < _var_count; j++ ) + // calculate avg, covariance matrix, c + for( cls = 0; cls < nclasses; cls++ ) { - int n = count_data[j]; - avg_data[j] = n ? sum1[j] / n : 0.; - } + double det = 1; + int i, j; + Mat& w = inv_eigen_values[cls]; + int* count_data = count[cls].ptr(); + double* avg_data = avg[cls].ptr(); + double* sum1 = sum[cls].ptr(); - count_data = count[cls]->data.i; - avg_data = avg[cls]->data.db; - sum1 = sum[cls]->data.db; + completeSymm(productsum[cls], 0); - for( i = 0; i < _var_count; i++ ) - { - double* avg2_data = avg[cls]->data.db; - double* sum2 = sum[cls]->data.db; - double* prod_data = productsum[cls]->data.db + i*_var_count; - double* cov_data = cov->data.db + i*_var_count; - double s1val = sum1[i]; - double avg1 = avg_data[i]; - int _count = count_data[i]; - - for( j = 0; j <= i; j++ ) + for( j = 0; j < nvars; j++ ) { - double avg2 = avg2_data[j]; - double cov_val = prod_data[j] - avg1 * sum2[j] - avg2 * s1val + avg1 * avg2 * _count; - cov_val = (_count > 1) ? cov_val / (_count - 1) : cov_val; - cov_data[j] = cov_val; + int n = count_data[j]; + avg_data[j] = n ? sum1[j] / n : 0.; } - } - CV_CALL( cvCompleteSymm( cov, 1 )); - CV_CALL( cvSVD( cov, w, cov_rotate_mats[cls], 0, CV_SVD_U_T )); - CV_CALL( cvMaxS( w, min_variation, w )); - for( j = 0; j < _var_count; j++ ) - det *= w->data.db[j]; + count_data = count[cls].ptr(); + avg_data = avg[cls].ptr(); + sum1 = sum[cls].ptr(); - CV_CALL( cvDiv( NULL, w, w )); - c->data.db[cls] = det > 0 ? log(det) : -700; - } - - result = true; - - __END__; + for( i = 0; i < nvars; i++ ) + { + double* avg2_data = avg[cls].ptr(); + double* sum2 = sum[cls].ptr(); + double* prod_data = productsum[cls].ptr(i); + double* cov_data = cov.ptr(i); + double s1val = sum1[i]; + double avg1 = avg_data[i]; + int _count = count_data[i]; + + for( j = 0; j <= i; j++ ) + { + double avg2 = avg2_data[j]; + double cov_val = prod_data[j] - avg1 * sum2[j] - avg2 * s1val + avg1 * avg2 * _count; + cov_val = (_count > 1) ? cov_val / (_count - 1) : cov_val; + cov_data[j] = cov_val; + } + } - if( !result || cvGetErrStatus() < 0 ) - clear(); + completeSymm( cov, 1 ); - cvReleaseMat( &cov ); - cvReleaseMat( &__cls_labels ); - cvReleaseMat( &__var_idx ); - cvFree( &train_data ); + SVD::compute(cov, w, cov_rotate_mats[cls], noArray()); + transpose(cov_rotate_mats[cls], cov_rotate_mats[cls]); + cv::max(w, min_variation, w); + for( j = 0; j < nvars; j++ ) + det *= w.at(j); - return result; -} + divide(1., w, w); + c.at(cls) = det > 0 ? log(det) : -700; + } -struct predict_body : cv::ParallelLoopBody { - predict_body(CvMat* _c, CvMat** _cov_rotate_mats, CvMat** _inv_eigen_values, CvMat** _avg, - const CvMat* _samples, const int* _vidx, CvMat* _cls_labels, - CvMat* _results, float* _value, int _var_count1, CvMat* _results_prob - ) - { - c = _c; - cov_rotate_mats = _cov_rotate_mats; - inv_eigen_values = _inv_eigen_values; - avg = _avg; - samples = _samples; - vidx = _vidx; - cls_labels = _cls_labels; - results = _results; - value = _value; - var_count1 = _var_count1; - results_prob = _results_prob; - } - - CvMat* c; - CvMat** cov_rotate_mats; - CvMat** inv_eigen_values; - CvMat** avg; - const CvMat* samples; - const int* vidx; - CvMat* cls_labels; - - CvMat* results_prob; - CvMat* results; - float* value; - int var_count1; - - void operator()( const cv::Range& range ) const - { - - int cls = -1; - int rtype = 0, rstep = 0, rptype = 0, rpstep = 0; - int nclasses = cls_labels->cols; - int _var_count = avg[0]->cols; - double probability = 0; - - if (results) - { - rtype = CV_MAT_TYPE(results->type); - rstep = CV_IS_MAT_CONT(results->type) ? 1 : results->step/CV_ELEM_SIZE(rtype); - } - if (results_prob) - { - rptype = CV_MAT_TYPE(results_prob->type); - rpstep = CV_IS_MAT_CONT(results_prob->type) ? 1 : results_prob->step/CV_ELEM_SIZE(rptype); + return true; } - // allocate memory and initializing headers for calculating - cv::AutoBuffer buffer(nclasses + var_count1); - CvMat diff = cvMat( 1, var_count1, CV_64FC1, &buffer[0] ); - for(int k = range.start; k < range.end; k += 1 ) + class NBPredictBody : public ParallelLoopBody { - int ival; - double opt = FLT_MAX; - - for(int i = 0; i < nclasses; i++ ) + public: + NBPredictBody( const Mat& _c, const vector& _cov_rotate_mats, + const vector& _inv_eigen_values, + const vector& _avg, + const Mat& _samples, const Mat& _vidx, const Mat& _cls_labels, + Mat& _results, Mat& _results_prob, bool _rawOutput ) { - double cur = c->data.db[i]; - CvMat* u = cov_rotate_mats[i]; - CvMat* w = inv_eigen_values[i]; + c = &_c; + cov_rotate_mats = &_cov_rotate_mats; + inv_eigen_values = &_inv_eigen_values; + avg = &_avg; + samples = &_samples; + vidx = &_vidx; + cls_labels = &_cls_labels; + results = &_results; + results_prob = _results_prob.data ? &_results_prob : 0; + rawOutput = _rawOutput; + } - const double* avg_data = avg[i]->data.db; - const float* x = (const float*)(samples->data.ptr + samples->step*k); + const Mat* c; + const vector* cov_rotate_mats; + const vector* inv_eigen_values; + const vector* avg; + const Mat* samples; + const Mat* vidx; + const Mat* cls_labels; - // cov = u w u' --> cov^(-1) = u w^(-1) u' - for(int j = 0; j < _var_count; j++ ) - diff.data.db[j] = avg_data[j] - x[vidx ? vidx[j] : j]; + Mat* results_prob; + Mat* results; + float* value; + bool rawOutput; - cvGEMM( &diff, u, 1, 0, 0, &diff, CV_GEMM_B_T ); - for(int j = 0; j < _var_count; j++ ) + void operator()( const Range& range ) const + { + int cls = -1; + int rtype = 0, rptype = 0; + size_t rstep = 0, rpstep = 0; + int nclasses = (int)cls_labels->total(); + int nvars = avg->at(0).cols; + double probability = 0; + const int* vptr = vidx && !vidx->empty() ? vidx->ptr() : 0; + + if (results) { - double d = diff.data.db[j]; - cur += d*d*w->data.db[j]; + rtype = results->type(); + rstep = results->isContinuous() ? 1 : results->step/results->elemSize(); } - - if( cur < opt ) + if (results_prob) { - cls = i; - opt = cur; + rptype = results_prob->type(); + rpstep = results_prob->isContinuous() ? 1 : results_prob->step/results_prob->elemSize(); + } + // allocate memory and initializing headers for calculating + cv::AutoBuffer _buffer(nvars*2); + double* _diffin = _buffer; + double* _diffout = _buffer + nvars; + Mat diffin( 1, nvars, CV_64FC1, _diffin ); + Mat diffout( 1, nvars, CV_64FC1, _diffout ); + + for(int k = range.start; k < range.end; k++ ) + { + double opt = FLT_MAX; + + for(int i = 0; i < nclasses; i++ ) + { + double cur = c->at(i); + const Mat& u = cov_rotate_mats->at(i); + const Mat& w = inv_eigen_values->at(i); + + const double* avg_data = avg->at(i).ptr(); + const float* x = samples->ptr(k); + + // cov = u w u' --> cov^(-1) = u w^(-1) u' + for(int j = 0; j < nvars; j++ ) + _diffin[j] = avg_data[j] - x[vptr ? vptr[j] : j]; + + gemm( diffin, u, 1, noArray(), 0, diffout, GEMM_2_T ); + for(int j = 0; j < nvars; j++ ) + { + double d = _diffout[j]; + cur += d*d*w.ptr()[j]; + } + + if( cur < opt ) + { + cls = i; + opt = cur; + } + probability = exp( -0.5 * cur ); + + if( results_prob ) + { + if ( rptype == CV_32FC1 ) + results_prob->ptr()[k*rpstep + i] = (float)probability; + else + results_prob->ptr()[k*rpstep + i] = probability; + } + } + + int ival = rawOutput ? cls : cls_labels->at(cls); + if( results ) + { + if( rtype == CV_32SC1 ) + results->ptr()[k*rstep] = ival; + else + results->ptr()[k*rstep] = (float)ival; + } } - /* probability = exp( -0.5 * cur ) */ - probability = exp( -0.5 * cur ); - } - - ival = cls_labels->data.i[cls]; - if( results ) - { - if( rtype == CV_32SC1 ) - results->data.i[k*rstep] = ival; - else - results->data.fl[k*rstep] = (float)ival; - } - if ( results_prob ) - { - if ( rptype == CV_32FC1 ) - results_prob->data.fl[k*rpstep] = (float)probability; - else - results_prob->data.db[k*rpstep] = probability; } - if( k == 0 ) - *value = (float)ival; - } - } -}; - - -float CvNormalBayesClassifier::predict( const CvMat* samples, CvMat* results, CvMat* results_prob ) const -{ - float value = 0; - - if( !CV_IS_MAT(samples) || CV_MAT_TYPE(samples->type) != CV_32FC1 || samples->cols != var_all ) - CV_Error( CV_StsBadArg, - "The input samples must be 32f matrix with the number of columns = var_all" ); + }; - if( samples->rows > 1 && !results ) - CV_Error( CV_StsNullPtr, - "When the number of input samples is >1, the output vector of results must be passed" ); - - if( results ) + float predict( InputArray _samples, OutputArray _results, int flags ) const { - if( !CV_IS_MAT(results) || (CV_MAT_TYPE(results->type) != CV_32FC1 && - CV_MAT_TYPE(results->type) != CV_32SC1) || - (results->cols != 1 && results->rows != 1) || - results->cols + results->rows - 1 != samples->rows ) - CV_Error( CV_StsBadArg, "The output array must be integer or floating-point vector " - "with the number of elements = number of rows in the input matrix" ); + return predictProb(_samples, _results, noArray(), flags); } - if( results_prob ) + float predictProb( InputArray _samples, OutputArray _results, OutputArray _resultsProb, int flags ) const { - if( !CV_IS_MAT(results_prob) || (CV_MAT_TYPE(results_prob->type) != CV_32FC1 && - CV_MAT_TYPE(results_prob->type) != CV_64FC1) || - (results_prob->cols != 1 && results_prob->rows != 1) || - results_prob->cols + results_prob->rows - 1 != samples->rows ) - CV_Error( CV_StsBadArg, "The output array must be double or float vector " - "with the number of elements = number of rows in the input matrix" ); - } + int value=0; + Mat samples = _samples.getMat(), results, resultsProb; + int nsamples = samples.rows, nclasses = (int)cls_labels.total(); + bool rawOutput = (flags & RAW_OUTPUT) != 0; - const int* vidx = var_idx ? var_idx->data.i : 0; + if( samples.type() != CV_32F || samples.cols != nallvars ) + CV_Error( CV_StsBadArg, + "The input samples must be 32f matrix with the number of columns = nallvars" ); - cv::parallel_for_(cv::Range(0, samples->rows), - predict_body(c, cov_rotate_mats, inv_eigen_values, avg, samples, - vidx, cls_labels, results, &value, var_count, results_prob)); + if( samples.rows > 1 && _results.needed() ) + CV_Error( CV_StsNullPtr, + "When the number of input samples is >1, the output vector of results must be passed" ); - return value; -} + if( _results.needed() ) + { + _results.create(nsamples, 1, CV_32S); + results = _results.getMat(); + } + else + results = Mat(1, 1, CV_32S, &value); + if( _resultsProb.needed() ) + { + _resultsProb.create(nsamples, nclasses, CV_32F); + resultsProb = _resultsProb.getMat(); + } -void CvNormalBayesClassifier::write( CvFileStorage* fs, const char* name ) const -{ - CV_FUNCNAME( "CvNormalBayesClassifier::write" ); + cv::parallel_for_(cv::Range(0, nsamples), + NBPredictBody(c, cov_rotate_mats, inv_eigen_values, avg, samples, + var_idx, cls_labels, results, resultsProb, rawOutput)); - __BEGIN__; + return (float)value; + } - int nclasses, i; + void write( FileStorage& fs ) const + { + int nclasses = (int)cls_labels.total(), i; - nclasses = cls_labels->cols; + fs << "var_count" << (var_idx.empty() ? nallvars : (int)var_idx.total()); + fs << "var_all" << nallvars; - cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_NBAYES ); + if( !var_idx.empty() ) + fs << "var_idx" << var_idx; + fs << "cls_labels" << cls_labels; - CV_CALL( cvWriteInt( fs, "var_count", var_count )); - CV_CALL( cvWriteInt( fs, "var_all", var_all )); + fs << "count" << "["; + for( i = 0; i < nclasses; i++ ) + fs << count[i]; - if( var_idx ) - CV_CALL( cvWrite( fs, "var_idx", var_idx )); - CV_CALL( cvWrite( fs, "cls_labels", cls_labels )); + fs << "]" << "sum" << "["; + for( i = 0; i < nclasses; i++ ) + fs << sum[i]; - CV_CALL( cvStartWriteStruct( fs, "count", CV_NODE_SEQ )); - for( i = 0; i < nclasses; i++ ) - CV_CALL( cvWrite( fs, NULL, count[i] )); - CV_CALL( cvEndWriteStruct( fs )); + fs << "]" << "productsum" << "["; + for( i = 0; i < nclasses; i++ ) + fs << productsum[i]; - CV_CALL( cvStartWriteStruct( fs, "sum", CV_NODE_SEQ )); - for( i = 0; i < nclasses; i++ ) - CV_CALL( cvWrite( fs, NULL, sum[i] )); - CV_CALL( cvEndWriteStruct( fs )); + fs << "]" << "avg" << "["; + for( i = 0; i < nclasses; i++ ) + fs << avg[i]; - CV_CALL( cvStartWriteStruct( fs, "productsum", CV_NODE_SEQ )); - for( i = 0; i < nclasses; i++ ) - CV_CALL( cvWrite( fs, NULL, productsum[i] )); - CV_CALL( cvEndWriteStruct( fs )); + fs << "]" << "inv_eigen_values" << "["; + for( i = 0; i < nclasses; i++ ) + fs << inv_eigen_values[i]; - CV_CALL( cvStartWriteStruct( fs, "avg", CV_NODE_SEQ )); - for( i = 0; i < nclasses; i++ ) - CV_CALL( cvWrite( fs, NULL, avg[i] )); - CV_CALL( cvEndWriteStruct( fs )); + fs << "]" << "cov_rotate_mats" << "["; + for( i = 0; i < nclasses; i++ ) + fs << cov_rotate_mats[i]; - CV_CALL( cvStartWriteStruct( fs, "inv_eigen_values", CV_NODE_SEQ )); - for( i = 0; i < nclasses; i++ ) - CV_CALL( cvWrite( fs, NULL, inv_eigen_values[i] )); - CV_CALL( cvEndWriteStruct( fs )); + fs << "]"; - CV_CALL( cvStartWriteStruct( fs, "cov_rotate_mats", CV_NODE_SEQ )); - for( i = 0; i < nclasses; i++ ) - CV_CALL( cvWrite( fs, NULL, cov_rotate_mats[i] )); - CV_CALL( cvEndWriteStruct( fs )); + fs << "c" << c; + } - CV_CALL( cvWrite( fs, "c", c )); + void read( const FileNode& fn ) + { + clear(); - cvEndWriteStruct( fs ); + fn["var_all"] >> nallvars; - __END__; -} + if( nallvars <= 0 ) + CV_Error( CV_StsParseError, + "The field \"var_count\" of NBayes classifier is missing or non-positive" ); + fn["var_idx"] >> var_idx; + fn["cls_labels"] >> cls_labels; -void CvNormalBayesClassifier::read( CvFileStorage* fs, CvFileNode* root_node ) -{ - bool ok = false; - CV_FUNCNAME( "CvNormalBayesClassifier::read" ); - - __BEGIN__; - - int nclasses, i; - size_t data_size; - CvFileNode* node; - CvSeq* seq; - CvSeqReader reader; - - clear(); - - CV_CALL( var_count = cvReadIntByName( fs, root_node, "var_count", -1 )); - CV_CALL( var_all = cvReadIntByName( fs, root_node, "var_all", -1 )); - CV_CALL( var_idx = (CvMat*)cvReadByName( fs, root_node, "var_idx" )); - CV_CALL( cls_labels = (CvMat*)cvReadByName( fs, root_node, "cls_labels" )); - if( !cls_labels ) - CV_ERROR( CV_StsParseError, "No \"cls_labels\" in NBayes classifier" ); - if( cls_labels->cols < 1 ) - CV_ERROR( CV_StsBadArg, "Number of classes is less 1" ); - if( var_count <= 0 ) - CV_ERROR( CV_StsParseError, - "The field \"var_count\" of NBayes classifier is missing" ); - nclasses = cls_labels->cols; - - data_size = nclasses*6*sizeof(CvMat*); - CV_CALL( count = (CvMat**)cvAlloc( data_size )); - memset( count, 0, data_size ); - - sum = count + nclasses; - productsum = sum + nclasses; - avg = productsum + nclasses; - inv_eigen_values = avg + nclasses; - cov_rotate_mats = inv_eigen_values + nclasses; - - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "count" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) || seq->total != nclasses) - CV_ERROR( CV_StsBadArg, "" ); - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - for( i = 0; i < nclasses; i++ ) - { - CV_CALL( count[i] = (CvMat*)cvRead( fs, (CvFileNode*)reader.ptr )); - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - } + int nclasses = (int)cls_labels.total(), i; - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "sum" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) || seq->total != nclasses) - CV_ERROR( CV_StsBadArg, "" ); - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - for( i = 0; i < nclasses; i++ ) - { - CV_CALL( sum[i] = (CvMat*)cvRead( fs, (CvFileNode*)reader.ptr )); - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - } + if( cls_labels.empty() || nclasses < 1 ) + CV_Error( CV_StsParseError, "No or invalid \"cls_labels\" in NBayes classifier" ); - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "productsum" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) || seq->total != nclasses) - CV_ERROR( CV_StsBadArg, "" ); - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - for( i = 0; i < nclasses; i++ ) - { - CV_CALL( productsum[i] = (CvMat*)cvRead( fs, (CvFileNode*)reader.ptr )); - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - } + FileNodeIterator + count_it = fn["count"].begin(), + sum_it = fn["sum"].begin(), + productsum_it = fn["productsum"].begin(), + avg_it = fn["avg"].begin(), + inv_eigen_values_it = fn["inv_eigen_values"].begin(), + cov_rotate_mats_it = fn["cov_rotate_mats"].begin(); - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "avg" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) || seq->total != nclasses) - CV_ERROR( CV_StsBadArg, "" ); - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - for( i = 0; i < nclasses; i++ ) - { - CV_CALL( avg[i] = (CvMat*)cvRead( fs, (CvFileNode*)reader.ptr )); - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - } + count.resize(nclasses); + sum.resize(nclasses); + productsum.resize(nclasses); + avg.resize(nclasses); + inv_eigen_values.resize(nclasses); + cov_rotate_mats.resize(nclasses); - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "inv_eigen_values" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) || seq->total != nclasses) - CV_ERROR( CV_StsBadArg, "" ); - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - for( i = 0; i < nclasses; i++ ) - { - CV_CALL( inv_eigen_values[i] = (CvMat*)cvRead( fs, (CvFileNode*)reader.ptr )); - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); + for( i = 0; i < nclasses; i++, ++count_it, ++sum_it, ++productsum_it, ++avg_it, + ++inv_eigen_values_it, ++cov_rotate_mats_it ) + { + *count_it >> count[i]; + *sum_it >> sum[i]; + *productsum_it >> productsum[i]; + *avg_it >> avg[i]; + *inv_eigen_values_it >> inv_eigen_values[i]; + *cov_rotate_mats_it >> cov_rotate_mats[i]; + } + + fn["c"] >> c; } - CV_CALL( node = cvGetFileNodeByName( fs, root_node, "cov_rotate_mats" )); - seq = node->data.seq; - if( !CV_NODE_IS_SEQ(node->tag) || seq->total != nclasses) - CV_ERROR( CV_StsBadArg, "" ); - CV_CALL( cvStartReadSeq( seq, &reader, 0 )); - for( i = 0; i < nclasses; i++ ) + void clear() { - CV_CALL( cov_rotate_mats[i] = (CvMat*)cvRead( fs, (CvFileNode*)reader.ptr )); - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); + count.clear(); + sum.clear(); + productsum.clear(); + avg.clear(); + inv_eigen_values.clear(); + cov_rotate_mats.clear(); + + var_idx.release(); + cls_labels.release(); + c.release(); + nallvars = 0; } - CV_CALL( c = (CvMat*)cvReadByName( fs, root_node, "c" )); - - ok = true; + bool isTrained() const { return !avg.empty(); } + bool isClassifier() const { return true; } + int getVarCount() const { return nallvars; } + String getDefaultModelName() const { return "opencv_ml_nbayes"; } - __END__; - - if( !ok ) - clear(); -} + int nallvars; + Mat var_idx, cls_labels, c; + vector count, sum, productsum, avg, inv_eigen_values, cov_rotate_mats; +}; -using namespace cv; -CvNormalBayesClassifier::CvNormalBayesClassifier( const Mat& _train_data, const Mat& _responses, - const Mat& _var_idx, const Mat& _sample_idx ) +Ptr NormalBayesClassifier::create(const Params&) { - var_count = var_all = 0; - var_idx = 0; - cls_labels = 0; - count = 0; - sum = 0; - productsum = 0; - avg = 0; - inv_eigen_values = 0; - cov_rotate_mats = 0; - c = 0; - default_model_name = "my_nb"; - - CvMat tdata = _train_data, responses = _responses, vidx = _var_idx, sidx = _sample_idx; - train(&tdata, &responses, vidx.data.ptr ? &vidx : 0, - sidx.data.ptr ? &sidx : 0); + Ptr p = makePtr(); + return p; } -bool CvNormalBayesClassifier::train( const Mat& _train_data, const Mat& _responses, - const Mat& _var_idx, const Mat& _sample_idx, bool update ) -{ - CvMat tdata = _train_data, responses = _responses, vidx = _var_idx, sidx = _sample_idx; - return train(&tdata, &responses, vidx.data.ptr ? &vidx : 0, - sidx.data.ptr ? &sidx : 0, update); } - -float CvNormalBayesClassifier::predict( const Mat& _samples, Mat* _results, Mat* _results_prob ) const -{ - CvMat samples = _samples, results, *presults = 0, results_prob, *presults_prob = 0; - - if( _results ) - { - if( !(_results->data && _results->type() == CV_32F && - (_results->cols == 1 || _results->rows == 1) && - _results->cols + _results->rows - 1 == _samples.rows) ) - _results->create(_samples.rows, 1, CV_32F); - presults = &(results = *_results); - } - - if( _results_prob ) - { - if( !(_results_prob->data && _results_prob->type() == CV_64F && - (_results_prob->cols == 1 || _results_prob->rows == 1) && - _results_prob->cols + _results_prob->rows - 1 == _samples.rows) ) - _results_prob->create(_samples.rows, 1, CV_64F); - presults_prob = &(results_prob = *_results_prob); - } - - return predict(&samples, presults, presults_prob); } /* End of file. */ diff --git a/modules/ml/src/precomp.hpp b/modules/ml/src/precomp.hpp index 551ff81791be970d5c342e499d147fca5ddea514..d308ae98ec4c2e389ef6d48352dd920bee200aa1 100644 --- a/modules/ml/src/precomp.hpp +++ b/modules/ml/src/precomp.hpp @@ -38,8 +38,8 @@ // //M*/ -#ifndef __OPENCV_PRECOMP_H__ -#define __OPENCV_PRECOMP_H__ +#ifndef __OPENCV_ML_PRECOMP_HPP__ +#define __OPENCV_ML_PRECOMP_HPP__ #include "opencv2/core.hpp" #include "opencv2/ml.hpp" @@ -56,321 +56,218 @@ #include #include #include +#include -#define ML_IMPL CV_IMPL -#define __BEGIN__ __CV_BEGIN__ -#define __END__ __CV_END__ -#define EXIT __CV_EXIT__ - -#define CV_MAT_ELEM_FLAG( mat, type, comp, vect, tflag ) \ - (( tflag == CV_ROW_SAMPLE ) \ - ? (CV_MAT_ELEM( mat, type, comp, vect )) \ - : (CV_MAT_ELEM( mat, type, vect, comp ))) - -/* Convert matrix to vector */ -#define ICV_MAT2VEC( mat, vdata, vstep, num ) \ - if( MIN( (mat).rows, (mat).cols ) != 1 ) \ - CV_ERROR( CV_StsBadArg, "" ); \ - (vdata) = ((mat).data.ptr); \ - if( (mat).rows == 1 ) \ - { \ - (vstep) = CV_ELEM_SIZE( (mat).type ); \ - (num) = (mat).cols; \ - } \ - else \ - { \ - (vstep) = (mat).step; \ - (num) = (mat).rows; \ - } +/****************************************************************************************\ + * Main struct definitions * + \****************************************************************************************/ -/* get raw data */ -#define ICV_RAWDATA( mat, flags, rdata, sstep, cstep, m, n ) \ - (rdata) = (mat).data.ptr; \ - if( CV_IS_ROW_SAMPLE( flags ) ) \ - { \ - (sstep) = (mat).step; \ - (cstep) = CV_ELEM_SIZE( (mat).type ); \ - (m) = (mat).rows; \ - (n) = (mat).cols; \ - } \ - else \ - { \ - (cstep) = (mat).step; \ - (sstep) = CV_ELEM_SIZE( (mat).type ); \ - (n) = (mat).rows; \ - (m) = (mat).cols; \ - } +/* log(2*PI) */ +#define CV_LOG2PI (1.8378770664093454835606594728112) -#define ICV_IS_MAT_OF_TYPE( mat, mat_type) \ - (CV_IS_MAT( mat ) && CV_MAT_TYPE( mat->type ) == (mat_type) && \ - (mat)->cols > 0 && (mat)->rows > 0) - -/* - uchar* data; int sstep, cstep; - trainData->data - uchar* classes; int clstep; int ncl;- trainClasses - uchar* tmask; int tmstep; int ntm; - typeMask - uchar* missed;int msstep, mcstep; -missedMeasurements... - int mm, mn; == m,n == size,dim - uchar* sidx;int sistep; - sampleIdx - uchar* cidx;int cistep; - compIdx - int k, l; == n,m == dim,size (length of cidx, sidx) - int m, n; == size,dim -*/ -#define ICV_DECLARE_TRAIN_ARGS() \ - uchar* data; \ - int sstep, cstep; \ - uchar* classes; \ - int clstep; \ - int ncl; \ - uchar* tmask; \ - int tmstep; \ - int ntm; \ - uchar* missed; \ - int msstep, mcstep; \ - int mm, mn; \ - uchar* sidx; \ - int sistep; \ - uchar* cidx; \ - int cistep; \ - int k, l; \ - int m, n; \ - \ - data = classes = tmask = missed = sidx = cidx = NULL; \ - sstep = cstep = clstep = ncl = tmstep = ntm = msstep = mcstep = mm = mn = 0; \ - sistep = cistep = k = l = m = n = 0; - -#define ICV_TRAIN_DATA_REQUIRED( param, flags ) \ - if( !ICV_IS_MAT_OF_TYPE( (param), CV_32FC1 ) ) \ - { \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ - else \ - { \ - ICV_RAWDATA( *(param), (flags), data, sstep, cstep, m, n ); \ - k = n; \ - l = m; \ - } +namespace cv +{ +namespace ml +{ + using std::vector; -#define ICV_TRAIN_CLASSES_REQUIRED( param ) \ - if( !ICV_IS_MAT_OF_TYPE( (param), CV_32FC1 ) ) \ - { \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ - else \ - { \ - ICV_MAT2VEC( *(param), classes, clstep, ncl ); \ - if( m != ncl ) \ - { \ - CV_ERROR( CV_StsBadArg, "Unmatched sizes" ); \ - } \ - } + #define CV_DTREE_CAT_DIR(idx,subset) \ + (2*((subset[(idx)>>5]&(1 << ((idx) & 31)))==0)-1) -#define ICV_ARG_NULL( param ) \ - if( (param) != NULL ) \ - { \ - CV_ERROR( CV_StsBadArg, #param " parameter must be NULL" ); \ - } + template struct cmp_lt_idx + { + cmp_lt_idx(const _Tp* _arr) : arr(_arr) {} + bool operator ()(int a, int b) const { return arr[a] < arr[b]; } + const _Tp* arr; + }; -#define ICV_MISSED_MEASUREMENTS_OPTIONAL( param, flags ) \ - if( param ) \ - { \ - if( !ICV_IS_MAT_OF_TYPE( param, CV_8UC1 ) ) \ - { \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ - else \ - { \ - ICV_RAWDATA( *(param), (flags), missed, msstep, mcstep, mm, mn ); \ - if( mm != m || mn != n ) \ - { \ - CV_ERROR( CV_StsBadArg, "Unmatched sizes" ); \ - } \ - } \ - } + template struct cmp_lt_ptr + { + cmp_lt_ptr() {} + bool operator ()(const _Tp* a, const _Tp* b) const { return *a < *b; } + }; -#define ICV_COMP_IDX_OPTIONAL( param ) \ - if( param ) \ - { \ - if( !ICV_IS_MAT_OF_TYPE( param, CV_32SC1 ) ) \ - { \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ - else \ - { \ - ICV_MAT2VEC( *(param), cidx, cistep, k ); \ - if( k > n ) \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ + static inline void setRangeVector(std::vector& vec, int n) + { + vec.resize(n); + for( int i = 0; i < n; i++ ) + vec[i] = i; } -#define ICV_SAMPLE_IDX_OPTIONAL( param ) \ - if( param ) \ - { \ - if( !ICV_IS_MAT_OF_TYPE( param, CV_32SC1 ) ) \ - { \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ - else \ - { \ - ICV_MAT2VEC( *sampleIdx, sidx, sistep, l ); \ - if( l > m ) \ - CV_ERROR( CV_StsBadArg, "Invalid " #param " parameter" ); \ - } \ + static inline void writeTermCrit(FileStorage& fs, const TermCriteria& termCrit) + { + if( (termCrit.type & TermCriteria::EPS) != 0 ) + fs << "epsilon" << termCrit.epsilon; + if( (termCrit.type & TermCriteria::COUNT) != 0 ) + fs << "iterations" << termCrit.maxCount; } -/****************************************************************************************/ -#define ICV_CONVERT_FLOAT_ARRAY_TO_MATRICE( array, matrice ) \ -{ \ - CvMat a, b; \ - int dims = (matrice)->cols; \ - int nsamples = (matrice)->rows; \ - int type = CV_MAT_TYPE((matrice)->type); \ - int i, offset = dims; \ - \ - CV_ASSERT( type == CV_32FC1 || type == CV_64FC1 ); \ - offset *= ((type == CV_32FC1) ? sizeof(float) : sizeof(double));\ - \ - b = cvMat( 1, dims, CV_32FC1 ); \ - cvGetRow( matrice, &a, 0 ); \ - for( i = 0; i < nsamples; i++, a.data.ptr += offset ) \ - { \ - b.data.fl = (float*)array[i]; \ - CV_CALL( cvConvert( &b, &a ) ); \ - } \ -} - -/****************************************************************************************\ -* Auxiliary functions declarations * -\****************************************************************************************/ - -/* Generates a set of classes centers in quantity that are generated as - uniform random vectors in parallelepiped, where is concentrated. Vectors in - should have horizontal orientation. If != NULL, the function doesn't - allocate any memory and stores generated centers in , returns . - If == NULL, the function allocates memory and creates the matrice. Centers - are supposed to be oriented horizontally. */ -CvMat* icvGenerateRandomClusterCenters( int seed, - const CvMat* data, - int num_of_clusters, - CvMat* centers CV_DEFAULT(0)); - -/* Fills the using by choosing the maximal probability. Outliers are - fixed by and have cluster label (-1). Function also controls that there - weren't "empty" clusters by filling empty clusters with the maximal probability vector. - If probs_sums != NULL, filles it with the sums of probabilities for each sample (it is - useful for normalizing probabilities' matrice of FCM) */ -void icvFindClusterLabels( const CvMat* probs, float outlier_thresh, float r, - const CvMat* labels ); - -typedef struct CvSparseVecElem32f -{ - int idx; - float val; -} -CvSparseVecElem32f; - -/* Prepare training data and related parameters */ -#define CV_TRAIN_STATMODEL_DEFRAGMENT_TRAIN_DATA 1 -#define CV_TRAIN_STATMODEL_SAMPLES_AS_ROWS 2 -#define CV_TRAIN_STATMODEL_SAMPLES_AS_COLUMNS 4 -#define CV_TRAIN_STATMODEL_CATEGORICAL_RESPONSE 8 -#define CV_TRAIN_STATMODEL_ORDERED_RESPONSE 16 -#define CV_TRAIN_STATMODEL_RESPONSES_ON_OUTPUT 32 -#define CV_TRAIN_STATMODEL_ALWAYS_COPY_TRAIN_DATA 64 -#define CV_TRAIN_STATMODEL_SPARSE_AS_SPARSE 128 - -int -cvPrepareTrainData( const char* /*funcname*/, - const CvMat* train_data, int tflag, - const CvMat* responses, int response_type, - const CvMat* var_idx, - const CvMat* sample_idx, - bool always_copy_data, - const float*** out_train_samples, - int* _sample_count, - int* _var_count, - int* _var_all, - CvMat** out_responses, - CvMat** out_response_map, - CvMat** out_var_idx, - CvMat** out_sample_idx=0 ); - -void -cvSortSamplesByClasses( const float** samples, const CvMat* classes, - int* class_ranges, const uchar** mask CV_DEFAULT(0) ); - -void -cvCombineResponseMaps (CvMat* _responses, - const CvMat* old_response_map, - CvMat* new_response_map, - CvMat** out_response_map); - -void -cvPreparePredictData( const CvArr* sample, int dims_all, const CvMat* comp_idx, - int class_count, const CvMat* prob, float** row_sample, - int as_sparse CV_DEFAULT(0) ); - -/* copies clustering [or batch "predict"] results - (labels and/or centers and/or probs) back to the output arrays */ -void -cvWritebackLabels( const CvMat* labels, CvMat* dst_labels, - const CvMat* centers, CvMat* dst_centers, - const CvMat* probs, CvMat* dst_probs, - const CvMat* sample_idx, int samples_all, - const CvMat* comp_idx, int dims_all ); -#define cvWritebackResponses cvWritebackLabels - -#define XML_FIELD_NAME "_name" -CvFileNode* icvFileNodeGetChild(CvFileNode* father, const char* name); -CvFileNode* icvFileNodeGetChildArrayElem(CvFileNode* father, const char* name,int index); -CvFileNode* icvFileNodeGetNext(CvFileNode* n, const char* name); - - -void cvCheckTrainData( const CvMat* train_data, int tflag, - const CvMat* missing_mask, - int* var_all, int* sample_all ); - -CvMat* cvPreprocessIndexArray( const CvMat* idx_arr, int data_arr_size, bool check_for_duplicates=false ); - -CvMat* cvPreprocessVarType( const CvMat* type_mask, const CvMat* var_idx, - int var_all, int* response_type ); - -CvMat* cvPreprocessOrderedResponses( const CvMat* responses, - const CvMat* sample_idx, int sample_all ); - -CvMat* cvPreprocessCategoricalResponses( const CvMat* responses, - const CvMat* sample_idx, int sample_all, - CvMat** out_response_map, CvMat** class_counts=0 ); - -const float** cvGetTrainSamples( const CvMat* train_data, int tflag, - const CvMat* var_idx, const CvMat* sample_idx, - int* _var_count, int* _sample_count, - bool always_copy_data=false ); - -namespace cv -{ - struct DTreeBestSplitFinder + static inline TermCriteria readTermCrit(const FileNode& fn) { - DTreeBestSplitFinder(){ splitSize = 0, tree = 0; node = 0; } - DTreeBestSplitFinder( CvDTree* _tree, CvDTreeNode* _node); - DTreeBestSplitFinder( const DTreeBestSplitFinder& finder, Split ); - virtual ~DTreeBestSplitFinder() {} - virtual void operator()(const BlockedRange& range); - void join( DTreeBestSplitFinder& rhs ); - Ptr bestSplit; - Ptr split; - int splitSize; - CvDTree* tree; - CvDTreeNode* node; - }; + TermCriteria termCrit; + double epsilon = (double)fn["epsilon"]; + if( epsilon > 0 ) + { + termCrit.type |= TermCriteria::EPS; + termCrit.epsilon = epsilon; + } + int iters = (int)fn["iterations"]; + if( iters > 0 ) + { + termCrit.type |= TermCriteria::COUNT; + termCrit.maxCount = iters; + } + return termCrit; + } - struct ForestTreeBestSplitFinder : DTreeBestSplitFinder + class DTreesImpl : public DTrees { - ForestTreeBestSplitFinder() : DTreeBestSplitFinder() {} - ForestTreeBestSplitFinder( CvForestTree* _tree, CvDTreeNode* _node ); - ForestTreeBestSplitFinder( const ForestTreeBestSplitFinder& finder, Split ); - virtual void operator()(const BlockedRange& range); + public: + struct WNode + { + WNode() + { + class_idx = sample_count = depth = complexity = 0; + parent = left = right = split = defaultDir = -1; + Tn = INT_MAX; + value = maxlr = alpha = node_risk = tree_risk = tree_error = 0.; + } + + int class_idx; + double Tn; + double value; + + int parent; + int left; + int right; + int defaultDir; + + int split; + + int sample_count; + int depth; + double maxlr; + + // global pruning data + int complexity; + double alpha; + double node_risk, tree_risk, tree_error; + }; + + struct WSplit + { + WSplit() + { + varIdx = next = 0; + inversed = false; + quality = c = 0.f; + subsetOfs = -1; + } + + int varIdx; + bool inversed; + float quality; + int next; + float c; + int subsetOfs; + }; + + struct WorkData + { + WorkData(const Ptr& _data); + + Ptr data; + vector wnodes; + vector wsplits; + vector wsubsets; + vector cv_Tn; + vector cv_node_risk; + vector cv_node_error; + vector cv_labels; + vector sample_weights; + vector cat_responses; + vector ord_responses; + vector sidx; + int maxSubsetSize; + }; + + DTreesImpl(); + virtual ~DTreesImpl(); + virtual void clear(); + + String getDefaultModelName() const { return "opencv_ml_dtree"; } + bool isTrained() const { return !roots.empty(); } + bool isClassifier() const { return _isClassifier; } + int getVarCount() const { return varType.empty() ? 0 : (int)(varType.size() - 1); } + int getCatCount(int vi) const { return catOfs[vi][1] - catOfs[vi][0]; } + int getSubsetSize(int vi) const { return (getCatCount(vi) + 31)/32; } + + virtual void setDParams(const Params& _params); + virtual Params getDParams() const; + virtual void startTraining( const Ptr& trainData, int flags ); + virtual void endTraining(); + virtual void initCompVarIdx(); + virtual bool train( const Ptr& trainData, int flags ); + + virtual int addTree( const vector& sidx ); + virtual int addNodeAndTrySplit( int parent, const vector& sidx ); + virtual const vector& getActiveVars(); + virtual int findBestSplit( const vector& _sidx ); + virtual void calcValue( int nidx, const vector& _sidx ); + + virtual WSplit findSplitOrdClass( int vi, const vector& _sidx, double initQuality ); + + // simple k-means, slightly modified to take into account the "weight" (L1-norm) of each vector. + virtual void clusterCategories( const double* vectors, int n, int m, double* csums, int k, int* labels ); + virtual WSplit findSplitCatClass( int vi, const vector& _sidx, double initQuality, int* subset ); + + virtual WSplit findSplitOrdReg( int vi, const vector& _sidx, double initQuality ); + virtual WSplit findSplitCatReg( int vi, const vector& _sidx, double initQuality, int* subset ); + + virtual int calcDir( int splitidx, const vector& _sidx, vector& _sleft, vector& _sright ); + virtual int pruneCV( int root ); + + virtual double updateTreeRNC( int root, double T, int fold ); + virtual bool cutTree( int root, double T, int fold, double min_alpha ); + virtual float predictTrees( const Range& range, const Mat& sample, int flags ) const; + virtual float predict( InputArray inputs, OutputArray outputs, int flags ) const; + + virtual void writeTrainingParams( FileStorage& fs ) const; + virtual void writeParams( FileStorage& fs ) const; + virtual void writeSplit( FileStorage& fs, int splitidx ) const; + virtual void writeNode( FileStorage& fs, int nidx, int depth ) const; + virtual void writeTree( FileStorage& fs, int root ) const; + virtual void write( FileStorage& fs ) const; + + virtual void readParams( const FileNode& fn ); + virtual int readSplit( const FileNode& fn ); + virtual int readNode( const FileNode& fn ); + virtual int readTree( const FileNode& fn ); + virtual void read( const FileNode& fn ); + + virtual const std::vector& getRoots() const { return roots; } + virtual const std::vector& getNodes() const { return nodes; } + virtual const std::vector& getSplits() const { return splits; } + virtual const std::vector& getSubsets() const { return subsets; } + + Params params0, params; + + vector varIdx; + vector compVarIdx; + vector varType; + vector catOfs; + vector catMap; + vector roots; + vector nodes; + vector splits; + vector subsets; + vector classLabels; + vector missingSubst; + bool _isClassifier; + + Ptr w; }; -} -#endif /* __ML_H__ */ +}} + +#endif /* __OPENCV_ML_PRECOMP_HPP__ */ diff --git a/modules/ml/src/rtrees.cpp b/modules/ml/src/rtrees.cpp index c41b8421427997505759409ebb736c265b0d2edd..7c9cbaf2685060310263a03e32676e62122a07e8 100644 --- a/modules/ml/src/rtrees.cpp +++ b/modules/ml/src/rtrees.cpp @@ -7,9 +7,11 @@ // copy or use the software. // // -// Intel License Agreement +// License Agreement +// For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -22,7 +24,7 @@ // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // -// * The name of Intel Corporation may not be used to endorse or promote products +// * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and @@ -40,833 +42,386 @@ #include "precomp.hpp" -CvForestTree::CvForestTree() -{ - forest = NULL; -} - - -CvForestTree::~CvForestTree() -{ - clear(); -} +namespace cv { +namespace ml { - -bool CvForestTree::train( CvDTreeTrainData* _data, - const CvMat* _subsample_idx, - CvRTrees* _forest ) +////////////////////////////////////////////////////////////////////////////////////////// +// Random trees // +////////////////////////////////////////////////////////////////////////////////////////// +RTrees::Params::Params() + : DTrees::Params(5, 10, 0.f, false, 10, 0, false, false, Mat()) { - clear(); - forest = _forest; - - data = _data; - data->shared = true; - return do_train(_subsample_idx); + calcVarImportance = false; + nactiveVars = 0; + termCrit = TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 50, 0.1); } - -bool -CvForestTree::train( const CvMat*, int, const CvMat*, const CvMat*, - const CvMat*, const CvMat*, const CvMat*, CvDTreeParams ) +RTrees::Params::Params( int _maxDepth, int _minSampleCount, + double _regressionAccuracy, bool _useSurrogates, + int _maxCategories, const Mat& _priors, + bool _calcVarImportance, int _nactiveVars, + TermCriteria _termCrit ) + : DTrees::Params(_maxDepth, _minSampleCount, _regressionAccuracy, _useSurrogates, + _maxCategories, 0, false, false, _priors) { - assert(0); - return false; + calcVarImportance = _calcVarImportance; + nactiveVars = _nactiveVars; + termCrit = _termCrit; } -bool -CvForestTree::train( CvDTreeTrainData*, const CvMat* ) +class DTreesImplForRTrees : public DTreesImpl { - assert(0); - return false; -} - - +public: + DTreesImplForRTrees() {} + virtual ~DTreesImplForRTrees() {} -namespace cv -{ - -ForestTreeBestSplitFinder::ForestTreeBestSplitFinder( CvForestTree* _tree, CvDTreeNode* _node ) : - DTreeBestSplitFinder(_tree, _node) {} - -ForestTreeBestSplitFinder::ForestTreeBestSplitFinder( const ForestTreeBestSplitFinder& finder, Split spl ) : - DTreeBestSplitFinder( finder, spl ) {} - -void ForestTreeBestSplitFinder::operator()(const BlockedRange& range) -{ - int vi, vi1 = range.begin(), vi2 = range.end(); - int n = node->sample_count; - CvDTreeTrainData* data = tree->get_data(); - AutoBuffer inn_buf(2*n*(sizeof(int) + sizeof(float))); - - CvForestTree* ftree = (CvForestTree*)tree; - const CvMat* active_var_mask = ftree->forest->get_active_var_mask(); - - for( vi = vi1; vi < vi2; vi++ ) + void setRParams(const RTrees::Params& p) { - CvDTreeSplit *res; - int ci = data->var_type->data.i[vi]; - if( node->num_valid[vi] <= 1 - || (active_var_mask && !active_var_mask->data.ptr[vi]) ) - continue; - - if( data->is_classifier ) - { - if( ci >= 0 ) - res = ftree->find_split_cat_class( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - else - res = ftree->find_split_ord_class( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - } - else - { - if( ci >= 0 ) - res = ftree->find_split_cat_reg( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - else - res = ftree->find_split_ord_reg( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - } - - if( res && bestSplit->quality < split->quality ) - memcpy( bestSplit.get(), split.get(), splitSize ); + rparams = p; } -} -} -CvDTreeSplit* CvForestTree::find_best_split( CvDTreeNode* node ) -{ - CvMat* active_var_mask = 0; - if( forest ) + RTrees::Params getRParams() const { - int var_count; - CvRNG* rng = forest->get_rng(); - - active_var_mask = forest->get_active_var_mask(); - var_count = active_var_mask->cols; - - CV_Assert( var_count == data->var_count ); - - for( int vi = 0; vi < var_count; vi++ ) - { - uchar temp; - int i1 = cvRandInt(rng) % var_count; - int i2 = cvRandInt(rng) % var_count; - CV_SWAP( active_var_mask->data.ptr[i1], - active_var_mask->data.ptr[i2], temp ); - } + return rparams; } - cv::ForestTreeBestSplitFinder finder( this, node ); - - cv::parallel_reduce(cv::BlockedRange(0, data->var_count), finder); - - CvDTreeSplit *bestSplit = 0; - if( finder.bestSplit->quality > 0 ) + void clear() { - bestSplit = data->new_split_cat( 0, -1.0f ); - memcpy( bestSplit, finder.bestSplit, finder.splitSize ); + DTreesImpl::clear(); + oobError = 0.; + rng = RNG((uint64)-1); } - return bestSplit; -} - -void CvForestTree::read( CvFileStorage* fs, CvFileNode* fnode, CvRTrees* _forest, CvDTreeTrainData* _data ) -{ - CvDTree::read( fs, fnode, _data ); - forest = _forest; -} - - -void CvForestTree::read( CvFileStorage*, CvFileNode* ) -{ - assert(0); -} - -void CvForestTree::read( CvFileStorage* _fs, CvFileNode* _node, - CvDTreeTrainData* _data ) -{ - CvDTree::read( _fs, _node, _data ); -} - - -////////////////////////////////////////////////////////////////////////////////////////// -// Random trees // -////////////////////////////////////////////////////////////////////////////////////////// -CvRTParams::CvRTParams() : CvDTreeParams( 5, 10, 0, false, 10, 0, false, false, 0 ), - calc_var_importance(false), nactive_vars(0) -{ - term_crit = cvTermCriteria( CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 50, 0.1 ); -} - -CvRTParams::CvRTParams( int _max_depth, int _min_sample_count, - float _regression_accuracy, bool _use_surrogates, - int _max_categories, const float* _priors, bool _calc_var_importance, - int _nactive_vars, int max_num_of_trees_in_the_forest, - float forest_accuracy, int termcrit_type ) : - CvDTreeParams( _max_depth, _min_sample_count, _regression_accuracy, - _use_surrogates, _max_categories, 0, - false, false, _priors ), - calc_var_importance(_calc_var_importance), - nactive_vars(_nactive_vars) -{ - term_crit = cvTermCriteria(termcrit_type, - max_num_of_trees_in_the_forest, forest_accuracy); -} - -CvRTrees::CvRTrees() -{ - nclasses = 0; - oob_error = 0; - ntrees = 0; - trees = NULL; - data = NULL; - active_var_mask = NULL; - var_importance = NULL; - rng = &cv::theRNG(); - default_model_name = "my_random_trees"; -} - - -void CvRTrees::clear() -{ - int k; - for( k = 0; k < ntrees; k++ ) - delete trees[k]; - cvFree( &trees ); - - delete data; - data = 0; - - cvReleaseMat( &active_var_mask ); - cvReleaseMat( &var_importance ); - ntrees = 0; -} - - -CvRTrees::~CvRTrees() -{ - clear(); -} - -cv::String CvRTrees::getName() const -{ - return CV_TYPE_NAME_ML_RTREES; -} - -CvMat* CvRTrees::get_active_var_mask() -{ - return active_var_mask; -} - - -CvRNG* CvRTrees::get_rng() -{ - return &rng->state; -} - -bool CvRTrees::train( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, - const CvMat* _sample_idx, const CvMat* _var_type, - const CvMat* _missing_mask, CvRTParams params ) -{ - clear(); - - CvDTreeParams tree_params( params.max_depth, params.min_sample_count, - params.regression_accuracy, params.use_surrogates, params.max_categories, - params.cv_folds, params.use_1se_rule, false, params.priors ); - - data = new CvDTreeTrainData(); - data->set_data( _train_data, _tflag, _responses, _var_idx, - _sample_idx, _var_type, _missing_mask, tree_params, true); - - int var_count = data->var_count; - if( params.nactive_vars > var_count ) - params.nactive_vars = var_count; - else if( params.nactive_vars == 0 ) - params.nactive_vars = (int)sqrt((double)var_count); - else if( params.nactive_vars < 0 ) - CV_Error( CV_StsBadArg, " must be non-negative" ); - - // Create mask of active variables at the tree nodes - active_var_mask = cvCreateMat( 1, var_count, CV_8UC1 ); - if( params.calc_var_importance ) + const vector& getActiveVars() { - var_importance = cvCreateMat( 1, var_count, CV_32FC1 ); - cvZero(var_importance); - } - { // initialize active variables mask - CvMat submask1, submask2; - CV_Assert( (active_var_mask->cols >= 1) && (params.nactive_vars > 0) && (params.nactive_vars <= active_var_mask->cols) ); - cvGetCols( active_var_mask, &submask1, 0, params.nactive_vars ); - cvSet( &submask1, cvScalar(1) ); - if( params.nactive_vars < active_var_mask->cols ) + int i, nvars = (int)allVars.size(), m = (int)activeVars.size(); + for( i = 0; i < nvars; i++ ) { - cvGetCols( active_var_mask, &submask2, params.nactive_vars, var_count ); - cvZero( &submask2 ); + int i1 = rng.uniform(0, nvars); + int i2 = rng.uniform(0, nvars); + std::swap(allVars[i1], allVars[i2]); } + for( i = 0; i < m; i++ ) + activeVars[i] = allVars[i]; + return activeVars; } - return grow_forest( params.term_crit ); -} - -bool CvRTrees::train( CvMLData* _data, CvRTParams params ) -{ - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* missing = _data->get_missing(); - const CvMat* var_types = _data->get_var_types(); - const CvMat* train_sidx = _data->get_train_sample_idx(); - const CvMat* var_idx = _data->get_var_idx(); - - return train( values, CV_ROW_SAMPLE, response, var_idx, - train_sidx, var_types, missing, params ); -} - -bool CvRTrees::grow_forest( const CvTermCriteria term_crit ) -{ - CvMat* sample_idx_mask_for_tree = 0; - CvMat* sample_idx_for_tree = 0; - - const int max_ntrees = term_crit.max_iter; - const double max_oob_err = term_crit.epsilon; - - const int dims = data->var_count; - float maximal_response = 0; - - CvMat* oob_sample_votes = 0; - CvMat* oob_responses = 0; - - float* oob_samples_perm_ptr= 0; - - float* samples_ptr = 0; - uchar* missing_ptr = 0; - float* true_resp_ptr = 0; - bool is_oob_or_vimportance = (max_oob_err > 0 && term_crit.type != CV_TERMCRIT_ITER) || var_importance; - - // oob_predictions_sum[i] = sum of predicted values for the i-th sample - // oob_num_of_predictions[i] = number of summands - // (number of predictions for the i-th sample) - // initialize these variable to avoid warning C4701 - CvMat oob_predictions_sum = cvMat( 1, 1, CV_32FC1 ); - CvMat oob_num_of_predictions = cvMat( 1, 1, CV_32FC1 ); - - nsamples = data->sample_count; - nclasses = data->get_num_classes(); - - if ( is_oob_or_vimportance ) + void startTraining( const Ptr& trainData, int flags ) { - if( data->is_classifier ) - { - oob_sample_votes = cvCreateMat( nsamples, nclasses, CV_32SC1 ); - cvZero(oob_sample_votes); - } - else - { - // oob_responses[0,i] = oob_predictions_sum[i] - // = sum of predicted values for the i-th sample - // oob_responses[1,i] = oob_num_of_predictions[i] - // = number of summands (number of predictions for the i-th sample) - oob_responses = cvCreateMat( 2, nsamples, CV_32FC1 ); - cvZero(oob_responses); - cvGetRow( oob_responses, &oob_predictions_sum, 0 ); - cvGetRow( oob_responses, &oob_num_of_predictions, 1 ); - } - - oob_samples_perm_ptr = (float*)cvAlloc( sizeof(float)*nsamples*dims ); - samples_ptr = (float*)cvAlloc( sizeof(float)*nsamples*dims ); - missing_ptr = (uchar*)cvAlloc( sizeof(uchar)*nsamples*dims ); - true_resp_ptr = (float*)cvAlloc( sizeof(float)*nsamples ); - - data->get_vectors( 0, samples_ptr, missing_ptr, true_resp_ptr ); - - double minval, maxval; - CvMat responses = cvMat(1, nsamples, CV_32FC1, true_resp_ptr); - cvMinMaxLoc( &responses, &minval, &maxval ); - maximal_response = (float)MAX( MAX( fabs(minval), fabs(maxval) ), 0 ); + DTreesImpl::startTraining(trainData, flags); + int nvars = w->data->getNVars(); + int i, m = rparams.nactiveVars > 0 ? rparams.nactiveVars : cvRound(std::sqrt((double)nvars)); + m = std::min(std::max(m, 1), nvars); + allVars.resize(nvars); + activeVars.resize(m); + for( i = 0; i < nvars; i++ ) + allVars[i] = varIdx[i]; } - trees = (CvForestTree**)cvAlloc( sizeof(trees[0])*max_ntrees ); - memset( trees, 0, sizeof(trees[0])*max_ntrees ); - - sample_idx_mask_for_tree = cvCreateMat( 1, nsamples, CV_8UC1 ); - sample_idx_for_tree = cvCreateMat( 1, nsamples, CV_32SC1 ); - - ntrees = 0; - while( ntrees < max_ntrees ) + void endTraining() { - int i, oob_samples_count = 0; - double ncorrect_responses = 0; // used for estimation of variable importance - CvForestTree* tree = 0; + DTreesImpl::endTraining(); + vector a, b; + std::swap(allVars, a); + std::swap(activeVars, b); + } - cvZero( sample_idx_mask_for_tree ); - for(i = 0; i < nsamples; i++ ) //form sample for creation one tree + bool train( const Ptr& trainData, int flags ) + { + Params dp(rparams.maxDepth, rparams.minSampleCount, rparams.regressionAccuracy, + rparams.useSurrogates, rparams.maxCategories, rparams.CVFolds, + rparams.use1SERule, rparams.truncatePrunedTree, rparams.priors); + setDParams(dp); + startTraining(trainData, flags); + int treeidx, ntrees = (rparams.termCrit.type & TermCriteria::COUNT) != 0 ? + rparams.termCrit.maxCount : 10000; + int i, j, k, vi, vi_, n = (int)w->sidx.size(); + int nclasses = (int)classLabels.size(); + double eps = (rparams.termCrit.type & TermCriteria::EPS) != 0 && + rparams.termCrit.epsilon > 0 ? rparams.termCrit.epsilon : 0.; + vector sidx(n); + vector oobmask(n); + vector oobidx; + vector oobperm; + vector oobres(n, 0.); + vector oobcount(n, 0); + vector oobvotes(n*nclasses, 0); + int nvars = w->data->getNVars(); + int nallvars = w->data->getNAllVars(); + const int* vidx = !varIdx.empty() ? &varIdx[0] : 0; + vector samplebuf(nallvars); + Mat samples = w->data->getSamples(); + float* psamples = samples.ptr(); + size_t sstep0 = samples.step1(), sstep1 = 1; + Mat sample0, sample(nallvars, 1, CV_32F, &samplebuf[0]); + int predictFlags = _isClassifier ? (PREDICT_MAX_VOTE + RAW_OUTPUT) : PREDICT_SUM; + + bool calcOOBError = eps > 0 || rparams.calcVarImportance; + double max_response = 0.; + + if( w->data->getLayout() == COL_SAMPLE ) + std::swap(sstep0, sstep1); + + if( !_isClassifier ) { - int idx = (*rng)(nsamples); - sample_idx_for_tree->data.i[i] = idx; - sample_idx_mask_for_tree->data.ptr[idx] = 0xFF; + for( i = 0; i < n; i++ ) + { + double val = std::abs(w->ord_responses[w->sidx[i]]); + max_response = std::max(max_response, val); + } } - trees[ntrees] = new CvForestTree(); - tree = trees[ntrees]; - tree->train( data, sample_idx_for_tree, this ); + if( rparams.calcVarImportance ) + varImportance.resize(nallvars, 0.f); - if ( is_oob_or_vimportance ) + for( treeidx = 0; treeidx < ntrees; treeidx++ ) { - CvMat sample, missing; - // form array of OOB samples indices and get these samples - sample = cvMat( 1, dims, CV_32FC1, samples_ptr ); - missing = cvMat( 1, dims, CV_8UC1, missing_ptr ); - - oob_error = 0; - for( i = 0; i < nsamples; i++, - sample.data.fl += dims, missing.data.ptr += dims ) - { - CvDTreeNode* predicted_node = 0; - // check if the sample is OOB - if( sample_idx_mask_for_tree->data.ptr[i] ) - continue; - - // predict oob samples - if( !predicted_node ) - predicted_node = tree->predict(&sample, &missing, true); - - if( !data->is_classifier ) //regression - { - double avg_resp, resp = predicted_node->value; - oob_predictions_sum.data.fl[i] += (float)resp; - oob_num_of_predictions.data.fl[i] += 1; - - // compute oob error - avg_resp = oob_predictions_sum.data.fl[i]/oob_num_of_predictions.data.fl[i]; - avg_resp -= true_resp_ptr[i]; - oob_error += avg_resp*avg_resp; - resp = (resp - true_resp_ptr[i])/maximal_response; - ncorrect_responses += exp( -resp*resp ); - } - else //classification - { - double prdct_resp; - CvPoint max_loc; - CvMat votes; - - cvGetRow(oob_sample_votes, &votes, i); - votes.data.i[predicted_node->class_idx]++; + for( i = 0; i < n; i++ ) + oobmask[i] = (uchar)1; - // compute oob error - cvMinMaxLoc( &votes, 0, 0, 0, &max_loc ); - - prdct_resp = data->cat_map->data.i[max_loc.x]; - oob_error += (fabs(prdct_resp - true_resp_ptr[i]) < FLT_EPSILON) ? 0 : 1; - - ncorrect_responses += cvRound(predicted_node->value - true_resp_ptr[i]) == 0; - } - oob_samples_count++; + for( i = 0; i < n; i++ ) + { + j = rng.uniform(0, n); + sidx[i] = w->sidx[j]; + oobmask[j] = (uchar)0; } - if( oob_samples_count > 0 ) - oob_error /= (double)oob_samples_count; + int root = addTree( sidx ); + if( root < 0 ) + return false; - // estimate variable importance - if( var_importance && oob_samples_count > 0 ) + if( calcOOBError ) { - int m; + oobidx.clear(); + for( i = 0; i < n; i++ ) + { + if( !oobmask[i] ) + oobidx.push_back(i); + } + int n_oob = (int)oobidx.size(); + // if there is no out-of-bag samples, we can not compute OOB error + // nor update the variable importance vector; so we proceed to the next tree + if( n_oob == 0 ) + continue; + double ncorrect_responses = 0.; - memcpy( oob_samples_perm_ptr, samples_ptr, dims*nsamples*sizeof(float)); - for( m = 0; m < dims; m++ ) + oobError = 0.; + for( i = 0; i < n_oob; i++ ) { - double ncorrect_responses_permuted = 0; - // randomly permute values of the m-th variable in the oob samples - float* mth_var_ptr = oob_samples_perm_ptr + m; + j = oobidx[i]; + sample = Mat( nallvars, 1, CV_32F, psamples + sstep0*w->sidx[j], sstep1*sizeof(psamples[0]) ); - for( i = 0; i < nsamples; i++ ) + double val = predictTrees(Range(treeidx, treeidx+1), sample, predictFlags); + if( !_isClassifier ) { - int i1, i2; - float temp; - - if( sample_idx_mask_for_tree->data.ptr[i] ) //the sample is not OOB - continue; - i1 = (*rng)(nsamples); - i2 = (*rng)(nsamples); - CV_SWAP( mth_var_ptr[i1*dims], mth_var_ptr[i2*dims], temp ); - - // turn values of (m-1)-th variable, that were permuted - // at the previous iteration, untouched - if( m > 1 ) - oob_samples_perm_ptr[i*dims+m-1] = samples_ptr[i*dims+m-1]; + oobres[j] += val; + oobcount[j]++; + double true_val = w->ord_responses[w->sidx[j]]; + double a = oobres[j]/oobcount[j] - true_val; + oobError += a*a; + val = (val - true_val)/max_response; + ncorrect_responses += std::exp( -val*val ); } - - // predict "permuted" cases and calculate the number of votes for the - // correct class in the variable-m-permuted oob data - sample = cvMat( 1, dims, CV_32FC1, oob_samples_perm_ptr ); - missing = cvMat( 1, dims, CV_8UC1, missing_ptr ); - for( i = 0; i < nsamples; i++, - sample.data.fl += dims, missing.data.ptr += dims ) + else { - double predct_resp, true_resp; + int ival = cvRound(val); + int* votes = &oobvotes[j*nclasses]; + votes[ival]++; + int best_class = 0; + for( k = 1; k < nclasses; k++ ) + if( votes[best_class] < votes[k] ) + best_class = k; + int diff = best_class != w->cat_responses[w->sidx[j]]; + oobError += diff; + ncorrect_responses += diff == 0; + } + } - if( sample_idx_mask_for_tree->data.ptr[i] ) //the sample is not OOB - continue; + oobError /= n_oob; + if( rparams.calcVarImportance && n_oob > 1 ) + { + oobperm.resize(n_oob); + for( i = 0; i < n_oob; i++ ) + oobperm[i] = oobidx[i]; + + for( vi_ = 0; vi_ < nvars; vi_++ ) + { + vi = vidx ? vidx[vi_] : vi_; + double ncorrect_responses_permuted = 0; + for( i = 0; i < n_oob; i++ ) + { + int i1 = rng.uniform(0, n_oob); + int i2 = rng.uniform(0, n_oob); + std::swap(i1, i2); + } - predct_resp = tree->predict(&sample, &missing, true)->value; - true_resp = true_resp_ptr[i]; - if( data->is_classifier ) - ncorrect_responses_permuted += cvRound(true_resp - predct_resp) == 0; - else + for( i = 0; i < n_oob; i++ ) { - true_resp = (true_resp - predct_resp)/maximal_response; - ncorrect_responses_permuted += exp( -true_resp*true_resp ); + j = oobidx[i]; + int vj = oobperm[i]; + sample0 = Mat( nallvars, 1, CV_32F, psamples + sstep0*w->sidx[j], sstep1*sizeof(psamples[0]) ); + for( k = 0; k < nallvars; k++ ) + sample.at(k) = sample0.at(k); + sample.at(vi) = psamples[sstep0*w->sidx[vj] + sstep1*vi]; + + double val = predictTrees(Range(treeidx, treeidx+1), sample, predictFlags); + if( !_isClassifier ) + { + val = (val - w->ord_responses[w->sidx[j]])/max_response; + ncorrect_responses_permuted += exp( -val*val ); + } + else + ncorrect_responses_permuted += cvRound(val) == w->cat_responses[w->sidx[j]]; } + varImportance[vi] += (float)(ncorrect_responses - ncorrect_responses_permuted); } - var_importance->data.fl[m] += (float)(ncorrect_responses - - ncorrect_responses_permuted); } } + if( calcOOBError && oobError < eps ) + break; } - ntrees++; - if( term_crit.type != CV_TERMCRIT_ITER && oob_error < max_oob_err ) - break; - } - - if( var_importance ) - { - for ( int vi = 0; vi < var_importance->cols; vi++ ) - var_importance->data.fl[vi] = ( var_importance->data.fl[vi] > 0 ) ? - var_importance->data.fl[vi] : 0; - cvNormalize( var_importance, var_importance, 1., 0, CV_L1 ); - } - - cvFree( &oob_samples_perm_ptr ); - cvFree( &samples_ptr ); - cvFree( &missing_ptr ); - cvFree( &true_resp_ptr ); - - cvReleaseMat( &sample_idx_mask_for_tree ); - cvReleaseMat( &sample_idx_for_tree ); - cvReleaseMat( &oob_sample_votes ); - cvReleaseMat( &oob_responses ); - - return true; -} - - -const CvMat* CvRTrees::get_var_importance() -{ - return var_importance; -} - - -float CvRTrees::get_proximity( const CvMat* sample1, const CvMat* sample2, - const CvMat* missing1, const CvMat* missing2 ) const -{ - float result = 0; - - for( int i = 0; i < ntrees; i++ ) - result += trees[i]->predict( sample1, missing1 ) == - trees[i]->predict( sample2, missing2 ) ? 1 : 0; - result = result/(float)ntrees; - - return result; -} - -float CvRTrees::calc_error( CvMLData* _data, int type , std::vector *resp ) -{ - float err = 0; - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* missing = _data->get_missing(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - const CvMat* var_types = _data->get_var_types(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(response->type) ? - 1 : response->step / CV_ELEM_SIZE(response->type); - bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; - float* pred_resp = 0; - if( resp && (sample_count > 0) ) - { - resp->resize( sample_count ); - pred_resp = &((*resp)[0]); - } - if ( is_classifier ) - { - for( int i = 0; i < sample_count; i++ ) + if( rparams.calcVarImportance ) { - CvMat sample, miss; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - if( missing ) - cvGetRow( missing, &miss, si ); - float r = (float)predict( &sample, missing ? &miss : 0 ); - if( pred_resp ) - pred_resp[i] = r; - int d = fabs((double)r - response->data.fl[si*r_step]) <= FLT_EPSILON ? 0 : 1; - err += d; + for( vi_ = 0; vi_ < nallvars; vi_++ ) + varImportance[vi_] = std::max(varImportance[vi_], 0.f); + normalize(varImportance, varImportance, 1., 0, NORM_L1); } - err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; + endTraining(); + return true; } - else + + void writeTrainingParams( FileStorage& fs ) const { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample, miss; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - if( missing ) - cvGetRow( missing, &miss, si ); - float r = (float)predict( &sample, missing ? &miss : 0 ); - if( pred_resp ) - pred_resp[i] = r; - float d = r - response->data.fl[si*r_step]; - err += d*d; - } - err = sample_count ? err / (float)sample_count : -FLT_MAX; + DTreesImpl::writeTrainingParams(fs); + fs << "nactive_vars" << rparams.nactiveVars; } - return err; -} -float CvRTrees::get_train_error() -{ - float err = -1; - - int sample_count = data->sample_count; - int var_count = data->var_count; - - float *values_ptr = (float*)cvAlloc( sizeof(float)*sample_count*var_count ); - uchar *missing_ptr = (uchar*)cvAlloc( sizeof(uchar)*sample_count*var_count ); - float *responses_ptr = (float*)cvAlloc( sizeof(float)*sample_count ); - - data->get_vectors( 0, values_ptr, missing_ptr, responses_ptr); - - if (data->is_classifier) + void write( FileStorage& fs ) const { - int err_count = 0; - float *vp = values_ptr; - uchar *mp = missing_ptr; - for (int si = 0; si < sample_count; si++, vp += var_count, mp += var_count) - { - CvMat sample = cvMat( 1, var_count, CV_32FC1, vp ); - CvMat missing = cvMat( 1, var_count, CV_8UC1, mp ); - float r = predict( &sample, &missing ); - if (fabs(r - responses_ptr[si]) >= FLT_EPSILON) - err_count++; - } - err = (float)err_count / (float)sample_count; - } - else - CV_Error( CV_StsBadArg, "This method is not supported for regression problems" ); + if( roots.empty() ) + CV_Error( CV_StsBadArg, "RTrees have not been trained" ); - cvFree( &values_ptr ); - cvFree( &missing_ptr ); - cvFree( &responses_ptr ); + writeParams(fs); - return err; -} + fs << "oob_error" << oobError; + if( !varImportance.empty() ) + fs << "var_importance" << varImportance; + int k, ntrees = (int)roots.size(); -float CvRTrees::predict( const CvMat* sample, const CvMat* missing ) const -{ - double result = -1; - int k; + fs << "ntrees" << ntrees + << "trees" << "["; - if( nclasses > 0 ) //classification - { - int max_nvotes = 0; - cv::AutoBuffer _votes(nclasses); - int* votes = _votes; - memset( votes, 0, sizeof(*votes)*nclasses ); for( k = 0; k < ntrees; k++ ) { - CvDTreeNode* predicted_node = trees[k]->predict( sample, missing ); - int nvotes; - int class_idx = predicted_node->class_idx; - CV_Assert( 0 <= class_idx && class_idx < nclasses ); - - nvotes = ++votes[class_idx]; - if( nvotes > max_nvotes ) - { - max_nvotes = nvotes; - result = predicted_node->value; - } + fs << "{"; + writeTree(fs, roots[k]); + fs << "}"; } - } - else // regression - { - result = 0; - for( k = 0; k < ntrees; k++ ) - result += trees[k]->predict( sample, missing )->value; - result /= (double)ntrees; - } - return (float)result; -} + fs << "]"; + } -float CvRTrees::predict_prob( const CvMat* sample, const CvMat* missing) const -{ - if( nclasses == 2 ) //classification + void readParams( const FileNode& fn ) { - cv::AutoBuffer _votes(nclasses); - int* votes = _votes; - memset( votes, 0, sizeof(*votes)*nclasses ); - for( int k = 0; k < ntrees; k++ ) - { - CvDTreeNode* predicted_node = trees[k]->predict( sample, missing ); - int class_idx = predicted_node->class_idx; - CV_Assert( 0 <= class_idx && class_idx < nclasses ); - - ++votes[class_idx]; - } - - return float(votes[1])/ntrees; + DTreesImpl::readParams(fn); + rparams.maxDepth = params0.maxDepth; + rparams.minSampleCount = params0.minSampleCount; + rparams.regressionAccuracy = params0.regressionAccuracy; + rparams.useSurrogates = params0.useSurrogates; + rparams.maxCategories = params0.maxCategories; + rparams.priors = params0.priors; + + FileNode tparams_node = fn["training_params"]; + rparams.nactiveVars = (int)tparams_node["nactive_vars"]; } - else // regression - CV_Error(CV_StsBadArg, "This function works for binary classification problems only..."); - - return -1; -} - -void CvRTrees::write( CvFileStorage* fs, const char* name ) const -{ - int k; - - if( ntrees < 1 || !trees || nsamples < 1 ) - CV_Error( CV_StsBadArg, "Invalid CvRTrees object" ); - cv::String modelNodeName = this->getName(); - cvStartWriteStruct( fs, name, CV_NODE_MAP, modelNodeName.c_str() ); - - cvWriteInt( fs, "nclasses", nclasses ); - cvWriteInt( fs, "nsamples", nsamples ); - cvWriteInt( fs, "nactive_vars", (int)cvSum(active_var_mask).val[0] ); - cvWriteReal( fs, "oob_error", oob_error ); + void read( const FileNode& fn ) + { + clear(); - if( var_importance ) - cvWrite( fs, "var_importance", var_importance ); + //int nclasses = (int)fn["nclasses"]; + //int nsamples = (int)fn["nsamples"]; + oobError = (double)fn["oob_error"]; + int ntrees = (int)fn["ntrees"]; - cvWriteInt( fs, "ntrees", ntrees ); + fn["var_importance"] >> varImportance; - data->write_params( fs ); + readParams(fn); - cvStartWriteStruct( fs, "trees", CV_NODE_SEQ ); + FileNode trees_node = fn["trees"]; + FileNodeIterator it = trees_node.begin(); + CV_Assert( ntrees == (int)trees_node.size() ); - for( k = 0; k < ntrees; k++ ) - { - cvStartWriteStruct( fs, 0, CV_NODE_MAP ); - trees[k]->write( fs ); - cvEndWriteStruct( fs ); + for( int treeidx = 0; treeidx < ntrees; treeidx++, ++it ) + { + FileNode nfn = (*it)["nodes"]; + readTree(nfn); + } } - cvEndWriteStruct( fs ); //trees - cvEndWriteStruct( fs ); //CV_TYPE_NAME_ML_RTREES -} + RTrees::Params rparams; + double oobError; + vector varImportance; + vector allVars, activeVars; + RNG rng; +}; -void CvRTrees::read( CvFileStorage* fs, CvFileNode* fnode ) +class RTreesImpl : public RTrees { - int nactive_vars, var_count, k; - CvSeqReader reader; - CvFileNode* trees_fnode = 0; - - clear(); - - nclasses = cvReadIntByName( fs, fnode, "nclasses", -1 ); - nsamples = cvReadIntByName( fs, fnode, "nsamples" ); - nactive_vars = cvReadIntByName( fs, fnode, "nactive_vars", -1 ); - oob_error = cvReadRealByName(fs, fnode, "oob_error", -1 ); - ntrees = cvReadIntByName( fs, fnode, "ntrees", -1 ); - - var_importance = (CvMat*)cvReadByName( fs, fnode, "var_importance" ); - - if( nclasses < 0 || nsamples <= 0 || nactive_vars < 0 || oob_error < 0 || ntrees <= 0) - CV_Error( CV_StsParseError, "Some , , , " - ", , of tags are missing" ); - - rng = &cv::theRNG(); - - trees = (CvForestTree**)cvAlloc( sizeof(trees[0])*ntrees ); - memset( trees, 0, sizeof(trees[0])*ntrees ); +public: + RTreesImpl() {} + virtual ~RTreesImpl() {} - data = new CvDTreeTrainData(); - data->read_params( fs, fnode ); - data->shared = true; + String getDefaultModelName() const { return "opencv_ml_rtrees"; } - trees_fnode = cvGetFileNodeByName( fs, fnode, "trees" ); - if( !trees_fnode || !CV_NODE_IS_SEQ(trees_fnode->tag) ) - CV_Error( CV_StsParseError, " tag is missing" ); - - cvStartReadSeq( trees_fnode->data.seq, &reader ); - if( reader.seq->total != ntrees ) - CV_Error( CV_StsParseError, - " is not equal to the number of trees saved in file" ); - - for( k = 0; k < ntrees; k++ ) + bool train( const Ptr& trainData, int flags ) { - trees[k] = new CvForestTree(); - trees[k]->read( fs, (CvFileNode*)reader.ptr, this, data ); - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + return impl.train(trainData, flags); } - var_count = data->var_count; - active_var_mask = cvCreateMat( 1, var_count, CV_8UC1 ); + float predict( InputArray samples, OutputArray results, int flags ) const { - // initialize active variables mask - CvMat submask1; - cvGetCols( active_var_mask, &submask1, 0, nactive_vars ); - cvSet( &submask1, cvScalar(1) ); - - if( nactive_vars < var_count ) - { - CvMat submask2; - cvGetCols( active_var_mask, &submask2, nactive_vars, var_count ); - cvZero( &submask2 ); - } + return impl.predict(samples, results, flags); } -} - -int CvRTrees::get_tree_count() const -{ - return ntrees; -} + void write( FileStorage& fs ) const + { + impl.write(fs); + } -CvForestTree* CvRTrees::get_tree(int i) const -{ - return (unsigned)i < (unsigned)ntrees ? trees[i] : 0; -} + void read( const FileNode& fn ) + { + impl.read(fn); + } -using namespace cv; + void setRParams(const Params& p) { impl.setRParams(p); } + Params getRParams() const { return impl.getRParams(); } -bool CvRTrees::train( const Mat& _train_data, int _tflag, - const Mat& _responses, const Mat& _var_idx, - const Mat& _sample_idx, const Mat& _var_type, - const Mat& _missing_mask, CvRTParams _params ) -{ - train_data_hdr = _train_data; - train_data_mat = _train_data; - responses_hdr = _responses; - responses_mat = _responses; + Mat getVarImportance() const { return Mat_(impl.varImportance, true); } + int getVarCount() const { return impl.getVarCount(); } - CvMat vidx = _var_idx, sidx = _sample_idx, vtype = _var_type, mmask = _missing_mask; + bool isTrained() const { return impl.isTrained(); } + bool isClassifier() const { return impl.isClassifier(); } - return train(&train_data_hdr, _tflag, &responses_hdr, vidx.data.ptr ? &vidx : 0, - sidx.data.ptr ? &sidx : 0, vtype.data.ptr ? &vtype : 0, - mmask.data.ptr ? &mmask : 0, _params); -} + const vector& getRoots() const { return impl.getRoots(); } + const vector& getNodes() const { return impl.getNodes(); } + const vector& getSplits() const { return impl.getSplits(); } + const vector& getSubsets() const { return impl.getSubsets(); } + DTreesImplForRTrees impl; +}; -float CvRTrees::predict( const Mat& _sample, const Mat& _missing ) const -{ - CvMat sample = _sample, mmask = _missing; - return predict(&sample, mmask.data.ptr ? &mmask : 0); -} -float CvRTrees::predict_prob( const Mat& _sample, const Mat& _missing) const +Ptr RTrees::create(const Params& params) { - CvMat sample = _sample, mmask = _missing; - return predict_prob(&sample, mmask.data.ptr ? &mmask : 0); + Ptr p = makePtr(); + p->setRParams(params); + return p; } -Mat CvRTrees::getVarImportance() -{ - return cvarrToMat(get_var_importance()); -} +}} // End of file. diff --git a/modules/ml/src/svm.cpp b/modules/ml/src/svm.cpp index 341a817c94453f5284a74217ecd6fd39adde5814..985cc625203b68b91ece1a40e729871468073cff 100644 --- a/modules/ml/src/svm.cpp +++ b/modules/ml/src/svm.cpp @@ -7,9 +7,11 @@ // copy or use the software. // // -// Intel License Agreement +// License Agreement +// For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -22,7 +24,7 @@ // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // -// * The name of Intel Corporation may not be used to endorse or promote products +// * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and @@ -40,6 +42,9 @@ #include "precomp.hpp" +#include +#include + /****************************************************************************************\ COPYRIGHT NOTICE ---------------- @@ -81,2929 +86,2091 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \****************************************************************************************/ -using namespace cv; - -#define CV_SVM_MIN_CACHE_SIZE (40 << 20) /* 40Mb */ +namespace cv { namespace ml { -#include -#include - -#if 1 typedef float Qfloat; -#define QFLOAT_TYPE CV_32F -#else -typedef double Qfloat; -#define QFLOAT_TYPE CV_64F -#endif +const int QFLOAT_TYPE = DataDepth::value; // Param Grid -bool CvParamGrid::check() const +static void checkParamGrid(const ParamGrid& pg) { - bool ok = false; - - CV_FUNCNAME( "CvParamGrid::check" ); - __BEGIN__; - - if( min_val > max_val ) - CV_ERROR( CV_StsBadArg, "Lower bound of the grid must be less then the upper one" ); - if( min_val < DBL_EPSILON ) - CV_ERROR( CV_StsBadArg, "Lower bound of the grid must be positive" ); - if( step < 1. + FLT_EPSILON ) - CV_ERROR( CV_StsBadArg, "Grid step must greater then 1" ); + if( pg.minVal > pg.maxVal ) + CV_Error( CV_StsBadArg, "Lower bound of the grid must be less then the upper one" ); + if( pg.minVal < DBL_EPSILON ) + CV_Error( CV_StsBadArg, "Lower bound of the grid must be positive" ); + if( pg.logStep < 1. + FLT_EPSILON ) + CV_Error( CV_StsBadArg, "Grid step must greater then 1" ); +} - ok = true; +// SVM training parameters +SVM::Params::Params() +{ + svmType = SVM::C_SVC; + kernelType = SVM::RBF; + degree = 0; + gamma = 1; + coef0 = 0; + C = 1; + nu = 0; + p = 0; + termCrit = TermCriteria( CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 1000, FLT_EPSILON ); +} - __END__; - return ok; +SVM::Params::Params( int _svmType, int _kernelType, + double _degree, double _gamma, double _coef0, + double _Con, double _nu, double _p, + const Mat& _classWeights, TermCriteria _termCrit ) +{ + svmType = _svmType; + kernelType = _kernelType; + degree = _degree; + gamma = _gamma; + coef0 = _coef0; + C = _Con; + nu = _nu; + p = _p; + classWeights = _classWeights; + termCrit = _termCrit; } -CvParamGrid CvSVM::get_default_grid( int param_id ) +/////////////////////////////////////// SVM kernel /////////////////////////////////////// +class SVMKernelImpl : public SVM::Kernel { - CvParamGrid grid; - if( param_id == CvSVM::C ) +public: + SVMKernelImpl() { - grid.min_val = 0.1; - grid.max_val = 500; - grid.step = 5; // total iterations = 5 } - else if( param_id == CvSVM::GAMMA ) + + SVMKernelImpl( const SVM::Params& _params ) { - grid.min_val = 1e-5; - grid.max_val = 0.6; - grid.step = 15; // total iterations = 4 + params = _params; } - else if( param_id == CvSVM::P ) + + virtual ~SVMKernelImpl() { - grid.min_val = 0.01; - grid.max_val = 100; - grid.step = 7; // total iterations = 4 } - else if( param_id == CvSVM::NU ) + + int getType() const { - grid.min_val = 0.01; - grid.max_val = 0.2; - grid.step = 3; // total iterations = 3 + return params.kernelType; } - else if( param_id == CvSVM::COEF ) + + void calc_non_rbf_base( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results, + double alpha, double beta ) { - grid.min_val = 0.1; - grid.max_val = 300; - grid.step = 14; // total iterations = 3 + int j, k; + for( j = 0; j < vcount; j++ ) + { + const float* sample = &vecs[j*var_count]; + double s = 0; + for( k = 0; k <= var_count - 4; k += 4 ) + s += sample[k]*another[k] + sample[k+1]*another[k+1] + + sample[k+2]*another[k+2] + sample[k+3]*another[k+3]; + for( ; k < var_count; k++ ) + s += sample[k]*another[k]; + results[j] = (Qfloat)(s*alpha + beta); + } } - else if( param_id == CvSVM::DEGREE ) + + void calc_linear( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - grid.min_val = 0.01; - grid.max_val = 4; - grid.step = 7; // total iterations = 3 + calc_non_rbf_base( vcount, var_count, vecs, another, results, 1, 0 ); } - else - cvError( CV_StsBadArg, "CvSVM::get_default_grid", "Invalid type of parameter " - "(use one of CvSVM::C, CvSVM::GAMMA et al.)", __FILE__, __LINE__ ); - return grid; -} - -// SVM training parameters -CvSVMParams::CvSVMParams() : - svm_type(CvSVM::C_SVC), kernel_type(CvSVM::RBF), degree(0), - gamma(1), coef0(0), C(1), nu(0), p(0), class_weights(0) -{ - term_crit = cvTermCriteria( CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 1000, FLT_EPSILON ); -} - - -CvSVMParams::CvSVMParams( int _svm_type, int _kernel_type, - double _degree, double _gamma, double _coef0, - double _Con, double _nu, double _p, - CvMat* _class_weights, CvTermCriteria _term_crit ) : - svm_type(_svm_type), kernel_type(_kernel_type), - degree(_degree), gamma(_gamma), coef0(_coef0), - C(_Con), nu(_nu), p(_p), class_weights(_class_weights), term_crit(_term_crit) -{ -} - - -/////////////////////////////////////// SVM kernel /////////////////////////////////////// - -CvSVMKernel::CvSVMKernel() -{ - clear(); -} - - -void CvSVMKernel::clear() -{ - params = 0; - calc_func = 0; -} - - -CvSVMKernel::~CvSVMKernel() -{ -} - -CvSVMKernel::CvSVMKernel( const CvSVMParams* _params, Calc _calc_func ) -{ - clear(); - create( _params, _calc_func ); -} - - -bool CvSVMKernel::create( const CvSVMParams* _params, Calc _calc_func ) -{ - clear(); - params = _params; - calc_func = _calc_func; - - if( !calc_func ) - calc_func = params->kernel_type == CvSVM::RBF ? &CvSVMKernel::calc_rbf : - params->kernel_type == CvSVM::POLY ? &CvSVMKernel::calc_poly : - params->kernel_type == CvSVM::SIGMOID ? &CvSVMKernel::calc_sigmoid : - params->kernel_type == CvSVM::CHI2 ? &CvSVMKernel::calc_chi2 : - params->kernel_type == CvSVM::INTER ? &CvSVMKernel::calc_intersec : - &CvSVMKernel::calc_linear; - - return true; -} - - -void CvSVMKernel::calc_non_rbf_base( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results, - double alpha, double beta ) -{ - int j, k; - for( j = 0; j < vcount; j++ ) + void calc_poly( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - const float* sample = vecs[j]; - double s = 0; - for( k = 0; k <= var_count - 4; k += 4 ) - s += sample[k]*another[k] + sample[k+1]*another[k+1] + - sample[k+2]*another[k+2] + sample[k+3]*another[k+3]; - for( ; k < var_count; k++ ) - s += sample[k]*another[k]; - results[j] = (Qfloat)(s*alpha + beta); + Mat R( 1, vcount, QFLOAT_TYPE, results ); + calc_non_rbf_base( vcount, var_count, vecs, another, results, params.gamma, params.coef0 ); + if( vcount > 0 ) + pow( R, params.degree, R ); } -} - - -void CvSVMKernel::calc_linear( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - calc_non_rbf_base( vcount, var_count, vecs, another, results, 1, 0 ); -} - - -void CvSVMKernel::calc_poly( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - CvMat R = cvMat( 1, vcount, QFLOAT_TYPE, results ); - calc_non_rbf_base( vcount, var_count, vecs, another, results, params->gamma, params->coef0 ); - if( vcount > 0 ) - cvPow( &R, &R, params->degree ); -} - -void CvSVMKernel::calc_sigmoid( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - int j; - calc_non_rbf_base( vcount, var_count, vecs, another, results, - -2*params->gamma, -2*params->coef0 ); - // TODO: speedup this - for( j = 0; j < vcount; j++ ) + void calc_sigmoid( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - Qfloat t = results[j]; - double e = exp(-fabs(t)); - if( t > 0 ) - results[j] = (Qfloat)((1. - e)/(1. + e)); - else - results[j] = (Qfloat)((e - 1.)/(e + 1.)); + int j; + calc_non_rbf_base( vcount, var_count, vecs, another, results, + -2*params.gamma, -2*params.coef0 ); + // TODO: speedup this + for( j = 0; j < vcount; j++ ) + { + Qfloat t = results[j]; + Qfloat e = std::exp(-std::abs(t)); + if( t > 0 ) + results[j] = (Qfloat)((1. - e)/(1. + e)); + else + results[j] = (Qfloat)((e - 1.)/(e + 1.)); + } } -} - -void CvSVMKernel::calc_rbf( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - CvMat R = cvMat( 1, vcount, QFLOAT_TYPE, results ); - double gamma = -params->gamma; - int j, k; - for( j = 0; j < vcount; j++ ) + void calc_rbf( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - const float* sample = vecs[j]; - double s = 0; + double gamma = -params.gamma; + int j, k; - for( k = 0; k <= var_count - 4; k += 4 ) + for( j = 0; j < vcount; j++ ) { - double t0 = sample[k] - another[k]; - double t1 = sample[k+1] - another[k+1]; + const float* sample = &vecs[j*var_count]; + double s = 0; + + for( k = 0; k <= var_count - 4; k += 4 ) + { + double t0 = sample[k] - another[k]; + double t1 = sample[k+1] - another[k+1]; + + s += t0*t0 + t1*t1; - s += t0*t0 + t1*t1; + t0 = sample[k+2] - another[k+2]; + t1 = sample[k+3] - another[k+3]; - t0 = sample[k+2] - another[k+2]; - t1 = sample[k+3] - another[k+3]; + s += t0*t0 + t1*t1; + } - s += t0*t0 + t1*t1; + for( ; k < var_count; k++ ) + { + double t0 = sample[k] - another[k]; + s += t0*t0; + } + results[j] = (Qfloat)(s*gamma); } - for( ; k < var_count; k++ ) + if( vcount > 0 ) { - double t0 = sample[k] - another[k]; - s += t0*t0; + Mat R( 1, vcount, QFLOAT_TYPE, results ); + exp( R, R ); } - results[j] = (Qfloat)(s*gamma); } - if( vcount > 0 ) - cvExp( &R, &R ); -} - -/// Histogram intersection kernel -void CvSVMKernel::calc_intersec( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - int j, k; - for( j = 0; j < vcount; j++ ) + /// Histogram intersection kernel + void calc_intersec( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - const float* sample = vecs[j]; - double s = 0; - for( k = 0; k <= var_count - 4; k += 4 ) - s += std::min(sample[k],another[k]) + std::min(sample[k+1],another[k+1]) + - std::min(sample[k+2],another[k+2]) + std::min(sample[k+3],another[k+3]); - for( ; k < var_count; k++ ) - s += std::min(sample[k],another[k]); - results[j] = (Qfloat)(s); + int j, k; + for( j = 0; j < vcount; j++ ) + { + const float* sample = &vecs[j*var_count]; + double s = 0; + for( k = 0; k <= var_count - 4; k += 4 ) + s += std::min(sample[k],another[k]) + std::min(sample[k+1],another[k+1]) + + std::min(sample[k+2],another[k+2]) + std::min(sample[k+3],another[k+3]); + for( ; k < var_count; k++ ) + s += std::min(sample[k],another[k]); + results[j] = (Qfloat)(s); + } } -} -/// Exponential chi2 kernel -void CvSVMKernel::calc_chi2( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - CvMat R = cvMat( 1, vcount, QFLOAT_TYPE, results ); - double gamma = -params->gamma; - int j, k; - for( j = 0; j < vcount; j++ ) - { - const float* sample = vecs[j]; - double chi2 = 0; - for(k = 0 ; k < var_count; k++ ) + /// Exponential chi2 kernel + void calc_chi2( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - double d = sample[k]-another[k]; - double devisor = sample[k]+another[k]; - /// if devisor == 0, the Chi2 distance would be zero, but calculation would rise an error because of deviding by zero - if (devisor != 0) + Mat R( 1, vcount, QFLOAT_TYPE, results ); + double gamma = -params.gamma; + int j, k; + for( j = 0; j < vcount; j++ ) { - chi2 += d*d/devisor; + const float* sample = &vecs[j*var_count]; + double chi2 = 0; + for(k = 0 ; k < var_count; k++ ) + { + double d = sample[k]-another[k]; + double devisor = sample[k]+another[k]; + /// if devisor == 0, the Chi2 distance would be zero, + // but calculation would rise an error because of deviding by zero + if (devisor != 0) + { + chi2 += d*d/devisor; + } + } + results[j] = (Qfloat) (gamma*chi2); } + if( vcount > 0 ) + exp( R, R ); } - results[j] = (Qfloat) (gamma*chi2); - } - if( vcount > 0 ) - cvExp( &R, &R ); - -} - -void CvSVMKernel::calc( int vcount, int var_count, const float** vecs, - const float* another, Qfloat* results ) -{ - const Qfloat max_val = (Qfloat)(FLT_MAX*1e-3); - int j; - (this->*calc_func)( vcount, var_count, vecs, another, results ); - for( j = 0; j < vcount; j++ ) + void calc( int vcount, int var_count, const float* vecs, + const float* another, Qfloat* results ) { - if( results[j] > max_val ) - results[j] = max_val; + switch( params.kernelType ) + { + case SVM::LINEAR: + calc_linear(vcount, var_count, vecs, another, results); + break; + case SVM::RBF: + calc_rbf(vcount, var_count, vecs, another, results); + break; + case SVM::POLY: + calc_poly(vcount, var_count, vecs, another, results); + break; + case SVM::SIGMOID: + calc_sigmoid(vcount, var_count, vecs, another, results); + break; + case SVM::CHI2: + calc_chi2(vcount, var_count, vecs, another, results); + break; + case SVM::INTER: + calc_intersec(vcount, var_count, vecs, another, results); + break; + default: + CV_Error(CV_StsBadArg, "Unknown kernel type"); + } + const Qfloat max_val = (Qfloat)(FLT_MAX*1e-3); + for( int j = 0; j < vcount; j++ ) + { + if( results[j] > max_val ) + results[j] = max_val; + } } -} - -// Generalized SMO+SVMlight algorithm -// Solves: -// -// min [0.5(\alpha^T Q \alpha) + b^T \alpha] -// -// y^T \alpha = \delta -// y_i = +1 or -1 -// 0 <= alpha_i <= Cp for y_i = 1 -// 0 <= alpha_i <= Cn for y_i = -1 -// -// Given: -// -// Q, b, y, Cp, Cn, and an initial feasible point \alpha -// l is the size of vectors and matrices -// eps is the stopping criterion -// -// solution will be put in \alpha, objective value will be put in obj -// - -void CvSVMSolver::clear() -{ - G = 0; - alpha = 0; - y = 0; - b = 0; - buf[0] = buf[1] = 0; - cvReleaseMemStorage( &storage ); - kernel = 0; - select_working_set_func = 0; - calc_rho_func = 0; - - rows = 0; - samples = 0; - get_row_func = 0; -} + SVM::Params params; +}; -CvSVMSolver::CvSVMSolver() -{ - storage = 0; - clear(); -} +///////////////////////////////////////////////////////////////////////// -CvSVMSolver::~CvSVMSolver() +static void sortSamplesByClasses( const Mat& _samples, const Mat& _responses, + vector& sidx_all, vector& class_ranges ) { - clear(); -} - + int i, nsamples = _samples.rows; + CV_Assert( _responses.isContinuous() && _responses.checkVector(1, CV_32S) == nsamples ); -CvSVMSolver::CvSVMSolver( int _sample_count, int _var_count, const float** _samples, schar* _y, - int _alpha_count, double* _alpha, double _Cp, double _Cn, - CvMemStorage* _storage, CvSVMKernel* _kernel, GetRow _get_row, - SelectWorkingSet _select_working_set, CalcRho _calc_rho ) -{ - storage = 0; - create( _sample_count, _var_count, _samples, _y, _alpha_count, _alpha, _Cp, _Cn, - _storage, _kernel, _get_row, _select_working_set, _calc_rho ); -} + setRangeVector(sidx_all, nsamples); + const int* rptr = _responses.ptr(); + std::sort(sidx_all.begin(), sidx_all.end(), cmp_lt_idx(rptr)); + class_ranges.clear(); + class_ranges.push_back(0); -bool CvSVMSolver::create( int _sample_count, int _var_count, const float** _samples, schar* _y, - int _alpha_count, double* _alpha, double _Cp, double _Cn, - CvMemStorage* _storage, CvSVMKernel* _kernel, GetRow _get_row, - SelectWorkingSet _select_working_set, CalcRho _calc_rho ) -{ - bool ok = false; - int i, svm_type; - - CV_FUNCNAME( "CvSVMSolver::create" ); - - __BEGIN__; - - int rows_hdr_size; - - clear(); - - sample_count = _sample_count; - var_count = _var_count; - samples = _samples; - y = _y; - alpha_count = _alpha_count; - alpha = _alpha; - kernel = _kernel; - - C[0] = _Cn; - C[1] = _Cp; - eps = kernel->params->term_crit.epsilon; - max_iter = kernel->params->term_crit.max_iter; - storage = cvCreateChildMemStorage( _storage ); - - b = (double*)cvMemStorageAlloc( storage, alpha_count*sizeof(b[0])); - alpha_status = (schar*)cvMemStorageAlloc( storage, alpha_count*sizeof(alpha_status[0])); - G = (double*)cvMemStorageAlloc( storage, alpha_count*sizeof(G[0])); - for( i = 0; i < 2; i++ ) - buf[i] = (Qfloat*)cvMemStorageAlloc( storage, sample_count*2*sizeof(buf[i][0]) ); - svm_type = kernel->params->svm_type; - - select_working_set_func = _select_working_set; - if( !select_working_set_func ) - select_working_set_func = svm_type == CvSVM::NU_SVC || svm_type == CvSVM::NU_SVR ? - &CvSVMSolver::select_working_set_nu_svm : &CvSVMSolver::select_working_set; - - calc_rho_func = _calc_rho; - if( !calc_rho_func ) - calc_rho_func = svm_type == CvSVM::NU_SVC || svm_type == CvSVM::NU_SVR ? - &CvSVMSolver::calc_rho_nu_svm : &CvSVMSolver::calc_rho; - - get_row_func = _get_row; - if( !get_row_func ) - get_row_func = params->svm_type == CvSVM::EPS_SVR || - params->svm_type == CvSVM::NU_SVR ? &CvSVMSolver::get_row_svr : - params->svm_type == CvSVM::C_SVC || - params->svm_type == CvSVM::NU_SVC ? &CvSVMSolver::get_row_svc : - &CvSVMSolver::get_row_one_class; - - cache_line_size = sample_count*sizeof(Qfloat); - // cache size = max(num_of_samples^2*sizeof(Qfloat)*0.25, 64Kb) - // (assuming that for large training sets ~25% of Q matrix is used) - cache_size = MAX( cache_line_size*sample_count/4, CV_SVM_MIN_CACHE_SIZE ); - - // the size of Q matrix row headers - rows_hdr_size = sample_count*sizeof(rows[0]); - if( rows_hdr_size > storage->block_size ) - CV_ERROR( CV_StsOutOfRange, "Too small storage block size" ); - - lru_list.prev = lru_list.next = &lru_list; - rows = (CvSVMKernelRow*)cvMemStorageAlloc( storage, rows_hdr_size ); - memset( rows, 0, rows_hdr_size ); - - ok = true; - - __END__; - - return ok; + for( i = 0; i < nsamples; i++ ) + { + if( i == nsamples-1 || rptr[sidx_all[i]] != rptr[sidx_all[i+1]] ) + class_ranges.push_back(i+1); + } } +//////////////////////// SVM implementation ////////////////////////////// -float* CvSVMSolver::get_row_base( int i, bool* _existed ) +ParamGrid SVM::getDefaultGrid( int param_id ) { - int i1 = i < sample_count ? i : i - sample_count; - CvSVMKernelRow* row = rows + i1; - bool existed = row->data != 0; - Qfloat* data; - - if( existed || cache_size <= 0 ) + ParamGrid grid; + if( param_id == SVM::C ) { - CvSVMKernelRow* del_row = existed ? row : lru_list.prev; - data = del_row->data; - assert( data != 0 ); - - // delete row from the LRU list - del_row->data = 0; - del_row->prev->next = del_row->next; - del_row->next->prev = del_row->prev; + grid.minVal = 0.1; + grid.maxVal = 500; + grid.logStep = 5; // total iterations = 5 } - else + else if( param_id == SVM::GAMMA ) { - data = (Qfloat*)cvMemStorageAlloc( storage, cache_line_size ); - cache_size -= cache_line_size; + grid.minVal = 1e-5; + grid.maxVal = 0.6; + grid.logStep = 15; // total iterations = 4 } - - // insert row into the LRU list - row->data = data; - row->prev = &lru_list; - row->next = lru_list.next; - row->prev->next = row->next->prev = row; - - if( !existed ) + else if( param_id == SVM::P ) { - kernel->calc( sample_count, var_count, samples, samples[i1], row->data ); + grid.minVal = 0.01; + grid.maxVal = 100; + grid.logStep = 7; // total iterations = 4 } - - if( _existed ) - *_existed = existed; - - return row->data; -} - - -float* CvSVMSolver::get_row_svc( int i, float* row, float*, bool existed ) -{ - if( !existed ) + else if( param_id == SVM::NU ) { - const schar* _y = y; - int j, len = sample_count; - assert( _y && i < sample_count ); - - if( _y[i] > 0 ) - { - for( j = 0; j < len; j++ ) - row[j] = _y[j]*row[j]; - } - else - { - for( j = 0; j < len; j++ ) - row[j] = -_y[j]*row[j]; - } + grid.minVal = 0.01; + grid.maxVal = 0.2; + grid.logStep = 3; // total iterations = 3 } - return row; -} - - -float* CvSVMSolver::get_row_one_class( int, float* row, float*, bool ) -{ - return row; -} - - -float* CvSVMSolver::get_row_svr( int i, float* row, float* dst, bool ) -{ - int j, len = sample_count; - Qfloat* dst_pos = dst; - Qfloat* dst_neg = dst + len; - if( i >= len ) + else if( param_id == SVM::COEF ) { - Qfloat* temp; - CV_SWAP( dst_pos, dst_neg, temp ); + grid.minVal = 0.1; + grid.maxVal = 300; + grid.logStep = 14; // total iterations = 3 } - - for( j = 0; j < len; j++ ) + else if( param_id == SVM::DEGREE ) { - Qfloat t = row[j]; - dst_pos[j] = t; - dst_neg[j] = -t; + grid.minVal = 0.01; + grid.maxVal = 4; + grid.logStep = 7; // total iterations = 3 } - return dst; -} - - - -float* CvSVMSolver::get_row( int i, float* dst ) -{ - bool existed = false; - float* row = get_row_base( i, &existed ); - return (this->*get_row_func)( i, row, dst, existed ); + else + cvError( CV_StsBadArg, "SVM::getDefaultGrid", "Invalid type of parameter " + "(use one of SVM::C, SVM::GAMMA et al.)", __FILE__, __LINE__ ); + return grid; } -#undef is_upper_bound -#define is_upper_bound(i) (alpha_status[i] > 0) - -#undef is_lower_bound -#define is_lower_bound(i) (alpha_status[i] < 0) - -#undef is_free -#define is_free(i) (alpha_status[i] == 0) - -#undef get_C -#define get_C(i) (C[y[i]>0]) - -#undef update_alpha_status -#define update_alpha_status(i) \ - alpha_status[i] = (schar)(alpha[i] >= get_C(i) ? 1 : alpha[i] <= 0 ? -1 : 0) - -#undef reconstruct_gradient -#define reconstruct_gradient() /* empty for now */ - - -bool CvSVMSolver::solve_generic( CvSVMSolutionInfo& si ) -{ - int iter = 0; - int i, j, k; - - // 1. initialize gradient and alpha status - for( i = 0; i < alpha_count; i++ ) - { - update_alpha_status(i); - G[i] = b[i]; - if( fabs(G[i]) > 1e200 ) - return false; - } - - for( i = 0; i < alpha_count; i++ ) - { - if( !is_lower_bound(i) ) +class SVMImpl : public SVM +{ +public: + struct DecisionFunc + { + DecisionFunc(double _rho, int _ofs) : rho(_rho), ofs(_ofs) {} + DecisionFunc() : rho(0.), ofs(0) {} + double rho; + int ofs; + }; + + // Generalized SMO+SVMlight algorithm + // Solves: + // + // min [0.5(\alpha^T Q \alpha) + b^T \alpha] + // + // y^T \alpha = \delta + // y_i = +1 or -1 + // 0 <= alpha_i <= Cp for y_i = 1 + // 0 <= alpha_i <= Cn for y_i = -1 + // + // Given: + // + // Q, b, y, Cp, Cn, and an initial feasible point \alpha + // l is the size of vectors and matrices + // eps is the stopping criterion + // + // solution will be put in \alpha, objective value will be put in obj + // + class Solver + { + public: + enum { MIN_CACHE_SIZE = (40 << 20) /* 40Mb */, MAX_CACHE_SIZE = (500 << 20) /* 500Mb */ }; + + typedef bool (Solver::*SelectWorkingSet)( int& i, int& j ); + typedef Qfloat* (Solver::*GetRow)( int i, Qfloat* row, Qfloat* dst, bool existed ); + typedef void (Solver::*CalcRho)( double& rho, double& r ); + + struct KernelRow { - const Qfloat *Q_i = get_row( i, buf[0] ); - double alpha_i = alpha[i]; - - for( j = 0; j < alpha_count; j++ ) - G[j] += alpha_i*Q_i[j]; + KernelRow() { idx = -1; prev = next = 0; } + KernelRow(int _idx, int _prev, int _next) : idx(_idx), prev(_prev), next(_next) {} + int idx; + int prev; + int next; + }; + + struct SolutionInfo + { + SolutionInfo() { obj = rho = upper_bound_p = upper_bound_n = r = 0; } + double obj; + double rho; + double upper_bound_p; + double upper_bound_n; + double r; // for Solver_NU + }; + + void clear() + { + alpha_vec = 0; + select_working_set_func = 0; + calc_rho_func = 0; + get_row_func = 0; + lru_cache.clear(); } - } - - // 2. optimization loop - for(;;) - { - const Qfloat *Q_i, *Q_j; - double C_i, C_j; - double old_alpha_i, old_alpha_j, alpha_i, alpha_j; - double delta_alpha_i, delta_alpha_j; -#ifdef _DEBUG - for( i = 0; i < alpha_count; i++ ) + Solver( const Mat& _samples, const vector& _y, + vector& _alpha, const vector& _b, + double _Cp, double _Cn, + const Ptr& _kernel, GetRow _get_row, + SelectWorkingSet _select_working_set, CalcRho _calc_rho, + TermCriteria _termCrit ) { - if( fabs(G[i]) > 1e+300 ) - return false; - - if( fabs(alpha[i]) > 1e16 ) - return false; + clear(); + + samples = _samples; + sample_count = samples.rows; + var_count = samples.cols; + + y_vec = _y; + alpha_vec = &_alpha; + alpha_count = (int)alpha_vec->size(); + b_vec = _b; + kernel = _kernel; + + C[0] = _Cn; + C[1] = _Cp; + eps = _termCrit.epsilon; + max_iter = _termCrit.maxCount; + + G_vec.resize(alpha_count); + alpha_status_vec.resize(alpha_count); + buf[0].resize(sample_count*2); + buf[1].resize(sample_count*2); + + select_working_set_func = _select_working_set; + CV_Assert(select_working_set_func != 0); + + calc_rho_func = _calc_rho; + CV_Assert(calc_rho_func != 0); + + get_row_func = _get_row; + CV_Assert(get_row_func != 0); + + // assume that for large training sets ~25% of Q matrix is used + int64 csize = (int64)sample_count*sample_count/4; + csize = std::max(csize, (int64)(MIN_CACHE_SIZE/sizeof(Qfloat)) ); + csize = std::min(csize, (int64)(MAX_CACHE_SIZE/sizeof(Qfloat)) ); + max_cache_size = (int)((csize + sample_count-1)/sample_count); + max_cache_size = std::min(std::max(max_cache_size, 1), sample_count); + cache_size = 0; + + lru_cache.clear(); + lru_cache.resize(sample_count+1, KernelRow(-1, 0, 0)); + lru_first = lru_last = 0; + lru_cache_data.create(max_cache_size, sample_count, QFLOAT_TYPE); } -#endif - - if( (this->*select_working_set_func)( i, j ) != 0 || iter++ >= max_iter ) - break; - - Q_i = get_row( i, buf[0] ); - Q_j = get_row( j, buf[1] ); - - C_i = get_C(i); - C_j = get_C(j); - alpha_i = old_alpha_i = alpha[i]; - alpha_j = old_alpha_j = alpha[j]; - - if( y[i] != y[j] ) + Qfloat* get_row_base( int i, bool* _existed ) { - double denom = Q_i[i]+Q_j[j]+2*Q_i[j]; - double delta = (-G[i]-G[j])/MAX(fabs(denom),FLT_EPSILON); - double diff = alpha_i - alpha_j; - alpha_i += delta; - alpha_j += delta; - - if( diff > 0 && alpha_j < 0 ) + int i1 = i < sample_count ? i : i - sample_count; + KernelRow& kr = lru_cache[i1+1]; + if( _existed ) + *_existed = kr.idx >= 0; + if( kr.idx < 0 ) { - alpha_j = 0; - alpha_i = diff; + if( cache_size < max_cache_size ) + { + kr.idx = cache_size; + cache_size++; + } + else + { + KernelRow& last = lru_cache[lru_last]; + kr.idx = last.idx; + last.idx = -1; + lru_cache[last.prev].next = 0; + lru_last = last.prev; + } + kernel->calc( sample_count, var_count, samples.ptr(), + samples.ptr(i1), lru_cache_data.ptr(kr.idx) ); } - else if( diff <= 0 && alpha_i < 0 ) + else { - alpha_i = 0; - alpha_j = -diff; + if( kr.next ) + lru_cache[kr.next].prev = kr.prev; + else + lru_last = kr.prev; + if( kr.prev ) + lru_cache[kr.prev].next = kr.next; + else + lru_first = kr.next; } + kr.next = lru_first; + kr.prev = 0; + lru_first = i1+1; - if( diff > C_i - C_j && alpha_i > C_i ) - { - alpha_i = C_i; - alpha_j = C_i - diff; - } - else if( diff <= C_i - C_j && alpha_j > C_j ) - { - alpha_j = C_j; - alpha_i = C_j + diff; - } + return lru_cache_data.ptr(kr.idx); } - else - { - double denom = Q_i[i]+Q_j[j]-2*Q_i[j]; - double delta = (G[i]-G[j])/MAX(fabs(denom),FLT_EPSILON); - double sum = alpha_i + alpha_j; - alpha_i -= delta; - alpha_j += delta; - if( sum > C_i && alpha_i > C_i ) - { - alpha_i = C_i; - alpha_j = sum - C_i; - } - else if( sum <= C_i && alpha_j < 0) + Qfloat* get_row_svc( int i, Qfloat* row, Qfloat*, bool existed ) + { + if( !existed ) { - alpha_j = 0; - alpha_i = sum; - } + const schar* _y = &y_vec[0]; + int j, len = sample_count; - if( sum > C_j && alpha_j > C_j ) - { - alpha_j = C_j; - alpha_i = sum - C_j; - } - else if( sum <= C_j && alpha_i < 0 ) - { - alpha_i = 0; - alpha_j = sum; + if( _y[i] > 0 ) + { + for( j = 0; j < len; j++ ) + row[j] = _y[j]*row[j]; + } + else + { + for( j = 0; j < len; j++ ) + row[j] = -_y[j]*row[j]; + } } + return row; } - // update alpha - alpha[i] = alpha_i; - alpha[j] = alpha_j; - update_alpha_status(i); - update_alpha_status(j); - - // update G - delta_alpha_i = alpha_i - old_alpha_i; - delta_alpha_j = alpha_j - old_alpha_j; + Qfloat* get_row_one_class( int, Qfloat* row, Qfloat*, bool ) + { + return row; + } - for( k = 0; k < alpha_count; k++ ) - G[k] += Q_i[k]*delta_alpha_i + Q_j[k]*delta_alpha_j; - } + Qfloat* get_row_svr( int i, Qfloat* row, Qfloat* dst, bool ) + { + int j, len = sample_count; + Qfloat* dst_pos = dst; + Qfloat* dst_neg = dst + len; + if( i >= len ) + std::swap(dst_pos, dst_neg); - // calculate rho - (this->*calc_rho_func)( si.rho, si.r ); + for( j = 0; j < len; j++ ) + { + Qfloat t = row[j]; + dst_pos[j] = t; + dst_neg[j] = -t; + } + return dst; + } - // calculate objective value - for( i = 0, si.obj = 0; i < alpha_count; i++ ) - si.obj += alpha[i] * (G[i] + b[i]); + Qfloat* get_row( int i, float* dst ) + { + bool existed = false; + float* row = get_row_base( i, &existed ); + return (this->*get_row_func)( i, row, dst, existed ); + } - si.obj *= 0.5; + #undef is_upper_bound + #define is_upper_bound(i) (alpha_status[i] > 0) - si.upper_bound_p = C[1]; - si.upper_bound_n = C[0]; + #undef is_lower_bound + #define is_lower_bound(i) (alpha_status[i] < 0) - return true; -} + #undef is_free + #define is_free(i) (alpha_status[i] == 0) + #undef get_C + #define get_C(i) (C[y[i]>0]) -// return 1 if already optimal, return 0 otherwise -bool -CvSVMSolver::select_working_set( int& out_i, int& out_j ) -{ - // return i,j which maximize -grad(f)^T d , under constraint - // if alpha_i == C, d != +1 - // if alpha_i == 0, d != -1 - double Gmax1 = -DBL_MAX; // max { -grad(f)_i * d | y_i*d = +1 } - int Gmax1_idx = -1; + #undef update_alpha_status + #define update_alpha_status(i) \ + alpha_status[i] = (schar)(alpha[i] >= get_C(i) ? 1 : alpha[i] <= 0 ? -1 : 0) - double Gmax2 = -DBL_MAX; // max { -grad(f)_i * d | y_i*d = -1 } - int Gmax2_idx = -1; + #undef reconstruct_gradient + #define reconstruct_gradient() /* empty for now */ - int i; + bool solve_generic( SolutionInfo& si ) + { + const schar* y = &y_vec[0]; + double* alpha = &alpha_vec->at(0); + schar* alpha_status = &alpha_status_vec[0]; + double* G = &G_vec[0]; + double* b = &b_vec[0]; - for( i = 0; i < alpha_count; i++ ) - { - double t; + int iter = 0; + int i, j, k; - if( y[i] > 0 ) // y = +1 - { - if( !is_upper_bound(i) && (t = -G[i]) > Gmax1 ) // d = +1 + // 1. initialize gradient and alpha status + for( i = 0; i < alpha_count; i++ ) { - Gmax1 = t; - Gmax1_idx = i; + update_alpha_status(i); + G[i] = b[i]; + if( fabs(G[i]) > 1e200 ) + return false; } - if( !is_lower_bound(i) && (t = G[i]) > Gmax2 ) // d = -1 - { - Gmax2 = t; - Gmax2_idx = i; - } - } - else // y = -1 - { - if( !is_upper_bound(i) && (t = -G[i]) > Gmax2 ) // d = +1 + + for( i = 0; i < alpha_count; i++ ) { - Gmax2 = t; - Gmax2_idx = i; + if( !is_lower_bound(i) ) + { + const Qfloat *Q_i = get_row( i, &buf[0][0] ); + double alpha_i = alpha[i]; + + for( j = 0; j < alpha_count; j++ ) + G[j] += alpha_i*Q_i[j]; + } } - if( !is_lower_bound(i) && (t = G[i]) > Gmax1 ) // d = -1 + + // 2. optimization loop + for(;;) { - Gmax1 = t; - Gmax1_idx = i; - } - } - } + const Qfloat *Q_i, *Q_j; + double C_i, C_j; + double old_alpha_i, old_alpha_j, alpha_i, alpha_j; + double delta_alpha_i, delta_alpha_j; - out_i = Gmax1_idx; - out_j = Gmax2_idx; + #ifdef _DEBUG + for( i = 0; i < alpha_count; i++ ) + { + if( fabs(G[i]) > 1e+300 ) + return false; - return Gmax1 + Gmax2 < eps; -} + if( fabs(alpha[i]) > 1e16 ) + return false; + } + #endif + if( (this->*select_working_set_func)( i, j ) != 0 || iter++ >= max_iter ) + break; -void -CvSVMSolver::calc_rho( double& rho, double& r ) -{ - int i, nr_free = 0; - double ub = DBL_MAX, lb = -DBL_MAX, sum_free = 0; + Q_i = get_row( i, &buf[0][0] ); + Q_j = get_row( j, &buf[1][0] ); - for( i = 0; i < alpha_count; i++ ) - { - double yG = y[i]*G[i]; + C_i = get_C(i); + C_j = get_C(j); - if( is_lower_bound(i) ) - { - if( y[i] > 0 ) - ub = MIN(ub,yG); - else - lb = MAX(lb,yG); - } - else if( is_upper_bound(i) ) - { - if( y[i] < 0) - ub = MIN(ub,yG); - else - lb = MAX(lb,yG); - } - else - { - ++nr_free; - sum_free += yG; - } - } - - rho = nr_free > 0 ? sum_free/nr_free : (ub + lb)*0.5; - r = 0; -} + alpha_i = old_alpha_i = alpha[i]; + alpha_j = old_alpha_j = alpha[j]; + if( y[i] != y[j] ) + { + double denom = Q_i[i]+Q_j[j]+2*Q_i[j]; + double delta = (-G[i]-G[j])/MAX(fabs(denom),FLT_EPSILON); + double diff = alpha_i - alpha_j; + alpha_i += delta; + alpha_j += delta; -bool -CvSVMSolver::select_working_set_nu_svm( int& out_i, int& out_j ) -{ - // return i,j which maximize -grad(f)^T d , under constraint - // if alpha_i == C, d != +1 - // if alpha_i == 0, d != -1 - double Gmax1 = -DBL_MAX; // max { -grad(f)_i * d | y_i = +1, d = +1 } - int Gmax1_idx = -1; + if( diff > 0 && alpha_j < 0 ) + { + alpha_j = 0; + alpha_i = diff; + } + else if( diff <= 0 && alpha_i < 0 ) + { + alpha_i = 0; + alpha_j = -diff; + } - double Gmax2 = -DBL_MAX; // max { -grad(f)_i * d | y_i = +1, d = -1 } - int Gmax2_idx = -1; + if( diff > C_i - C_j && alpha_i > C_i ) + { + alpha_i = C_i; + alpha_j = C_i - diff; + } + else if( diff <= C_i - C_j && alpha_j > C_j ) + { + alpha_j = C_j; + alpha_i = C_j + diff; + } + } + else + { + double denom = Q_i[i]+Q_j[j]-2*Q_i[j]; + double delta = (G[i]-G[j])/MAX(fabs(denom),FLT_EPSILON); + double sum = alpha_i + alpha_j; + alpha_i -= delta; + alpha_j += delta; - double Gmax3 = -DBL_MAX; // max { -grad(f)_i * d | y_i = -1, d = +1 } - int Gmax3_idx = -1; + if( sum > C_i && alpha_i > C_i ) + { + alpha_i = C_i; + alpha_j = sum - C_i; + } + else if( sum <= C_i && alpha_j < 0) + { + alpha_j = 0; + alpha_i = sum; + } - double Gmax4 = -DBL_MAX; // max { -grad(f)_i * d | y_i = -1, d = -1 } - int Gmax4_idx = -1; + if( sum > C_j && alpha_j > C_j ) + { + alpha_j = C_j; + alpha_i = sum - C_j; + } + else if( sum <= C_j && alpha_i < 0 ) + { + alpha_i = 0; + alpha_j = sum; + } + } - int i; + // update alpha + alpha[i] = alpha_i; + alpha[j] = alpha_j; + update_alpha_status(i); + update_alpha_status(j); - for( i = 0; i < alpha_count; i++ ) - { - double t; + // update G + delta_alpha_i = alpha_i - old_alpha_i; + delta_alpha_j = alpha_j - old_alpha_j; - if( y[i] > 0 ) // y == +1 - { - if( !is_upper_bound(i) && (t = -G[i]) > Gmax1 ) // d = +1 - { - Gmax1 = t; - Gmax1_idx = i; - } - if( !is_lower_bound(i) && (t = G[i]) > Gmax2 ) // d = -1 - { - Gmax2 = t; - Gmax2_idx = i; + for( k = 0; k < alpha_count; k++ ) + G[k] += Q_i[k]*delta_alpha_i + Q_j[k]*delta_alpha_j; } - } - else // y == -1 - { - if( !is_upper_bound(i) && (t = -G[i]) > Gmax3 ) // d = +1 - { - Gmax3 = t; - Gmax3_idx = i; - } - if( !is_lower_bound(i) && (t = G[i]) > Gmax4 ) // d = -1 - { - Gmax4 = t; - Gmax4_idx = i; - } - } - } - if( MAX(Gmax1 + Gmax2, Gmax3 + Gmax4) < eps ) - return 1; + // calculate rho + (this->*calc_rho_func)( si.rho, si.r ); - if( Gmax1 + Gmax2 > Gmax3 + Gmax4 ) - { - out_i = Gmax1_idx; - out_j = Gmax2_idx; - } - else - { - out_i = Gmax3_idx; - out_j = Gmax4_idx; - } - return 0; -} + // calculate objective value + for( i = 0, si.obj = 0; i < alpha_count; i++ ) + si.obj += alpha[i] * (G[i] + b[i]); + si.obj *= 0.5; -void -CvSVMSolver::calc_rho_nu_svm( double& rho, double& r ) -{ - int nr_free1 = 0, nr_free2 = 0; - double ub1 = DBL_MAX, ub2 = DBL_MAX; - double lb1 = -DBL_MAX, lb2 = -DBL_MAX; - double sum_free1 = 0, sum_free2 = 0; - double r1, r2; - - int i; + si.upper_bound_p = C[1]; + si.upper_bound_n = C[0]; - for( i = 0; i < alpha_count; i++ ) - { - double G_i = G[i]; - if( y[i] > 0 ) - { - if( is_lower_bound(i) ) - ub1 = MIN( ub1, G_i ); - else if( is_upper_bound(i) ) - lb1 = MAX( lb1, G_i ); - else - { - ++nr_free1; - sum_free1 += G_i; - } - } - else - { - if( is_lower_bound(i) ) - ub2 = MIN( ub2, G_i ); - else if( is_upper_bound(i) ) - lb2 = MAX( lb2, G_i ); - else - { - ++nr_free2; - sum_free2 += G_i; - } + return true; } - } - - r1 = nr_free1 > 0 ? sum_free1/nr_free1 : (ub1 + lb1)*0.5; - r2 = nr_free2 > 0 ? sum_free2/nr_free2 : (ub2 + lb2)*0.5; - - rho = (r1 - r2)*0.5; - r = (r1 + r2)*0.5; -} - - -/* -///////////////////////// construct and solve various formulations /////////////////////// -*/ - -bool CvSVMSolver::solve_c_svc( int _sample_count, int _var_count, const float** _samples, schar* _y, - double _Cp, double _Cn, CvMemStorage* _storage, - CvSVMKernel* _kernel, double* _alpha, CvSVMSolutionInfo& _si ) -{ - int i; - - if( !create( _sample_count, _var_count, _samples, _y, _sample_count, - _alpha, _Cp, _Cn, _storage, _kernel, &CvSVMSolver::get_row_svc, - &CvSVMSolver::select_working_set, &CvSVMSolver::calc_rho )) - return false; - - for( i = 0; i < sample_count; i++ ) - { - alpha[i] = 0; - b[i] = -1; - } - if( !solve_generic( _si )) - return false; - - for( i = 0; i < sample_count; i++ ) - alpha[i] *= y[i]; - - return true; -} - - -bool CvSVMSolver::solve_nu_svc( int _sample_count, int _var_count, const float** _samples, schar* _y, - CvMemStorage* _storage, CvSVMKernel* _kernel, - double* _alpha, CvSVMSolutionInfo& _si ) -{ - int i; - double sum_pos, sum_neg, inv_r; - - if( !create( _sample_count, _var_count, _samples, _y, _sample_count, - _alpha, 1., 1., _storage, _kernel, &CvSVMSolver::get_row_svc, - &CvSVMSolver::select_working_set_nu_svm, &CvSVMSolver::calc_rho_nu_svm )) - return false; - - sum_pos = kernel->params->nu * sample_count * 0.5; - sum_neg = kernel->params->nu * sample_count * 0.5; - - for( i = 0; i < sample_count; i++ ) - { - if( y[i] > 0 ) - { - alpha[i] = MIN(1.0, sum_pos); - sum_pos -= alpha[i]; - } - else + // return 1 if already optimal, return 0 otherwise + bool select_working_set( int& out_i, int& out_j ) { - alpha[i] = MIN(1.0, sum_neg); - sum_neg -= alpha[i]; - } - b[i] = 0; - } - - if( !solve_generic( _si )) - return false; - - inv_r = 1./_si.r; - - for( i = 0; i < sample_count; i++ ) - alpha[i] *= y[i]*inv_r; - - _si.rho *= inv_r; - _si.obj *= (inv_r*inv_r); - _si.upper_bound_p = inv_r; - _si.upper_bound_n = inv_r; - - return true; -} - - -bool CvSVMSolver::solve_one_class( int _sample_count, int _var_count, const float** _samples, - CvMemStorage* _storage, CvSVMKernel* _kernel, - double* _alpha, CvSVMSolutionInfo& _si ) -{ - int i, n; - double nu = _kernel->params->nu; - - if( !create( _sample_count, _var_count, _samples, 0, _sample_count, - _alpha, 1., 1., _storage, _kernel, &CvSVMSolver::get_row_one_class, - &CvSVMSolver::select_working_set, &CvSVMSolver::calc_rho )) - return false; - - y = (schar*)cvMemStorageAlloc( storage, sample_count*sizeof(y[0]) ); - n = cvRound( nu*sample_count ); - - for( i = 0; i < sample_count; i++ ) - { - y[i] = 1; - b[i] = 0; - alpha[i] = i < n ? 1 : 0; - } - - if( n < sample_count ) - alpha[n] = nu * sample_count - n; - else - alpha[n-1] = nu * sample_count - (n-1); - - return solve_generic(_si); -} - - -bool CvSVMSolver::solve_eps_svr( int _sample_count, int _var_count, const float** _samples, - const float* _y, CvMemStorage* _storage, - CvSVMKernel* _kernel, double* _alpha, CvSVMSolutionInfo& _si ) -{ - int i; - double p = _kernel->params->p, kernel_param_c = _kernel->params->C; - - if( !create( _sample_count, _var_count, _samples, 0, - _sample_count*2, 0, kernel_param_c, kernel_param_c, _storage, _kernel, &CvSVMSolver::get_row_svr, - &CvSVMSolver::select_working_set, &CvSVMSolver::calc_rho )) - return false; - - y = (schar*)cvMemStorageAlloc( storage, sample_count*2*sizeof(y[0]) ); - alpha = (double*)cvMemStorageAlloc( storage, alpha_count*sizeof(alpha[0]) ); - - for( i = 0; i < sample_count; i++ ) - { - alpha[i] = 0; - b[i] = p - _y[i]; - y[i] = 1; - - alpha[i+sample_count] = 0; - b[i+sample_count] = p + _y[i]; - y[i+sample_count] = -1; - } - - if( !solve_generic( _si )) - return false; - - for( i = 0; i < sample_count; i++ ) - _alpha[i] = alpha[i] - alpha[i+sample_count]; - - return true; -} - - -bool CvSVMSolver::solve_nu_svr( int _sample_count, int _var_count, const float** _samples, - const float* _y, CvMemStorage* _storage, - CvSVMKernel* _kernel, double* _alpha, CvSVMSolutionInfo& _si ) -{ - int i; - double kernel_param_c = _kernel->params->C, sum; - - if( !create( _sample_count, _var_count, _samples, 0, - _sample_count*2, 0, 1., 1., _storage, _kernel, &CvSVMSolver::get_row_svr, - &CvSVMSolver::select_working_set_nu_svm, &CvSVMSolver::calc_rho_nu_svm )) - return false; - - y = (schar*)cvMemStorageAlloc( storage, sample_count*2*sizeof(y[0]) ); - alpha = (double*)cvMemStorageAlloc( storage, alpha_count*sizeof(alpha[0]) ); - sum = kernel_param_c * _kernel->params->nu * sample_count * 0.5; - - for( i = 0; i < sample_count; i++ ) - { - alpha[i] = alpha[i + sample_count] = MIN(sum, kernel_param_c); - sum -= alpha[i]; - - b[i] = -_y[i]; - y[i] = 1; - - b[i + sample_count] = _y[i]; - y[i + sample_count] = -1; - } - - if( !solve_generic( _si )) - return false; - - for( i = 0; i < sample_count; i++ ) - _alpha[i] = alpha[i] - alpha[i+sample_count]; - - return true; -} - - -////////////////////////////////////////////////////////////////////////////////////////// - -CvSVM::CvSVM() -{ - decision_func = 0; - class_labels = 0; - class_weights = 0; - storage = 0; - var_idx = 0; - kernel = 0; - solver = 0; - default_model_name = "my_svm"; - - clear(); -} - - -CvSVM::~CvSVM() -{ - clear(); -} - - -void CvSVM::clear() -{ - cvFree( &decision_func ); - cvReleaseMat( &class_labels ); - cvReleaseMat( &class_weights ); - cvReleaseMemStorage( &storage ); - cvReleaseMat( &var_idx ); - delete kernel; - delete solver; - kernel = 0; - solver = 0; - var_all = 0; - sv = 0; - sv_total = 0; -} - - -CvSVM::CvSVM( const CvMat* _train_data, const CvMat* _responses, - const CvMat* _var_idx, const CvMat* _sample_idx, CvSVMParams _params ) -{ - decision_func = 0; - class_labels = 0; - class_weights = 0; - storage = 0; - var_idx = 0; - kernel = 0; - solver = 0; - default_model_name = "my_svm"; - - train( _train_data, _responses, _var_idx, _sample_idx, _params ); -} - - -int CvSVM::get_support_vector_count() const -{ - return sv_total; -} - - -const float* CvSVM::get_support_vector(int i) const -{ - return sv && (unsigned)i < (unsigned)sv_total ? sv[i] : 0; -} - -bool CvSVM::set_params( const CvSVMParams& _params ) -{ - bool ok = false; - - CV_FUNCNAME( "CvSVM::set_params" ); - - __BEGIN__; - - int kernel_type, svm_type; - - params = _params; - - kernel_type = params.kernel_type; - svm_type = params.svm_type; - - if( kernel_type != LINEAR && kernel_type != POLY && - kernel_type != SIGMOID && kernel_type != RBF && - kernel_type != INTER && kernel_type != CHI2) - CV_ERROR( CV_StsBadArg, "Unknown/unsupported kernel type" ); - - if( kernel_type == LINEAR ) - params.gamma = 1; - else if( params.gamma <= 0 ) - CV_ERROR( CV_StsOutOfRange, "gamma parameter of the kernel must be positive" ); - - if( kernel_type != SIGMOID && kernel_type != POLY ) - params.coef0 = 0; - else if( params.coef0 < 0 ) - CV_ERROR( CV_StsOutOfRange, "The kernel parameter must be positive or zero" ); - - if( kernel_type != POLY ) - params.degree = 0; - else if( params.degree <= 0 ) - CV_ERROR( CV_StsOutOfRange, "The kernel parameter must be positive" ); - - if( svm_type != C_SVC && svm_type != NU_SVC && - svm_type != ONE_CLASS && svm_type != EPS_SVR && - svm_type != NU_SVR ) - CV_ERROR( CV_StsBadArg, "Unknown/unsupported SVM type" ); - - if( svm_type == ONE_CLASS || svm_type == NU_SVC ) - params.C = 0; - else if( params.C <= 0 ) - CV_ERROR( CV_StsOutOfRange, "The parameter C must be positive" ); - - if( svm_type == C_SVC || svm_type == EPS_SVR ) - params.nu = 0; - else if( params.nu <= 0 || params.nu >= 1 ) - CV_ERROR( CV_StsOutOfRange, "The parameter nu must be between 0 and 1" ); - - if( svm_type != EPS_SVR ) - params.p = 0; - else if( params.p <= 0 ) - CV_ERROR( CV_StsOutOfRange, "The parameter p must be positive" ); - - if( svm_type != C_SVC ) - params.class_weights = 0; - - params.term_crit = cvCheckTermCriteria( params.term_crit, DBL_EPSILON, INT_MAX ); - params.term_crit.epsilon = MAX( params.term_crit.epsilon, DBL_EPSILON ); - ok = true; - - __END__; - - return ok; -} - - - -void CvSVM::create_kernel() -{ - kernel = new CvSVMKernel(¶ms,0); -} - - -void CvSVM::create_solver( ) -{ - solver = new CvSVMSolver; -} - - -// switching function -bool CvSVM::train1( int sample_count, int var_count, const float** samples, - const void* _responses, double Cp, double Cn, - CvMemStorage* _storage, double* alpha, double& rho ) -{ - bool ok = false; - - //CV_FUNCNAME( "CvSVM::train1" ); - - __BEGIN__; - - CvSVMSolutionInfo si; - int svm_type = params.svm_type; - - si.rho = 0; - - ok = svm_type == C_SVC ? solver->solve_c_svc( sample_count, var_count, samples, (schar*)_responses, - Cp, Cn, _storage, kernel, alpha, si ) : - svm_type == NU_SVC ? solver->solve_nu_svc( sample_count, var_count, samples, (schar*)_responses, - _storage, kernel, alpha, si ) : - svm_type == ONE_CLASS ? solver->solve_one_class( sample_count, var_count, samples, - _storage, kernel, alpha, si ) : - svm_type == EPS_SVR ? solver->solve_eps_svr( sample_count, var_count, samples, (float*)_responses, - _storage, kernel, alpha, si ) : - svm_type == NU_SVR ? solver->solve_nu_svr( sample_count, var_count, samples, (float*)_responses, - _storage, kernel, alpha, si ) : false; - - rho = si.rho; + // return i,j which maximize -grad(f)^T d , under constraint + // if alpha_i == C, d != +1 + // if alpha_i == 0, d != -1 + double Gmax1 = -DBL_MAX; // max { -grad(f)_i * d | y_i*d = +1 } + int Gmax1_idx = -1; - __END__; + double Gmax2 = -DBL_MAX; // max { -grad(f)_i * d | y_i*d = -1 } + int Gmax2_idx = -1; - return ok; -} - - -bool CvSVM::do_train( int svm_type, int sample_count, int var_count, const float** samples, - const CvMat* responses, CvMemStorage* temp_storage, double* alpha ) -{ - bool ok = false; - - CV_FUNCNAME( "CvSVM::do_train" ); - - __BEGIN__; - - CvSVMDecisionFunc* df = 0; - const int sample_size = var_count*sizeof(samples[0][0]); - int i, j, k; + const schar* y = &y_vec[0]; + const schar* alpha_status = &alpha_status_vec[0]; + const double* G = &G_vec[0]; - cvClearMemStorage( storage ); - - if( svm_type == ONE_CLASS || svm_type == EPS_SVR || svm_type == NU_SVR ) - { - int sv_count = 0; - - CV_CALL( decision_func = df = - (CvSVMDecisionFunc*)cvAlloc( sizeof(df[0]) )); - - df->rho = 0; - if( !train1( sample_count, var_count, samples, svm_type == ONE_CLASS ? 0 : - responses->data.i, 0, 0, temp_storage, alpha, df->rho )) - EXIT; - - for( i = 0; i < sample_count; i++ ) - sv_count += fabs(alpha[i]) > 0; - - CV_Assert(sv_count != 0); - - sv_total = df->sv_count = sv_count; - CV_CALL( df->alpha = (double*)cvMemStorageAlloc( storage, sv_count*sizeof(df->alpha[0])) ); - CV_CALL( sv = (float**)cvMemStorageAlloc( storage, sv_count*sizeof(sv[0]))); - - for( i = k = 0; i < sample_count; i++ ) - { - if( fabs(alpha[i]) > 0 ) + for( int i = 0; i < alpha_count; i++ ) { - CV_CALL( sv[k] = (float*)cvMemStorageAlloc( storage, sample_size )); - memcpy( sv[k], samples[i], sample_size ); - df->alpha[k++] = alpha[i]; - } - } - } - else - { - int class_count = class_labels->cols; - int* sv_tab = 0; - const float** temp_samples = 0; - int* class_ranges = 0; - schar* temp_y = 0; - assert( svm_type == CvSVM::C_SVC || svm_type == CvSVM::NU_SVC ); - - if( svm_type == CvSVM::C_SVC && params.class_weights ) - { - const CvMat* cw = params.class_weights; - - if( !CV_IS_MAT(cw) || (cw->cols != 1 && cw->rows != 1) || - cw->rows + cw->cols - 1 != class_count || - (CV_MAT_TYPE(cw->type) != CV_32FC1 && CV_MAT_TYPE(cw->type) != CV_64FC1) ) - CV_ERROR( CV_StsBadArg, "params.class_weights must be 1d floating-point vector " - "containing as many elements as the number of classes" ); - - CV_CALL( class_weights = cvCreateMat( cw->rows, cw->cols, CV_64F )); - CV_CALL( cvConvert( cw, class_weights )); - CV_CALL( cvScale( class_weights, class_weights, params.C )); - } + double t; - CV_CALL( decision_func = df = (CvSVMDecisionFunc*)cvAlloc( - (class_count*(class_count-1)/2)*sizeof(df[0]))); - - CV_CALL( sv_tab = (int*)cvMemStorageAlloc( temp_storage, sample_count*sizeof(sv_tab[0]) )); - memset( sv_tab, 0, sample_count*sizeof(sv_tab[0]) ); - CV_CALL( class_ranges = (int*)cvMemStorageAlloc( temp_storage, - (class_count + 1)*sizeof(class_ranges[0]))); - CV_CALL( temp_samples = (const float**)cvMemStorageAlloc( temp_storage, - sample_count*sizeof(temp_samples[0]))); - CV_CALL( temp_y = (schar*)cvMemStorageAlloc( temp_storage, sample_count)); - - class_ranges[class_count] = 0; - cvSortSamplesByClasses( samples, responses, class_ranges, 0 ); - //check that while cross-validation there were the samples from all the classes - if( class_ranges[class_count] <= 0 ) - CV_ERROR( CV_StsBadArg, "While cross-validation one or more of the classes have " - "been fell out of the sample. Try to enlarge " ); - - if( svm_type == NU_SVC ) - { - // check if nu is feasible - for(i = 0; i < class_count; i++ ) - { - int ci = class_ranges[i+1] - class_ranges[i]; - for( j = i+1; j< class_count; j++ ) + if( y[i] > 0 ) // y = +1 { - int cj = class_ranges[j+1] - class_ranges[j]; - if( params.nu*(ci + cj)*0.5 > MIN( ci, cj ) ) + if( !is_upper_bound(i) && (t = -G[i]) > Gmax1 ) // d = +1 { - // !!!TODO!!! add some diagnostic - EXIT; // exit immediately; will release the model and return NULL pointer + Gmax1 = t; + Gmax1_idx = i; + } + if( !is_lower_bound(i) && (t = G[i]) > Gmax2 ) // d = -1 + { + Gmax2 = t; + Gmax2_idx = i; + } + } + else // y = -1 + { + if( !is_upper_bound(i) && (t = -G[i]) > Gmax2 ) // d = +1 + { + Gmax2 = t; + Gmax2_idx = i; + } + if( !is_lower_bound(i) && (t = G[i]) > Gmax1 ) // d = -1 + { + Gmax1 = t; + Gmax1_idx = i; } } } + + out_i = Gmax1_idx; + out_j = Gmax2_idx; + + return Gmax1 + Gmax2 < eps; } - // train n*(n-1)/2 classifiers - for( i = 0; i < class_count; i++ ) + void calc_rho( double& rho, double& r ) { - for( j = i+1; j < class_count; j++, df++ ) + int nr_free = 0; + double ub = DBL_MAX, lb = -DBL_MAX, sum_free = 0; + const schar* y = &y_vec[0]; + const schar* alpha_status = &alpha_status_vec[0]; + const double* G = &G_vec[0]; + + for( int i = 0; i < alpha_count; i++ ) { - int si = class_ranges[i], ci = class_ranges[i+1] - si; - int sj = class_ranges[j], cj = class_ranges[j+1] - sj; - double Cp = params.C, Cn = Cp; - int k1 = 0, sv_count = 0; + double yG = y[i]*G[i]; - for( k = 0; k < ci; k++ ) + if( is_lower_bound(i) ) { - temp_samples[k] = samples[si + k]; - temp_y[k] = 1; + if( y[i] > 0 ) + ub = MIN(ub,yG); + else + lb = MAX(lb,yG); } - - for( k = 0; k < cj; k++ ) + else if( is_upper_bound(i) ) { - temp_samples[ci + k] = samples[sj + k]; - temp_y[ci + k] = -1; + if( y[i] < 0) + ub = MIN(ub,yG); + else + lb = MAX(lb,yG); } - - if( class_weights ) + else { - Cp = class_weights->data.db[i]; - Cn = class_weights->data.db[j]; + ++nr_free; + sum_free += yG; } + } + + rho = nr_free > 0 ? sum_free/nr_free : (ub + lb)*0.5; + r = 0; + } + + bool select_working_set_nu_svm( int& out_i, int& out_j ) + { + // return i,j which maximize -grad(f)^T d , under constraint + // if alpha_i == C, d != +1 + // if alpha_i == 0, d != -1 + double Gmax1 = -DBL_MAX; // max { -grad(f)_i * d | y_i = +1, d = +1 } + int Gmax1_idx = -1; - if( !train1( ci + cj, var_count, temp_samples, temp_y, - Cp, Cn, temp_storage, alpha, df->rho )) - EXIT; + double Gmax2 = -DBL_MAX; // max { -grad(f)_i * d | y_i = +1, d = -1 } + int Gmax2_idx = -1; - for( k = 0; k < ci + cj; k++ ) - sv_count += fabs(alpha[k]) > 0; + double Gmax3 = -DBL_MAX; // max { -grad(f)_i * d | y_i = -1, d = +1 } + int Gmax3_idx = -1; - df->sv_count = sv_count; + double Gmax4 = -DBL_MAX; // max { -grad(f)_i * d | y_i = -1, d = -1 } + int Gmax4_idx = -1; - CV_CALL( df->alpha = (double*)cvMemStorageAlloc( temp_storage, - sv_count*sizeof(df->alpha[0]))); - CV_CALL( df->sv_index = (int*)cvMemStorageAlloc( temp_storage, - sv_count*sizeof(df->sv_index[0]))); + const schar* y = &y_vec[0]; + const schar* alpha_status = &alpha_status_vec[0]; + const double* G = &G_vec[0]; - for( k = 0; k < ci; k++ ) + for( int i = 0; i < alpha_count; i++ ) + { + double t; + + if( y[i] > 0 ) // y == +1 { - if( fabs(alpha[k]) > 0 ) + if( !is_upper_bound(i) && (t = -G[i]) > Gmax1 ) // d = +1 { - sv_tab[si + k] = 1; - df->sv_index[k1] = si + k; - df->alpha[k1++] = alpha[k]; + Gmax1 = t; + Gmax1_idx = i; + } + if( !is_lower_bound(i) && (t = G[i]) > Gmax2 ) // d = -1 + { + Gmax2 = t; + Gmax2_idx = i; } } - - for( k = 0; k < cj; k++ ) + else // y == -1 { - if( fabs(alpha[ci + k]) > 0 ) + if( !is_upper_bound(i) && (t = -G[i]) > Gmax3 ) // d = +1 + { + Gmax3 = t; + Gmax3_idx = i; + } + if( !is_lower_bound(i) && (t = G[i]) > Gmax4 ) // d = -1 { - sv_tab[sj + k] = 1; - df->sv_index[k1] = sj + k; - df->alpha[k1++] = alpha[ci + k]; + Gmax4 = t; + Gmax4_idx = i; } } } - } - // allocate support vectors and initialize sv_tab - for( i = 0, k = 0; i < sample_count; i++ ) - { - if( sv_tab[i] ) - sv_tab[i] = ++k; - } + if( MAX(Gmax1 + Gmax2, Gmax3 + Gmax4) < eps ) + return 1; - sv_total = k; - CV_CALL( sv = (float**)cvMemStorageAlloc( storage, sv_total*sizeof(sv[0]))); - - for( i = 0, k = 0; i < sample_count; i++ ) - { - if( sv_tab[i] ) + if( Gmax1 + Gmax2 > Gmax3 + Gmax4 ) + { + out_i = Gmax1_idx; + out_j = Gmax2_idx; + } + else { - CV_CALL( sv[k] = (float*)cvMemStorageAlloc( storage, sample_size )); - memcpy( sv[k], samples[i], sample_size ); - k++; + out_i = Gmax3_idx; + out_j = Gmax4_idx; } + return 0; } - df = (CvSVMDecisionFunc*)decision_func; - - // set sv pointers - for( i = 0; i < class_count; i++ ) + void calc_rho_nu_svm( double& rho, double& r ) { - for( j = i+1; j < class_count; j++, df++ ) + int nr_free1 = 0, nr_free2 = 0; + double ub1 = DBL_MAX, ub2 = DBL_MAX; + double lb1 = -DBL_MAX, lb2 = -DBL_MAX; + double sum_free1 = 0, sum_free2 = 0; + + const schar* y = &y_vec[0]; + const schar* alpha_status = &alpha_status_vec[0]; + const double* G = &G_vec[0]; + + for( int i = 0; i < alpha_count; i++ ) { - for( k = 0; k < df->sv_count; k++ ) + double G_i = G[i]; + if( y[i] > 0 ) + { + if( is_lower_bound(i) ) + ub1 = MIN( ub1, G_i ); + else if( is_upper_bound(i) ) + lb1 = MAX( lb1, G_i ); + else + { + ++nr_free1; + sum_free1 += G_i; + } + } + else { - df->sv_index[k] = sv_tab[df->sv_index[k]]-1; - assert( (unsigned)df->sv_index[k] < (unsigned)sv_total ); + if( is_lower_bound(i) ) + ub2 = MIN( ub2, G_i ); + else if( is_upper_bound(i) ) + lb2 = MAX( lb2, G_i ); + else + { + ++nr_free2; + sum_free2 += G_i; + } } } - } - } - - optimize_linear_svm(); - ok = true; - __END__; + double r1 = nr_free1 > 0 ? sum_free1/nr_free1 : (ub1 + lb1)*0.5; + double r2 = nr_free2 > 0 ? sum_free2/nr_free2 : (ub2 + lb2)*0.5; - return ok; -} - - -void CvSVM::optimize_linear_svm() -{ - // we optimize only linear SVM: compress all the support vectors into one. - if( params.kernel_type != LINEAR ) - return; - - int class_count = class_labels ? class_labels->cols : - params.svm_type == CvSVM::ONE_CLASS ? 1 : 0; - - int i, df_count = class_count > 1 ? class_count*(class_count-1)/2 : 1; - CvSVMDecisionFunc* df = decision_func; - - for( i = 0; i < df_count; i++ ) - { - int sv_count = df[i].sv_count; - if( sv_count != 1 ) - break; - } - - // if every decision functions uses a single support vector; - // it's already compressed. skip it then. - if( i == df_count ) - return; - - int var_count = get_var_count(); - cv::AutoBuffer vbuf(var_count); - double* v = vbuf; - float** new_sv = (float**)cvMemStorageAlloc(storage, df_count*sizeof(new_sv[0])); - - for( i = 0; i < df_count; i++ ) - { - new_sv[i] = (float*)cvMemStorageAlloc(storage, var_count*sizeof(new_sv[i][0])); - float* dst = new_sv[i]; - memset(v, 0, var_count*sizeof(v[0])); - int j, k, sv_count = df[i].sv_count; - for( j = 0; j < sv_count; j++ ) - { - const float* src = class_count > 1 && df[i].sv_index ? sv[df[i].sv_index[j]] : sv[j]; - double a = df[i].alpha[j]; - for( k = 0; k < var_count; k++ ) - v[k] += src[k]*a; + rho = (r1 - r2)*0.5; + r = (r1 + r2)*0.5; } - for( k = 0; k < var_count; k++ ) - dst[k] = (float)v[k]; - df[i].sv_count = 1; - df[i].alpha[0] = 1.; - if( class_count > 1 && df[i].sv_index ) - df[i].sv_index[0] = i; - } - - sv = new_sv; - sv_total = df_count; -} - - -bool CvSVM::train( const CvMat* _train_data, const CvMat* _responses, - const CvMat* _var_idx, const CvMat* _sample_idx, CvSVMParams _params ) -{ - bool ok = false; - CvMat* responses = 0; - CvMemStorage* temp_storage = 0; - const float** samples = 0; - - CV_FUNCNAME( "CvSVM::train" ); - - __BEGIN__; - int svm_type, sample_count, var_count, sample_size; - int block_size = 1 << 16; - double* alpha; - - clear(); - CV_CALL( set_params( _params )); - - svm_type = _params.svm_type; - - /* Prepare training data and related parameters */ - CV_CALL( cvPrepareTrainData( "CvSVM::train", _train_data, CV_ROW_SAMPLE, - svm_type != CvSVM::ONE_CLASS ? _responses : 0, - svm_type == CvSVM::C_SVC || - svm_type == CvSVM::NU_SVC ? CV_VAR_CATEGORICAL : - CV_VAR_ORDERED, _var_idx, _sample_idx, - false, &samples, &sample_count, &var_count, &var_all, - &responses, &class_labels, &var_idx )); - - - sample_size = var_count*sizeof(samples[0][0]); + /* + ///////////////////////// construct and solve various formulations /////////////////////// + */ + static bool solve_c_svc( const Mat& _samples, const vector& _y, + double _Cp, double _Cn, const Ptr& _kernel, + vector& _alpha, SolutionInfo& _si, TermCriteria termCrit ) + { + int sample_count = _samples.rows; - // make the storage block size large enough to fit all - // the temporary vectors and output support vectors. - block_size = MAX( block_size, sample_count*(int)sizeof(CvSVMKernelRow)); - block_size = MAX( block_size, sample_count*2*(int)sizeof(double) + 1024 ); - block_size = MAX( block_size, sample_size*2 + 1024 ); + _alpha.assign(sample_count, 0.); + vector _b(sample_count, -1.); - CV_CALL( storage = cvCreateMemStorage(block_size + sizeof(CvMemBlock) + sizeof(CvSeqBlock))); - CV_CALL( temp_storage = cvCreateChildMemStorage(storage)); - CV_CALL( alpha = (double*)cvMemStorageAlloc(temp_storage, sample_count*sizeof(double))); + Solver solver( _samples, _y, _alpha, _b, _Cp, _Cn, _kernel, + &Solver::get_row_svc, + &Solver::select_working_set, + &Solver::calc_rho, + termCrit ); - create_kernel(); - create_solver(); + if( !solver.solve_generic( _si )) + return false; - if( !do_train( svm_type, sample_count, var_count, samples, responses, temp_storage, alpha )) - EXIT; + for( int i = 0; i < sample_count; i++ ) + _alpha[i] *= _y[i]; - ok = true; // model has been trained succesfully + return true; + } - __END__; - delete solver; - solver = 0; - cvReleaseMemStorage( &temp_storage ); - cvReleaseMat( &responses ); - cvFree( &samples ); + static bool solve_nu_svc( const Mat& _samples, const vector& _y, + double nu, const Ptr& _kernel, + vector& _alpha, SolutionInfo& _si, + TermCriteria termCrit ) + { + int sample_count = _samples.rows; - if( cvGetErrStatus() < 0 || !ok ) - clear(); + _alpha.resize(sample_count); + vector _b(sample_count, 0.); - return ok; -} + double sum_pos = nu * sample_count * 0.5; + double sum_neg = nu * sample_count * 0.5; -struct indexedratio -{ - double val; - int ind; - int count_smallest, count_biggest; - void eval() { val = (double) count_smallest/(count_smallest+count_biggest); } -}; - -static int CV_CDECL -icvCmpIndexedratio( const void* a, const void* b ) -{ - return ((const indexedratio*)a)->val < ((const indexedratio*)b)->val ? -1 - : ((const indexedratio*)a)->val > ((const indexedratio*)b)->val ? 1 - : 0; -} - -bool CvSVM::train_auto( const CvMat* _train_data, const CvMat* _responses, - const CvMat* _var_idx, const CvMat* _sample_idx, CvSVMParams _params, int k_fold, - CvParamGrid C_grid, CvParamGrid gamma_grid, CvParamGrid p_grid, - CvParamGrid nu_grid, CvParamGrid coef_grid, CvParamGrid degree_grid, - bool balanced) -{ - bool ok = false; - CvMat* responses = 0; - CvMat* responses_local = 0; - CvMemStorage* temp_storage = 0; - const float** samples = 0; - const float** samples_local = 0; - - CV_FUNCNAME( "CvSVM::train_auto" ); - __BEGIN__; - - int svm_type, sample_count, var_count, sample_size; - int block_size = 1 << 16; - double* alpha; - RNG* rng = &theRNG(); - - // all steps are logarithmic and must be > 1 - double degree_step = 10, g_step = 10, coef_step = 10, C_step = 10, nu_step = 10, p_step = 10; - double gamma = 0, curr_c = 0, degree = 0, coef = 0, p = 0, nu = 0; - double best_degree = 0, best_gamma = 0, best_coef = 0, best_C = 0, best_nu = 0, best_p = 0; - float min_error = FLT_MAX, error; - - if( _params.svm_type == CvSVM::ONE_CLASS ) - { - if(!train( _train_data, _responses, _var_idx, _sample_idx, _params )) - EXIT; - return true; - } + for( int i = 0; i < sample_count; i++ ) + { + double a; + if( _y[i] > 0 ) + { + a = std::min(1.0, sum_pos); + sum_pos -= a; + } + else + { + a = std::min(1.0, sum_neg); + sum_neg -= a; + } + _alpha[i] = a; + } - clear(); + Solver solver( _samples, _y, _alpha, _b, 1., 1., _kernel, + &Solver::get_row_svc, + &Solver::select_working_set_nu_svm, + &Solver::calc_rho_nu_svm, + termCrit ); - if( k_fold < 2 ) - CV_ERROR( CV_StsBadArg, "Parameter must be > 1" ); + if( !solver.solve_generic( _si )) + return false; - CV_CALL(set_params( _params )); - svm_type = _params.svm_type; + double inv_r = 1./_si.r; - // All the parameters except, possibly, are positive. - // is nonnegative - if( C_grid.step <= 1 ) - { - C_grid.min_val = C_grid.max_val = params.C; - C_grid.step = 10; - } - else - CV_CALL(C_grid.check()); + for( int i = 0; i < sample_count; i++ ) + _alpha[i] *= _y[i]*inv_r; - if( gamma_grid.step <= 1 ) - { - gamma_grid.min_val = gamma_grid.max_val = params.gamma; - gamma_grid.step = 10; - } - else - CV_CALL(gamma_grid.check()); + _si.rho *= inv_r; + _si.obj *= (inv_r*inv_r); + _si.upper_bound_p = inv_r; + _si.upper_bound_n = inv_r; - if( p_grid.step <= 1 ) - { - p_grid.min_val = p_grid.max_val = params.p; - p_grid.step = 10; - } - else - CV_CALL(p_grid.check()); + return true; + } - if( nu_grid.step <= 1 ) - { - nu_grid.min_val = nu_grid.max_val = params.nu; - nu_grid.step = 10; - } - else - CV_CALL(nu_grid.check()); + static bool solve_one_class( const Mat& _samples, double nu, + const Ptr& _kernel, + vector& _alpha, SolutionInfo& _si, + TermCriteria termCrit ) + { + int sample_count = _samples.rows; + vector _y(sample_count, 1); + vector _b(sample_count, 0.); - if( coef_grid.step <= 1 ) - { - coef_grid.min_val = coef_grid.max_val = params.coef0; - coef_grid.step = 10; - } - else - CV_CALL(coef_grid.check()); + int i, n = cvRound( nu*sample_count ); - if( degree_grid.step <= 1 ) - { - degree_grid.min_val = degree_grid.max_val = params.degree; - degree_grid.step = 10; - } - else - CV_CALL(degree_grid.check()); - - // these parameters are not used: - if( params.kernel_type != CvSVM::POLY ) - degree_grid.min_val = degree_grid.max_val = params.degree; - if( params.kernel_type == CvSVM::LINEAR ) - gamma_grid.min_val = gamma_grid.max_val = params.gamma; - if( params.kernel_type != CvSVM::POLY && params.kernel_type != CvSVM::SIGMOID ) - coef_grid.min_val = coef_grid.max_val = params.coef0; - if( svm_type == CvSVM::NU_SVC || svm_type == CvSVM::ONE_CLASS ) - C_grid.min_val = C_grid.max_val = params.C; - if( svm_type == CvSVM::C_SVC || svm_type == CvSVM::EPS_SVR ) - nu_grid.min_val = nu_grid.max_val = params.nu; - if( svm_type != CvSVM::EPS_SVR ) - p_grid.min_val = p_grid.max_val = params.p; - - CV_ASSERT( g_step > 1 && degree_step > 1 && coef_step > 1); - CV_ASSERT( p_step > 1 && C_step > 1 && nu_step > 1 ); - - /* Prepare training data and related parameters */ - CV_CALL(cvPrepareTrainData( "CvSVM::train_auto", _train_data, CV_ROW_SAMPLE, - svm_type != CvSVM::ONE_CLASS ? _responses : 0, - svm_type == CvSVM::C_SVC || - svm_type == CvSVM::NU_SVC ? CV_VAR_CATEGORICAL : - CV_VAR_ORDERED, _var_idx, _sample_idx, - false, &samples, &sample_count, &var_count, &var_all, - &responses, &class_labels, &var_idx )); - - sample_size = var_count*sizeof(samples[0][0]); - - // make the storage block size large enough to fit all - // the temporary vectors and output support vectors. - block_size = MAX( block_size, sample_count*(int)sizeof(CvSVMKernelRow)); - block_size = MAX( block_size, sample_count*2*(int)sizeof(double) + 1024 ); - block_size = MAX( block_size, sample_size*2 + 1024 ); - - CV_CALL( storage = cvCreateMemStorage(block_size + sizeof(CvMemBlock) + sizeof(CvSeqBlock))); - CV_CALL(temp_storage = cvCreateChildMemStorage(storage)); - CV_CALL(alpha = (double*)cvMemStorageAlloc(temp_storage, sample_count*sizeof(double))); - - create_kernel(); - create_solver(); + _alpha.resize(sample_count); + for( i = 0; i < sample_count; i++ ) + _alpha[i] = i < n ? 1 : 0; - { - const int testset_size = sample_count/k_fold; - const int trainset_size = sample_count - testset_size; - const int last_testset_size = sample_count - testset_size*(k_fold-1); - const int last_trainset_size = sample_count - last_testset_size; - const bool is_regression = (svm_type == EPS_SVR) || (svm_type == NU_SVR); + if( n < sample_count ) + _alpha[n] = nu * sample_count - n; + else + _alpha[n-1] = nu * sample_count - (n-1); - size_t resp_elem_size = CV_ELEM_SIZE(responses->type); - size_t size = 2*last_trainset_size*sizeof(samples[0]); + Solver solver( _samples, _y, _alpha, _b, 1., 1., _kernel, + &Solver::get_row_one_class, + &Solver::select_working_set, + &Solver::calc_rho, + termCrit ); - samples_local = (const float**) cvAlloc( size ); - memset( samples_local, 0, size ); + return solver.solve_generic(_si); + } - responses_local = cvCreateMat( 1, trainset_size, CV_MAT_TYPE(responses->type) ); - cvZero( responses_local ); + static bool solve_eps_svr( const Mat& _samples, const vector& _yf, + double p, double C, const Ptr& _kernel, + vector& _alpha, SolutionInfo& _si, + TermCriteria termCrit ) + { + int sample_count = _samples.rows; + int alpha_count = sample_count*2; - // randomly permute samples and responses - for(int i = 0; i < sample_count; i++ ) - { - int i1 = (*rng)(sample_count); - int i2 = (*rng)(sample_count); - const float* temp; - float t; - int y; - - CV_SWAP( samples[i1], samples[i2], temp ); - if( is_regression ) - CV_SWAP( responses->data.fl[i1], responses->data.fl[i2], t ); - else - CV_SWAP( responses->data.i[i1], responses->data.i[i2], y ); - } + CV_Assert( (int)_yf.size() == sample_count ); - if (!is_regression && class_labels->cols==2 && balanced) - { - // count class samples - int num_0=0,num_1=0; - for (int i=0; idata.i[i]==class_labels->data.i[0]) - ++num_0; - else - ++num_1; - } + _alpha.assign(alpha_count, 0.); + vector _y(alpha_count); + vector _b(alpha_count); - int label_smallest_class; - int label_biggest_class; - if (num_0 < num_1) - { - label_biggest_class = class_labels->data.i[1]; - label_smallest_class = class_labels->data.i[0]; - } - else - { - label_biggest_class = class_labels->data.i[0]; - label_smallest_class = class_labels->data.i[1]; - int y; - CV_SWAP(num_0,num_1,y); - } - const double class_ratio = (double) num_0/sample_count; - // calculate class ratio of each fold - indexedratio *ratios=0; - ratios = (indexedratio*) cvAlloc(k_fold*sizeof(*ratios)); - for (int k=0, i_begin=0; kdata.i[i]==label_smallest_class) - ++count0; - else - ++count1; - } - ratios[k].ind = k; - ratios[k].count_smallest = count0; - ratios[k].count_biggest = count1; - ratios[k].eval(); - } - // initial distance - qsort(ratios, k_fold, sizeof(ratios[0]), icvCmpIndexedratio); - double old_dist = 0.0; - for (int k=0; k 0.0) - { - if (ratios[0].count_biggest==0 || ratios[k_fold-1].count_smallest==0) - break; // we are not able to swap samples anymore - // what if we swap the samples, calculate the new distance - ratios[0].count_smallest++; - ratios[0].count_biggest--; - ratios[0].eval(); - ratios[k_fold-1].count_smallest--; - ratios[k_fold-1].count_biggest++; - ratios[k_fold-1].eval(); - qsort(ratios, k_fold, sizeof(ratios[0]), icvCmpIndexedratio); - new_dist = 0.0; - for (int k=0; kdata.i[i1]==label_biggest_class) - break; - } - // index of the smallest_class sample from the maximum ratio fold - int i2 = ratios[k_fold-1].ind * testset_size; - for ( ; i2data.i[i2]==label_smallest_class) - break; - } - // swap - const float* temp; - int y; - CV_SWAP( samples[i1], samples[i2], temp ); - CV_SWAP( responses->data.i[i1], responses->data.i[i2], y ); - old_dist = new_dist; - } - else - break; // does not improve, so break the loop - } - cvFree(&ratios); - } + _b[i] = p - _yf[i]; + _y[i] = 1; - int* cls_lbls = class_labels ? class_labels->data.i : 0; - curr_c = C_grid.min_val; - do - { - params.C = curr_c; - gamma = gamma_grid.min_val; - do - { - params.gamma = gamma; - p = p_grid.min_val; - do - { - params.p = p; - nu = nu_grid.min_val; - do - { - params.nu = nu; - coef = coef_grid.min_val; - do - { - params.coef0 = coef; - degree = degree_grid.min_val; - do - { - params.degree = degree; - - float** test_samples_ptr = (float**)samples; - uchar* true_resp = responses->data.ptr; - int test_size = testset_size; - int train_size = trainset_size; - - error = 0; - for(int k = 0; k < k_fold; k++ ) - { - memcpy( samples_local, samples, sizeof(samples[0])*test_size*k ); - memcpy( samples_local + test_size*k, test_samples_ptr + test_size, - sizeof(samples[0])*(sample_count - testset_size*(k+1)) ); + _b[i+sample_count] = p + _yf[i]; + _y[i+sample_count] = -1; + } - memcpy( responses_local->data.ptr, responses->data.ptr, resp_elem_size*test_size*k ); - memcpy( responses_local->data.ptr + resp_elem_size*test_size*k, - true_resp + resp_elem_size*test_size, - resp_elem_size*(sample_count - testset_size*(k+1)) ); + Solver solver( _samples, _y, _alpha, _b, C, C, _kernel, + &Solver::get_row_svr, + &Solver::select_working_set, + &Solver::calc_rho, + termCrit ); - if( k == k_fold - 1 ) - { - test_size = last_testset_size; - train_size = last_trainset_size; - responses_local->cols = last_trainset_size; - } + if( !solver.solve_generic( _si )) + return false; - // Train SVM on samples - if( !do_train( svm_type, train_size, var_count, - (const float**)samples_local, responses_local, temp_storage, alpha ) ) - EXIT; + for( int i = 0; i < sample_count; i++ ) + _alpha[i] -= _alpha[i+sample_count]; - // Compute test set error on samples - for(int i = 0; i < test_size; i++, true_resp += resp_elem_size, test_samples_ptr++ ) - { - float resp = predict( *test_samples_ptr, var_count ); - error += is_regression ? powf( resp - *(float*)true_resp, 2 ) - : ((int)resp != cls_lbls[*(int*)true_resp]); - } - } - if( min_error > error ) - { - min_error = error; - best_degree = degree; - best_gamma = gamma; - best_coef = coef; - best_C = curr_c; - best_nu = nu; - best_p = p; - } - degree *= degree_grid.step; - } - while( degree < degree_grid.max_val ); - coef *= coef_grid.step; - } - while( coef < coef_grid.max_val ); - nu *= nu_grid.step; - } - while( nu < nu_grid.max_val ); - p *= p_grid.step; + return true; } - while( p < p_grid.max_val ); - gamma *= gamma_grid.step; - } - while( gamma < gamma_grid.max_val ); - curr_c *= C_grid.step; - } - while( curr_c < C_grid.max_val ); - } - - min_error /= (float) sample_count; - params.C = best_C; - params.nu = best_nu; - params.p = best_p; - params.gamma = best_gamma; - params.degree = best_degree; - params.coef0 = best_coef; - CV_CALL(ok = do_train( svm_type, sample_count, var_count, samples, responses, temp_storage, alpha )); - - __END__; + static bool solve_nu_svr( const Mat& _samples, const vector& _yf, + double nu, double C, const Ptr& _kernel, + vector& _alpha, SolutionInfo& _si, + TermCriteria termCrit ) + { + int sample_count = _samples.rows; + int alpha_count = sample_count*2; + double sum = C * nu * sample_count * 0.5; - delete solver; - solver = 0; - cvReleaseMemStorage( &temp_storage ); - cvReleaseMat( &responses ); - cvReleaseMat( &responses_local ); - cvFree( &samples ); - cvFree( &samples_local ); + CV_Assert( (int)_yf.size() == sample_count ); - if( cvGetErrStatus() < 0 || !ok ) - clear(); + _alpha.resize(alpha_count); + vector _y(alpha_count); + vector _b(alpha_count); - return ok; -} + for( int i = 0; i < sample_count; i++ ) + { + _alpha[i] = _alpha[i + sample_count] = std::min(sum, C); + sum -= _alpha[i]; -float CvSVM::predict( const float* row_sample, int row_len, bool returnDFVal ) const -{ - assert( kernel ); - assert( row_sample ); + _b[i] = -_yf[i]; + _y[i] = 1; - int var_count = get_var_count(); - assert( row_len == var_count ); - (void)row_len; + _b[i + sample_count] = _yf[i]; + _y[i + sample_count] = -1; + } - int class_count = class_labels ? class_labels->cols : - params.svm_type == ONE_CLASS ? 1 : 0; + Solver solver( _samples, _y, _alpha, _b, 1., 1., _kernel, + &Solver::get_row_svr, + &Solver::select_working_set_nu_svm, + &Solver::calc_rho_nu_svm, + termCrit ); - float result = 0; - cv::AutoBuffer _buffer(sv_total + (class_count+1)*2); - float* buffer = _buffer; + if( !solver.solve_generic( _si )) + return false; - if( params.svm_type == EPS_SVR || - params.svm_type == NU_SVR || - params.svm_type == ONE_CLASS ) - { - CvSVMDecisionFunc* df = (CvSVMDecisionFunc*)decision_func; - int i, sv_count = df->sv_count; - double sum = -df->rho; + for( int i = 0; i < sample_count; i++ ) + _alpha[i] -= _alpha[i+sample_count]; - kernel->calc( sv_count, var_count, (const float**)sv, row_sample, buffer ); - for( i = 0; i < sv_count; i++ ) - sum += buffer[i]*df->alpha[i]; + return true; + } - result = params.svm_type == ONE_CLASS ? (float)(sum > 0) : (float)sum; + int sample_count; + int var_count; + int cache_size; + int max_cache_size; + Mat samples; + SVM::Params params; + vector lru_cache; + int lru_first; + int lru_last; + Mat lru_cache_data; + + int alpha_count; + + vector G_vec; + vector* alpha_vec; + vector y_vec; + // -1 - lower bound, 0 - free, 1 - upper bound + vector alpha_status_vec; + vector b_vec; + + vector buf[2]; + double eps; + int max_iter; + double C[2]; // C[0] == Cn, C[1] == Cp + Ptr kernel; + + SelectWorkingSet select_working_set_func; + CalcRho calc_rho_func; + GetRow get_row_func; + }; + + ////////////////////////////////////////////////////////////////////////////////////////// + SVMImpl() + { + clear(); } - else if( params.svm_type == C_SVC || - params.svm_type == NU_SVC ) + + ~SVMImpl() { - CvSVMDecisionFunc* df = (CvSVMDecisionFunc*)decision_func; - int* vote = (int*)(buffer + sv_total); - int i, j, k; + clear(); + } - memset( vote, 0, class_count*sizeof(vote[0])); - kernel->calc( sv_total, var_count, (const float**)sv, row_sample, buffer ); - double sum = 0.; + void clear() + { + decision_func.clear(); + df_alpha.clear(); + df_index.clear(); + sv.release(); + } - for( i = 0; i < class_count; i++ ) - { - for( j = i+1; j < class_count; j++, df++ ) - { - sum = -df->rho; - int sv_count = df->sv_count; - for( k = 0; k < sv_count; k++ ) - sum += df->alpha[k]*buffer[df->sv_index[k]]; + Mat getSupportVectors() const + { + return sv; + } - vote[sum > 0 ? i : j]++; - } - } + void setParams( const Params& _params, const Ptr& _kernel ) + { + params = _params; - for( i = 1, k = 0; i < class_count; i++ ) - { - if( vote[i] > vote[k] ) - k = i; - } - result = returnDFVal && class_count == 2 ? (float)sum : (float)(class_labels->data.i[k]); - } - else - CV_Error( CV_StsBadArg, "INTERNAL ERROR: Unknown SVM type, " - "the SVM structure is probably corrupted" ); + int kernelType = params.kernelType; + int svmType = params.svmType; - return result; -} + if( kernelType != LINEAR && kernelType != POLY && + kernelType != SIGMOID && kernelType != RBF && + kernelType != INTER && kernelType != CHI2) + CV_Error( CV_StsBadArg, "Unknown/unsupported kernel type" ); -float CvSVM::predict( const CvMat* sample, bool returnDFVal ) const -{ - float result = 0; - float* row_sample = 0; + if( kernelType == LINEAR ) + params.gamma = 1; + else if( params.gamma <= 0 ) + CV_Error( CV_StsOutOfRange, "gamma parameter of the kernel must be positive" ); - CV_FUNCNAME( "CvSVM::predict" ); + if( kernelType != SIGMOID && kernelType != POLY ) + params.coef0 = 0; + else if( params.coef0 < 0 ) + CV_Error( CV_StsOutOfRange, "The kernel parameter must be positive or zero" ); - __BEGIN__; + if( kernelType != POLY ) + params.degree = 0; + else if( params.degree <= 0 ) + CV_Error( CV_StsOutOfRange, "The kernel parameter must be positive" ); - int class_count; + if( svmType != C_SVC && svmType != NU_SVC && + svmType != ONE_CLASS && svmType != EPS_SVR && + svmType != NU_SVR ) + CV_Error( CV_StsBadArg, "Unknown/unsupported SVM type" ); - if( !kernel ) - CV_ERROR( CV_StsBadArg, "The SVM should be trained first" ); + if( svmType == ONE_CLASS || svmType == NU_SVC ) + params.C = 0; + else if( params.C <= 0 ) + CV_Error( CV_StsOutOfRange, "The parameter C must be positive" ); - class_count = class_labels ? class_labels->cols : - params.svm_type == ONE_CLASS ? 1 : 0; + if( svmType == C_SVC || svmType == EPS_SVR ) + params.nu = 0; + else if( params.nu <= 0 || params.nu >= 1 ) + CV_Error( CV_StsOutOfRange, "The parameter nu must be between 0 and 1" ); - CV_CALL( cvPreparePredictData( sample, var_all, var_idx, - class_count, 0, &row_sample )); - result = predict( row_sample, get_var_count(), returnDFVal ); + if( svmType != EPS_SVR ) + params.p = 0; + else if( params.p <= 0 ) + CV_Error( CV_StsOutOfRange, "The parameter p must be positive" ); - __END__; + if( svmType != C_SVC ) + params.classWeights.release(); - if( sample && (!CV_IS_MAT(sample) || sample->data.fl != row_sample) ) - cvFree( &row_sample ); + termCrit = params.termCrit; + if( !(termCrit.type & TermCriteria::EPS) ) + termCrit.epsilon = DBL_EPSILON; + termCrit.epsilon = std::max(termCrit.epsilon, DBL_EPSILON); + if( !(termCrit.type & TermCriteria::COUNT) ) + termCrit.maxCount = INT_MAX; + termCrit.maxCount = std::max(termCrit.maxCount, 1); - return result; -} + if( _kernel ) + kernel = _kernel; + else + kernel = makePtr(params); + } -struct predict_body_svm : ParallelLoopBody { - predict_body_svm(const CvSVM* _pointer, float* _result, const CvMat* _samples, CvMat* _results, bool _returnDFVal) + Params getParams() const { - pointer = _pointer; - result = _result; - samples = _samples; - results = _results; - returnDFVal = _returnDFVal; + return params; } - const CvSVM* pointer; - float* result; - const CvMat* samples; - CvMat* results; - bool returnDFVal; - - void operator()( const cv::Range& range ) const + Ptr getKernel() const { - for(int i = range.start; i < range.end; i++ ) - { - CvMat sample; - cvGetRow( samples, &sample, i ); - int r = (int)pointer->predict(&sample, returnDFVal); - if (results) - results->data.fl[i] = (float)r; - if (i == 0) - *result = (float)r; - } + return kernel; } -}; -float CvSVM::predict(const CvMat* samples, CV_OUT CvMat* results, bool returnDFVal) const -{ - float result = 0; - cv::parallel_for_(cv::Range(0, samples->rows), - predict_body_svm(this, &result, samples, results, returnDFVal) - ); - return result; -} + int getSVCount(int i) const + { + return (i < (int)(decision_func.size()-1) ? decision_func[i+1].ofs : + (int)df_index.size()) - decision_func[i].ofs; + } -void CvSVM::predict( cv::InputArray _samples, cv::OutputArray _results ) const -{ - _results.create(_samples.size().height, 1, CV_32F); - CvMat samples = _samples.getMat(), results = _results.getMat(); - predict(&samples, &results); -} + bool do_train( const Mat& _samples, const Mat& _responses ) + { + int svmType = params.svmType; + int i, j, k, sample_count = _samples.rows; + vector _alpha; + Solver::SolutionInfo sinfo; -CvSVM::CvSVM( const Mat& _train_data, const Mat& _responses, - const Mat& _var_idx, const Mat& _sample_idx, CvSVMParams _params ) -{ - decision_func = 0; - class_labels = 0; - class_weights = 0; - storage = 0; - var_idx = 0; - kernel = 0; - solver = 0; - default_model_name = "my_svm"; - - train( _train_data, _responses, _var_idx, _sample_idx, _params ); -} + CV_Assert( _samples.type() == CV_32F ); + var_count = _samples.cols; -bool CvSVM::train( const Mat& _train_data, const Mat& _responses, - const Mat& _var_idx, const Mat& _sample_idx, CvSVMParams _params ) -{ - CvMat tdata = _train_data, responses = _responses, vidx = _var_idx, sidx = _sample_idx; - return train(&tdata, &responses, vidx.data.ptr ? &vidx : 0, sidx.data.ptr ? &sidx : 0, _params); -} + if( svmType == ONE_CLASS || svmType == EPS_SVR || svmType == NU_SVR ) + { + int sv_count = 0; + decision_func.clear(); + vector _yf; + if( !_responses.empty() ) + _responses.convertTo(_yf, CV_32F); -bool CvSVM::train_auto( const Mat& _train_data, const Mat& _responses, - const Mat& _var_idx, const Mat& _sample_idx, CvSVMParams _params, int k_fold, - CvParamGrid C_grid, CvParamGrid gamma_grid, CvParamGrid p_grid, - CvParamGrid nu_grid, CvParamGrid coef_grid, CvParamGrid degree_grid, bool balanced ) -{ - CvMat tdata = _train_data, responses = _responses, vidx = _var_idx, sidx = _sample_idx; - return train_auto(&tdata, &responses, vidx.data.ptr ? &vidx : 0, - sidx.data.ptr ? &sidx : 0, _params, k_fold, C_grid, gamma_grid, p_grid, - nu_grid, coef_grid, degree_grid, balanced); -} + bool ok = + (svmType == ONE_CLASS ? Solver::solve_one_class( _samples, params.nu, kernel, _alpha, sinfo, termCrit ) : + svmType == EPS_SVR ? Solver::solve_eps_svr( _samples, _yf, params.p, params.C, kernel, _alpha, sinfo, termCrit ) : + svmType == NU_SVR ? Solver::solve_nu_svr( _samples, _yf, params.nu, params.C, kernel, _alpha, sinfo, termCrit ) : false); -float CvSVM::predict( const Mat& _sample, bool returnDFVal ) const -{ - CvMat sample = _sample; - return predict(&sample, returnDFVal); -} + if( !ok ) + return false; + for( i = 0; i < sample_count; i++ ) + sv_count += fabs(_alpha[i]) > 0; -void CvSVM::write_params( CvFileStorage* fs ) const -{ - //CV_FUNCNAME( "CvSVM::write_params" ); - - __BEGIN__; - - int svm_type = params.svm_type; - int kernel_type = params.kernel_type; - - const char* svm_type_str = - svm_type == CvSVM::C_SVC ? "C_SVC" : - svm_type == CvSVM::NU_SVC ? "NU_SVC" : - svm_type == CvSVM::ONE_CLASS ? "ONE_CLASS" : - svm_type == CvSVM::EPS_SVR ? "EPS_SVR" : - svm_type == CvSVM::NU_SVR ? "NU_SVR" : 0; - const char* kernel_type_str = - kernel_type == CvSVM::LINEAR ? "LINEAR" : - kernel_type == CvSVM::POLY ? "POLY" : - kernel_type == CvSVM::RBF ? "RBF" : - kernel_type == CvSVM::SIGMOID ? "SIGMOID" : 0; - - if( svm_type_str ) - cvWriteString( fs, "svm_type", svm_type_str ); - else - cvWriteInt( fs, "svm_type", svm_type ); + CV_Assert(sv_count != 0); - // save kernel - cvStartWriteStruct( fs, "kernel", CV_NODE_MAP + CV_NODE_FLOW ); + sv.create(sv_count, _samples.cols, CV_32F); + df_alpha.resize(sv_count); + df_index.resize(sv_count); - if( kernel_type_str ) - cvWriteString( fs, "type", kernel_type_str ); - else - cvWriteInt( fs, "type", kernel_type ); + for( i = k = 0; i < sample_count; i++ ) + { + if( std::abs(_alpha[i]) > 0 ) + { + _samples.row(i).copyTo(sv.row(k)); + df_alpha[k] = _alpha[i]; + df_index[k] = k; + k++; + } + } - if( kernel_type == CvSVM::POLY || !kernel_type_str ) - cvWriteReal( fs, "degree", params.degree ); + decision_func.push_back(DecisionFunc(sinfo.rho, 0)); + } + else + { + int class_count = (int)class_labels.total(); + vector svidx, sidx, sidx_all, sv_tab(sample_count, 0); + Mat temp_samples, class_weights; + vector class_ranges; + vector temp_y; + double nu = params.nu; + CV_Assert( svmType == C_SVC || svmType == NU_SVC ); + + if( svmType == C_SVC && !params.classWeights.empty() ) + { + const Mat cw = params.classWeights; - if( kernel_type != CvSVM::LINEAR || !kernel_type_str ) - cvWriteReal( fs, "gamma", params.gamma ); + if( (cw.cols != 1 && cw.rows != 1) || + (int)cw.total() != class_count || + (cw.type() != CV_32F && cw.type() != CV_64F) ) + CV_Error( CV_StsBadArg, "params.class_weights must be 1d floating-point vector " + "containing as many elements as the number of classes" ); - if( kernel_type == CvSVM::POLY || kernel_type == CvSVM::SIGMOID || !kernel_type_str ) - cvWriteReal( fs, "coef0", params.coef0 ); + cw.convertTo(class_weights, CV_64F, params.C); + //normalize(cw, class_weights, params.C, 0, NORM_L1, CV_64F); + } - cvEndWriteStruct(fs); + decision_func.clear(); + df_alpha.clear(); + df_index.clear(); - if( svm_type == CvSVM::C_SVC || svm_type == CvSVM::EPS_SVR || - svm_type == CvSVM::NU_SVR || !svm_type_str ) - cvWriteReal( fs, "C", params.C ); + sortSamplesByClasses( _samples, _responses, sidx_all, class_ranges ); - if( svm_type == CvSVM::NU_SVC || svm_type == CvSVM::ONE_CLASS || - svm_type == CvSVM::NU_SVR || !svm_type_str ) - cvWriteReal( fs, "nu", params.nu ); + //check that while cross-validation there were the samples from all the classes + if( class_ranges[class_count] <= 0 ) + CV_Error( CV_StsBadArg, "While cross-validation one or more of the classes have " + "been fell out of the sample. Try to enlarge " ); - if( svm_type == CvSVM::EPS_SVR || !svm_type_str ) - cvWriteReal( fs, "p", params.p ); + if( svmType == NU_SVC ) + { + // check if nu is feasible + for( i = 0; i < class_count; i++ ) + { + int ci = class_ranges[i+1] - class_ranges[i]; + for( j = i+1; j< class_count; j++ ) + { + int cj = class_ranges[j+1] - class_ranges[j]; + if( nu*(ci + cj)*0.5 > std::min( ci, cj ) ) + // TODO: add some diagnostic + return false; + } + } + } - cvStartWriteStruct( fs, "term_criteria", CV_NODE_MAP + CV_NODE_FLOW ); - if( params.term_crit.type & CV_TERMCRIT_EPS ) - cvWriteReal( fs, "epsilon", params.term_crit.epsilon ); - if( params.term_crit.type & CV_TERMCRIT_ITER ) - cvWriteInt( fs, "iterations", params.term_crit.max_iter ); - cvEndWriteStruct( fs ); + size_t samplesize = _samples.cols*_samples.elemSize(); - __END__; -} + // train n*(n-1)/2 classifiers + for( i = 0; i < class_count; i++ ) + { + for( j = i+1; j < class_count; j++ ) + { + int si = class_ranges[i], ci = class_ranges[i+1] - si; + int sj = class_ranges[j], cj = class_ranges[j+1] - sj; + double Cp = params.C, Cn = Cp; + temp_samples.create(ci + cj, _samples.cols, _samples.type()); + sidx.resize(ci + cj); + temp_y.resize(ci + cj); -static bool isSvmModelApplicable(int sv_total, int var_all, int var_count, int class_count) -{ - return (sv_total > 0 && var_count > 0 && var_count <= var_all && class_count >= 0); -} + // form input for the binary classification problem + for( k = 0; k < ci+cj; k++ ) + { + int idx = k < ci ? si+k : sj+k-ci; + memcpy(temp_samples.ptr(k), _samples.ptr(sidx_all[idx]), samplesize); + sidx[k] = sidx_all[idx]; + temp_y[k] = k < ci ? 1 : -1; + } + if( !class_weights.empty() ) + { + Cp = class_weights.at(i); + Cn = class_weights.at(j); + } -void CvSVM::write( CvFileStorage* fs, const char* name ) const -{ - CV_FUNCNAME( "CvSVM::write" ); + DecisionFunc df; + bool ok = params.svmType == C_SVC ? + Solver::solve_c_svc( temp_samples, temp_y, Cp, Cn, + kernel, _alpha, sinfo, termCrit ) : + params.svmType == NU_SVC ? + Solver::solve_nu_svc( temp_samples, temp_y, params.nu, + kernel, _alpha, sinfo, termCrit ) : + false; + if( !ok ) + return false; + df.rho = sinfo.rho; + df.ofs = (int)df_index.size(); + decision_func.push_back(df); + + for( k = 0; k < ci + cj; k++ ) + { + if( std::abs(_alpha[k]) > 0 ) + { + int idx = k < ci ? si+k : sj+k-ci; + sv_tab[sidx_all[idx]] = 1; + df_index.push_back(sidx_all[idx]); + df_alpha.push_back(_alpha[k]); + } + } + } + } - __BEGIN__; + // allocate support vectors and initialize sv_tab + for( i = 0, k = 0; i < sample_count; i++ ) + { + if( sv_tab[i] ) + sv_tab[i] = ++k; + } - int i, var_count = get_var_count(), df_count; - int class_count = class_labels ? class_labels->cols : - params.svm_type == CvSVM::ONE_CLASS ? 1 : 0; - const CvSVMDecisionFunc* df = decision_func; - if( !isSvmModelApplicable(sv_total, var_all, var_count, class_count) ) - CV_ERROR( CV_StsParseError, "SVM model data is invalid, check sv_count, var_* and class_count tags" ); + int sv_total = k; + sv.create(sv_total, _samples.cols, _samples.type()); - cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_SVM ); + for( i = 0; i < sample_count; i++ ) + { + if( !sv_tab[i] ) + continue; + memcpy(sv.ptr(sv_tab[i]-1), _samples.ptr(i), samplesize); + } - write_params( fs ); + // set sv pointers + int n = (int)df_index.size(); + for( i = 0; i < n; i++ ) + { + CV_Assert( sv_tab[df_index[i]] > 0 ); + df_index[i] = sv_tab[df_index[i]] - 1; + } + } - cvWriteInt( fs, "var_all", var_all ); - cvWriteInt( fs, "var_count", var_count ); + optimize_linear_svm(); + return true; + } - if( class_count ) + void optimize_linear_svm() { - cvWriteInt( fs, "class_count", class_count ); + // we optimize only linear SVM: compress all the support vectors into one. + if( params.kernelType != LINEAR ) + return; - if( class_labels ) - cvWrite( fs, "class_labels", class_labels ); - - if( class_weights ) - cvWrite( fs, "class_weights", class_weights ); - } + int i, df_count = (int)decision_func.size(); - if( var_idx ) - cvWrite( fs, "var_idx", var_idx ); + for( i = 0; i < df_count; i++ ) + { + if( getSVCount(i) != 1 ) + break; + } - // write the joint collection of support vectors - cvWriteInt( fs, "sv_total", sv_total ); - cvStartWriteStruct( fs, "support_vectors", CV_NODE_SEQ ); - for( i = 0; i < sv_total; i++ ) - { - cvStartWriteStruct( fs, 0, CV_NODE_SEQ + CV_NODE_FLOW ); - cvWriteRawData( fs, sv[i], var_count, "f" ); - cvEndWriteStruct( fs ); - } + // if every decision functions uses a single support vector; + // it's already compressed. skip it then. + if( i == df_count ) + return; - cvEndWriteStruct( fs ); + AutoBuffer vbuf(var_count); + double* v = vbuf; + Mat new_sv(df_count, var_count, CV_32F); - // write decision functions - df_count = class_count > 1 ? class_count*(class_count-1)/2 : 1; - df = decision_func; + vector new_df; - cvStartWriteStruct( fs, "decision_functions", CV_NODE_SEQ ); - for( i = 0; i < df_count; i++ ) - { - int sv_count = df[i].sv_count; - cvStartWriteStruct( fs, 0, CV_NODE_MAP ); - cvWriteInt( fs, "sv_count", sv_count ); - cvWriteReal( fs, "rho", df[i].rho ); - cvStartWriteStruct( fs, "alpha", CV_NODE_SEQ+CV_NODE_FLOW ); - cvWriteRawData( fs, df[i].alpha, df[i].sv_count, "d" ); - cvEndWriteStruct( fs ); - if( class_count > 1 ) + for( i = 0; i < df_count; i++ ) { - cvStartWriteStruct( fs, "index", CV_NODE_SEQ+CV_NODE_FLOW ); - cvWriteRawData( fs, df[i].sv_index, df[i].sv_count, "i" ); - cvEndWriteStruct( fs ); + float* dst = new_sv.ptr(i); + memset(v, 0, var_count*sizeof(v[0])); + int j, k, sv_count = getSVCount(i); + const DecisionFunc& df = decision_func[i]; + const int* sv_index = &df_index[df.ofs]; + const double* sv_alpha = &df_alpha[df.ofs]; + for( j = 0; j < sv_count; j++ ) + { + const float* src = sv.ptr(sv_index[j]); + double a = sv_alpha[j]; + for( k = 0; k < var_count; k++ ) + v[k] += src[k]*a; + } + for( k = 0; k < var_count; k++ ) + dst[k] = (float)v[k]; + new_df.push_back(DecisionFunc(df.rho, i)); } - else - CV_ASSERT( sv_count == sv_total ); - cvEndWriteStruct( fs ); - } - cvEndWriteStruct( fs ); - cvEndWriteStruct( fs ); - - __END__; -} + setRangeVector(df_index, df_count); + df_alpha.assign(df_count, 1.); + std::swap(sv, new_sv); + std::swap(decision_func, new_df); + } -void CvSVM::read_params( CvFileStorage* fs, CvFileNode* svm_node ) -{ - CV_FUNCNAME( "CvSVM::read_params" ); + bool train( const Ptr& data, int ) + { + clear(); - __BEGIN__; + int svmType = params.svmType; + Mat samples = data->getTrainSamples(); + Mat responses; - int svm_type, kernel_type; - CvSVMParams _params; + if( svmType == C_SVC || svmType == NU_SVC ) + { + responses = data->getTrainNormCatResponses(); + if( responses.empty() ) + CV_Error(CV_StsBadArg, "in the case of classification problem the responses must be categorical; " + "either specify varType when creating TrainData, or pass integer responses"); + class_labels = data->getClassLabels(); + } + else + responses = data->getTrainResponses(); - CvFileNode* tmp_node = cvGetFileNodeByName( fs, svm_node, "svm_type" ); - CvFileNode* kernel_node; - if( !tmp_node ) - CV_ERROR( CV_StsBadArg, "svm_type tag is not found" ); + if( !do_train( samples, responses )) + { + clear(); + return false; + } - if( CV_NODE_TYPE(tmp_node->tag) == CV_NODE_INT ) - svm_type = cvReadInt( tmp_node, -1 ); - else - { - const char* svm_type_str = cvReadString( tmp_node, "" ); - svm_type = - strcmp( svm_type_str, "C_SVC" ) == 0 ? CvSVM::C_SVC : - strcmp( svm_type_str, "NU_SVC" ) == 0 ? CvSVM::NU_SVC : - strcmp( svm_type_str, "ONE_CLASS" ) == 0 ? CvSVM::ONE_CLASS : - strcmp( svm_type_str, "EPS_SVR" ) == 0 ? CvSVM::EPS_SVR : - strcmp( svm_type_str, "NU_SVR" ) == 0 ? CvSVM::NU_SVR : -1; - - if( svm_type < 0 ) - CV_ERROR( CV_StsParseError, "Missing of invalid SVM type" ); + return true; } - kernel_node = cvGetFileNodeByName( fs, svm_node, "kernel" ); - if( !kernel_node ) - CV_ERROR( CV_StsParseError, "SVM kernel tag is not found" ); + bool trainAuto( const Ptr& data, int k_fold, + ParamGrid C_grid, ParamGrid gamma_grid, ParamGrid p_grid, + ParamGrid nu_grid, ParamGrid coef_grid, ParamGrid degree_grid, + bool balanced ) + { + int svmType = params.svmType; + RNG rng((uint64)-1); - tmp_node = cvGetFileNodeByName( fs, kernel_node, "type" ); - if( !tmp_node ) - CV_ERROR( CV_StsParseError, "SVM kernel type tag is not found" ); + if( svmType == ONE_CLASS ) + // current implementation of "auto" svm does not support the 1-class case. + return train( data, 0 ); - if( CV_NODE_TYPE(tmp_node->tag) == CV_NODE_INT ) - kernel_type = cvReadInt( tmp_node, -1 ); - else - { - const char* kernel_type_str = cvReadString( tmp_node, "" ); - kernel_type = - strcmp( kernel_type_str, "LINEAR" ) == 0 ? CvSVM::LINEAR : - strcmp( kernel_type_str, "POLY" ) == 0 ? CvSVM::POLY : - strcmp( kernel_type_str, "RBF" ) == 0 ? CvSVM::RBF : - strcmp( kernel_type_str, "SIGMOID" ) == 0 ? CvSVM::SIGMOID : -1; - - if( kernel_type < 0 ) - CV_ERROR( CV_StsParseError, "Missing of invalid SVM kernel type" ); - } + clear(); - _params.svm_type = svm_type; - _params.kernel_type = kernel_type; - _params.degree = cvReadRealByName( fs, kernel_node, "degree", 0 ); - _params.gamma = cvReadRealByName( fs, kernel_node, "gamma", 0 ); - _params.coef0 = cvReadRealByName( fs, kernel_node, "coef0", 0 ); + CV_Assert( k_fold >= 2 ); + + // All the parameters except, possibly, are positive. + // is nonnegative + #define CHECK_GRID(grid, param) \ + if( grid.logStep <= 1 ) \ + { \ + grid.minVal = grid.maxVal = params.param; \ + grid.logStep = 10; \ + } \ + else \ + checkParamGrid(grid) + + CHECK_GRID(C_grid, C); + CHECK_GRID(gamma_grid, gamma); + CHECK_GRID(p_grid, p); + CHECK_GRID(nu_grid, nu); + CHECK_GRID(coef_grid, coef0); + CHECK_GRID(degree_grid, degree); + + // these parameters are not used: + if( params.kernelType != POLY ) + degree_grid.minVal = degree_grid.maxVal = params.degree; + if( params.kernelType == LINEAR ) + gamma_grid.minVal = gamma_grid.maxVal = params.gamma; + if( params.kernelType != POLY && params.kernelType != SIGMOID ) + coef_grid.minVal = coef_grid.maxVal = params.coef0; + if( svmType == NU_SVC || svmType == ONE_CLASS ) + C_grid.minVal = C_grid.maxVal = params.C; + if( svmType == C_SVC || svmType == EPS_SVR ) + nu_grid.minVal = nu_grid.maxVal = params.nu; + if( svmType != EPS_SVR ) + p_grid.minVal = p_grid.maxVal = params.p; + + Mat samples = data->getTrainSamples(); + Mat responses; + bool is_classification = false; + Mat class_labels0 = class_labels; + int class_count = (int)class_labels.total(); + + if( svmType == C_SVC || svmType == NU_SVC ) + { + responses = data->getTrainNormCatResponses(); + class_labels = data->getClassLabels(); + is_classification = true; - _params.C = cvReadRealByName( fs, svm_node, "C", 0 ); - _params.nu = cvReadRealByName( fs, svm_node, "nu", 0 ); - _params.p = cvReadRealByName( fs, svm_node, "p", 0 ); - _params.class_weights = 0; + vector temp_class_labels; + setRangeVector(temp_class_labels, class_count); - tmp_node = cvGetFileNodeByName( fs, svm_node, "term_criteria" ); - if( tmp_node ) - { - _params.term_crit.epsilon = cvReadRealByName( fs, tmp_node, "epsilon", -1. ); - _params.term_crit.max_iter = cvReadIntByName( fs, tmp_node, "iterations", -1 ); - _params.term_crit.type = (_params.term_crit.epsilon >= 0 ? CV_TERMCRIT_EPS : 0) + - (_params.term_crit.max_iter >= 0 ? CV_TERMCRIT_ITER : 0); - } - else - _params.term_crit = cvTermCriteria( CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, 1000, FLT_EPSILON ); + // temporarily replace class labels with 0, 1, ..., NCLASSES-1 + Mat(temp_class_labels).copyTo(class_labels); + } + else + responses = data->getTrainResponses(); - set_params( _params ); + CV_Assert(samples.type() == CV_32F); - __END__; -} + int sample_count = samples.rows; + var_count = samples.cols; + size_t sample_size = var_count*samples.elemSize(); -void CvSVM::read( CvFileStorage* fs, CvFileNode* svm_node ) -{ - const double not_found_dbl = DBL_MAX; + vector sidx; + setRangeVector(sidx, sample_count); - CV_FUNCNAME( "CvSVM::read" ); + int i, j, k; - __BEGIN__; + // randomly permute training samples + for( i = 0; i < sample_count; i++ ) + { + int i1 = rng.uniform(0, sample_count); + int i2 = rng.uniform(0, sample_count); + std::swap(sidx[i1], sidx[i2]); + } - int i, var_count, df_count, class_count; - int block_size = 1 << 16, sv_size; - CvFileNode *sv_node, *df_node; - CvSVMDecisionFunc* df; - CvSeqReader reader; + if( is_classification && class_count == 2 && balanced ) + { + // reshuffle the training set in such a way that + // instances of each class are divided more or less evenly + // between the k_fold parts. + vector sidx0, sidx1; - if( !svm_node ) - CV_ERROR( CV_StsParseError, "The requested element is not found" ); + for( i = 0; i < sample_count; i++ ) + { + if( responses.at(sidx[i]) == 0 ) + sidx0.push_back(sidx[i]); + else + sidx1.push_back(sidx[i]); + } - clear(); + int n0 = (int)sidx0.size(), n1 = (int)sidx1.size(); + int a0 = 0, a1 = 0; + sidx.clear(); + for( k = 0; k < k_fold; k++ ) + { + int b0 = ((k+1)*n0 + k_fold/2)/k_fold, b1 = ((k+1)*n1 + k_fold/2)/k_fold; + int a = (int)sidx.size(), b = a + (b0 - a0) + (b1 - a1); + for( i = a0; i < b0; i++ ) + sidx.push_back(sidx0[i]); + for( i = a1; i < b1; i++ ) + sidx.push_back(sidx1[i]); + for( i = 0; i < (b - a); i++ ) + { + int i1 = rng.uniform(a, b); + int i2 = rng.uniform(a, b); + std::swap(sidx[i1], sidx[i2]); + } + a0 = b0; a1 = b1; + } + } - // read SVM parameters - read_params( fs, svm_node ); + int test_sample_count = (sample_count + k_fold/2)/k_fold; + int train_sample_count = sample_count - test_sample_count; - // and top-level data - sv_total = cvReadIntByName( fs, svm_node, "sv_total", -1 ); - var_all = cvReadIntByName( fs, svm_node, "var_all", -1 ); - var_count = cvReadIntByName( fs, svm_node, "var_count", var_all ); - class_count = cvReadIntByName( fs, svm_node, "class_count", 0 ); + Params best_params = params; + double min_error = FLT_MAX; - if( !isSvmModelApplicable(sv_total, var_all, var_count, class_count) ) - CV_ERROR( CV_StsParseError, "SVM model data is invalid, check sv_count, var_* and class_count tags" ); + int rtype = responses.type(); - CV_CALL( class_labels = (CvMat*)cvReadByName( fs, svm_node, "class_labels" )); - CV_CALL( class_weights = (CvMat*)cvReadByName( fs, svm_node, "class_weights" )); - CV_CALL( var_idx = (CvMat*)cvReadByName( fs, svm_node, "var_idx" )); + Mat temp_train_samples(train_sample_count, var_count, CV_32F); + Mat temp_test_samples(test_sample_count, var_count, CV_32F); + Mat temp_train_responses(train_sample_count, 1, rtype); + Mat temp_test_responses; - if( class_count > 1 && (!class_labels || - !CV_IS_MAT(class_labels) || class_labels->cols != class_count)) - CV_ERROR( CV_StsParseError, "Array of class labels is missing or invalid" ); + #define FOR_IN_GRID(var, grid) \ + for( params.var = grid.minVal; params.var == grid.minVal || params.var < grid.maxVal; params.var *= grid.logStep ) - if( var_count < var_all && (!var_idx || !CV_IS_MAT(var_idx) || var_idx->cols != var_count) ) - CV_ERROR( CV_StsParseError, "var_idx array is missing or invalid" ); + FOR_IN_GRID(C, C_grid) + FOR_IN_GRID(gamma, gamma_grid) + FOR_IN_GRID(p, p_grid) + FOR_IN_GRID(nu, nu_grid) + FOR_IN_GRID(coef0, coef_grid) + FOR_IN_GRID(degree, degree_grid) + { + double error = 0; + for( k = 0; k < k_fold; k++ ) + { + int start = (k*sample_count + k_fold/2)/k_fold; + for( i = 0; i < train_sample_count; i++ ) + { + j = sidx[(i+start)%sample_count]; + memcpy(temp_train_samples.ptr(i), samples.ptr(j), sample_size); + if( is_classification ) + temp_train_responses.at(i) = responses.at(j); + else if( !responses.empty() ) + temp_train_responses.at(i) = responses.at(j); + } - // read support vectors - sv_node = cvGetFileNodeByName( fs, svm_node, "support_vectors" ); - if( !sv_node || !CV_NODE_IS_SEQ(sv_node->tag)) - CV_ERROR( CV_StsParseError, "Missing or invalid sequence of support vectors" ); + // Train SVM on samples + if( !do_train( temp_train_samples, temp_train_responses )) + continue; - block_size = MAX( block_size, sv_total*(int)sizeof(CvSVMKernelRow)); - block_size = MAX( block_size, sv_total*2*(int)sizeof(double)); - block_size = MAX( block_size, var_all*(int)sizeof(double)); + for( i = 0; i < test_sample_count; i++ ) + { + j = sidx[(i+start+train_sample_count) % sample_count]; + memcpy(temp_train_samples.ptr(i), samples.ptr(j), sample_size); + } - CV_CALL( storage = cvCreateMemStorage(block_size + sizeof(CvMemBlock) + sizeof(CvSeqBlock))); - CV_CALL( sv = (float**)cvMemStorageAlloc( storage, - sv_total*sizeof(sv[0]) )); + predict(temp_test_samples, temp_test_responses, 0); + for( i = 0; i < test_sample_count; i++ ) + { + float val = temp_test_responses.at(i); + j = sidx[(i+start+train_sample_count) % sample_count]; + if( is_classification ) + error += (float)(val != responses.at(j)); + else + { + val -= responses.at(j); + error += val*val; + } + } + } + if( min_error > error ) + { + min_error = error; + best_params = params; + } + } - CV_CALL( cvStartReadSeq( sv_node->data.seq, &reader, 0 )); - sv_size = var_count*sizeof(sv[0][0]); + params = best_params; + class_labels = class_labels0; + return do_train( samples, responses ); + } - for( i = 0; i < sv_total; i++ ) + struct PredictBody : ParallelLoopBody { - CvFileNode* sv_elem = (CvFileNode*)reader.ptr; - CV_ASSERT( var_count == 1 || (CV_NODE_IS_SEQ(sv_elem->tag) && - sv_elem->data.seq->total == var_count) ); + PredictBody( const SVMImpl* _svm, const Mat& _samples, Mat& _results, bool _returnDFVal ) + { + svm = _svm; + results = &_results; + samples = &_samples; + returnDFVal = _returnDFVal; + } - CV_CALL( sv[i] = (float*)cvMemStorageAlloc( storage, sv_size )); - CV_CALL( cvReadRawData( fs, sv_elem, sv[i], "f" )); - CV_NEXT_SEQ_ELEM( sv_node->data.seq->elem_size, reader ); - } + void operator()( const Range& range ) const + { + int svmType = svm->params.svmType; + int sv_total = svm->sv.rows; + int class_count = !svm->class_labels.empty() ? (int)svm->class_labels.total() : svmType == ONE_CLASS ? 1 : 0; - // read decision functions - df_count = class_count > 1 ? class_count*(class_count-1)/2 : 1; - df_node = cvGetFileNodeByName( fs, svm_node, "decision_functions" ); - if( !df_node || !CV_NODE_IS_SEQ(df_node->tag) || - df_node->data.seq->total != df_count ) - CV_ERROR( CV_StsParseError, "decision_functions is missing or is not a collection " - "or has a wrong number of elements" ); + AutoBuffer _buffer(sv_total + (class_count+1)*2); + float* buffer = _buffer; - CV_CALL( df = decision_func = (CvSVMDecisionFunc*)cvAlloc( df_count*sizeof(df[0]) )); - cvStartReadSeq( df_node->data.seq, &reader, 0 ); + int i, j, dfi, k, si; - for( i = 0; i < df_count; i++ ) - { - CvFileNode* df_elem = (CvFileNode*)reader.ptr; - CvFileNode* alpha_node = cvGetFileNodeByName( fs, df_elem, "alpha" ); + if( svmType == EPS_SVR || svmType == NU_SVR || svmType == ONE_CLASS ) + { + for( si = range.start; si < range.end; si++ ) + { + const float* row_sample = samples->ptr(si); + svm->kernel->calc( sv_total, svm->var_count, svm->sv.ptr(), row_sample, buffer ); + + const SVMImpl::DecisionFunc* df = &svm->decision_func[0]; + double sum = -df->rho; + for( i = 0; i < sv_total; i++ ) + sum += buffer[i]*svm->df_alpha[i]; + float result = svm->params.svmType == ONE_CLASS && !returnDFVal ? (float)(sum > 0) : (float)sum; + results->at(si) = result; + } + } + else if( svmType == C_SVC || svmType == NU_SVC ) + { + int* vote = (int*)(buffer + sv_total); + + for( si = range.start; si < range.end; si++ ) + { + svm->kernel->calc( sv_total, svm->var_count, svm->sv.ptr(), + samples->ptr(si), buffer ); + double sum = 0.; + + memset( vote, 0, class_count*sizeof(vote[0])); + + for( i = dfi = 0; i < class_count; i++ ) + { + for( j = i+1; j < class_count; j++, dfi++ ) + { + const DecisionFunc& df = svm->decision_func[dfi]; + sum = -df.rho; + int sv_count = svm->getSVCount(dfi); + const double* alpha = &svm->df_alpha[df.ofs]; + const int* sv_index = &svm->df_index[df.ofs]; + for( k = 0; k < sv_count; k++ ) + sum += alpha[k]*buffer[sv_index[k]]; + + vote[sum > 0 ? i : j]++; + } + } - int sv_count = cvReadIntByName( fs, df_elem, "sv_count", -1 ); - if( sv_count <= 0 ) - CV_ERROR( CV_StsParseError, "sv_count is missing or non-positive" ); - df[i].sv_count = sv_count; + for( i = 1, k = 0; i < class_count; i++ ) + { + if( vote[i] > vote[k] ) + k = i; + } + float result = returnDFVal && class_count == 2 ? + (float)sum : (float)(svm->class_labels.at(k)); + results->at(si) = result; + } + } + else + CV_Error( CV_StsBadArg, "INTERNAL ERROR: Unknown SVM type, " + "the SVM structure is probably corrupted" ); + } - df[i].rho = cvReadRealByName( fs, df_elem, "rho", not_found_dbl ); - if( fabs(df[i].rho - not_found_dbl) < DBL_EPSILON ) - CV_ERROR( CV_StsParseError, "rho is missing" ); + const SVMImpl* svm; + const Mat* samples; + Mat* results; + bool returnDFVal; + }; - if( !alpha_node ) - CV_ERROR( CV_StsParseError, "alpha is missing in the decision function" ); + float predict( InputArray _samples, OutputArray _results, int flags ) const + { + float result = 0; + Mat samples = _samples.getMat(), results; + int nsamples = samples.rows; + bool returnDFVal = (flags & RAW_OUTPUT) != 0; - CV_CALL( df[i].alpha = (double*)cvMemStorageAlloc( storage, - sv_count*sizeof(df[i].alpha[0]))); - CV_ASSERT( sv_count == 1 || (CV_NODE_IS_SEQ(alpha_node->tag) && - alpha_node->data.seq->total == sv_count) ); - CV_CALL( cvReadRawData( fs, alpha_node, df[i].alpha, "d" )); + CV_Assert( samples.cols == var_count && samples.type() == CV_32F ); - if( class_count > 1 ) + if( _results.needed() ) { - CvFileNode* index_node = cvGetFileNodeByName( fs, df_elem, "index" ); - if( !index_node ) - CV_ERROR( CV_StsParseError, "index is missing in the decision function" ); - CV_CALL( df[i].sv_index = (int*)cvMemStorageAlloc( storage, - sv_count*sizeof(df[i].sv_index[0]))); - CV_ASSERT( sv_count == 1 || (CV_NODE_IS_SEQ(index_node->tag) && - index_node->data.seq->total == sv_count) ); - CV_CALL( cvReadRawData( fs, index_node, df[i].sv_index, "i" )); + _results.create( nsamples, 1, samples.type() ); + results = _results.getMat(); } else - df[i].sv_index = 0; + { + CV_Assert( nsamples == 1 ); + results = Mat(1, 1, CV_32F, &result); + } - CV_NEXT_SEQ_ELEM( df_node->data.seq->elem_size, reader ); + PredictBody invoker(this, samples, results, returnDFVal); + if( nsamples < 10 ) + invoker(Range(0, nsamples)); + else + parallel_for_(Range(0, nsamples), invoker); + return result; } - if( cvReadIntByName(fs, svm_node, "optimize_linear", 1) != 0 ) - optimize_linear_svm(); - create_kernel(); + double getDecisionFunction(int i, OutputArray _alpha, OutputArray _svidx ) const + { + CV_Assert( 0 <= i && i < (int)decision_func.size()); + const DecisionFunc& df = decision_func[i]; + int count = getSVCount(i); + Mat(1, count, CV_64F, (double*)&df_alpha[df.ofs]).copyTo(_alpha); + Mat(1, count, CV_32S, (int*)&df_index[df.ofs]).copyTo(_svidx); + return df.rho; + } - __END__; -} + void write_params( FileStorage& fs ) const + { + int svmType = params.svmType; + int kernelType = params.kernelType; -#if 0 + String svm_type_str = + svmType == C_SVC ? "C_SVC" : + svmType == NU_SVC ? "NU_SVC" : + svmType == ONE_CLASS ? "ONE_CLASS" : + svmType == EPS_SVR ? "EPS_SVR" : + svmType == NU_SVR ? "NU_SVR" : format("Uknown_%d", svmType); + String kernel_type_str = + kernelType == LINEAR ? "LINEAR" : + kernelType == POLY ? "POLY" : + kernelType == RBF ? "RBF" : + kernelType == SIGMOID ? "SIGMOID" : format("Unknown_%d", kernelType); -static void* -icvCloneSVM( const void* _src ) -{ - CvSVMModel* dst = 0; + fs << "svmType" << svm_type_str; - CV_FUNCNAME( "icvCloneSVM" ); + // save kernel + fs << "kernel" << "{" << "type" << kernel_type_str; - __BEGIN__; + if( kernelType == POLY ) + fs << "degree" << params.degree; - const CvSVMModel* src = (const CvSVMModel*)_src; - int var_count, class_count; - int i, sv_total, df_count; - int sv_size; + if( kernelType != LINEAR ) + fs << "gamma" << params.gamma; - if( !CV_IS_SVM(src) ) - CV_ERROR( !src ? CV_StsNullPtr : CV_StsBadArg, "Input pointer is NULL or invalid" ); + if( kernelType == POLY || kernelType == SIGMOID ) + fs << "coef0" << params.coef0; - // 0. create initial CvSVMModel structure - CV_CALL( dst = icvCreateSVM() ); - dst->params = src->params; - dst->params.weight_labels = 0; - dst->params.weights = 0; + fs << "}"; - dst->var_all = src->var_all; - if( src->class_labels ) - dst->class_labels = cvCloneMat( src->class_labels ); - if( src->class_weights ) - dst->class_weights = cvCloneMat( src->class_weights ); - if( src->comp_idx ) - dst->comp_idx = cvCloneMat( src->comp_idx ); + if( svmType == C_SVC || svmType == EPS_SVR || svmType == NU_SVR ) + fs << "C" << params.C; - var_count = src->comp_idx ? src->comp_idx->cols : src->var_all; - class_count = src->class_labels ? src->class_labels->cols : - src->params.svm_type == CvSVM::ONE_CLASS ? 1 : 0; - sv_total = dst->sv_total = src->sv_total; - CV_CALL( dst->storage = cvCreateMemStorage( src->storage->block_size )); - CV_CALL( dst->sv = (float**)cvMemStorageAlloc( dst->storage, - sv_total*sizeof(dst->sv[0]) )); + if( svmType == NU_SVC || svmType == ONE_CLASS || svmType == NU_SVR ) + fs << "nu" << params.nu; - sv_size = var_count*sizeof(dst->sv[0][0]); + if( svmType == EPS_SVR ) + fs << "p" << params.p; + + fs << "term_criteria" << "{:"; + if( params.termCrit.type & TermCriteria::EPS ) + fs << "epsilon" << params.termCrit.epsilon; + if( params.termCrit.type & TermCriteria::COUNT ) + fs << "iterations" << params.termCrit.maxCount; + fs << "}"; + } - for( i = 0; i < sv_total; i++ ) + bool isTrained() const { - CV_CALL( dst->sv[i] = (float*)cvMemStorageAlloc( dst->storage, sv_size )); - memcpy( dst->sv[i], src->sv[i], sv_size ); + return !sv.empty(); } - df_count = class_count > 1 ? class_count*(class_count-1)/2 : 1; + bool isClassifier() const + { + return params.svmType == C_SVC || params.svmType == NU_SVC || params.svmType == ONE_CLASS; + } - CV_CALL( dst->decision_func = cvAlloc( df_count*sizeof(CvSVMDecisionFunc) )); + int getVarCount() const + { + return var_count; + } - for( i = 0; i < df_count; i++ ) + String getDefaultModelName() const { - const CvSVMDecisionFunc *sdf = - (const CvSVMDecisionFunc*)src->decision_func+i; - CvSVMDecisionFunc *ddf = - (CvSVMDecisionFunc*)dst->decision_func+i; - int sv_count = sdf->sv_count; - ddf->sv_count = sv_count; - ddf->rho = sdf->rho; - CV_CALL( ddf->alpha = (double*)cvMemStorageAlloc( dst->storage, - sv_count*sizeof(ddf->alpha[0]))); - memcpy( ddf->alpha, sdf->alpha, sv_count*sizeof(ddf->alpha[0])); - - if( class_count > 1 ) - { - CV_CALL( ddf->sv_index = (int*)cvMemStorageAlloc( dst->storage, - sv_count*sizeof(ddf->sv_index[0]))); - memcpy( ddf->sv_index, sdf->sv_index, sv_count*sizeof(ddf->sv_index[0])); - } - else - ddf->sv_index = 0; + return "opencv_ml_svm"; } - __END__; + void write( FileStorage& fs ) const + { + int class_count = !class_labels.empty() ? (int)class_labels.total() : + params.svmType == ONE_CLASS ? 1 : 0; + if( !isTrained() ) + CV_Error( CV_StsParseError, "SVM model data is invalid, check sv_count, var_* and class_count tags" ); - if( cvGetErrStatus() < 0 && dst ) - icvReleaseSVM( &dst ); + write_params( fs ); - return dst; -} + fs << "var_count" << var_count; -static int icvRegisterSVMType() -{ - CvTypeInfo info; - memset( &info, 0, sizeof(info) ); - - info.flags = 0; - info.header_size = sizeof( info ); - info.is_instance = icvIsSVM; - info.release = (CvReleaseFunc)icvReleaseSVM; - info.read = icvReadSVM; - info.write = icvWriteSVM; - info.clone = icvCloneSVM; - info.type_name = CV_TYPE_NAME_ML_SVM; - cvRegisterType( &info ); - - return 1; -} + if( class_count > 0 ) + { + fs << "class_count" << class_count; + if( !class_labels.empty() ) + fs << "class_labels" << class_labels; -static int svm = icvRegisterSVMType(); - -/* The function trains SVM model with optimal parameters, obtained by using cross-validation. -The parameters to be estimated should be indicated by setting theirs values to FLT_MAX. -The optimal parameters are saved in */ -CV_IMPL CvStatModel* -cvTrainSVM_CrossValidation( const CvMat* train_data, int tflag, - const CvMat* responses, - CvStatModelParams* model_params, - const CvStatModelParams* cross_valid_params, - const CvMat* comp_idx, - const CvMat* sample_idx, - const CvParamGrid* degree_grid, - const CvParamGrid* gamma_grid, - const CvParamGrid* coef_grid, - const CvParamGrid* C_grid, - const CvParamGrid* nu_grid, - const CvParamGrid* p_grid ) -{ - CvStatModel* svm = 0; - - CV_FUNCNAME("cvTainSVMCrossValidation"); - __BEGIN__; - - double degree_step = 7, - g_step = 15, - coef_step = 14, - C_step = 20, - nu_step = 5, - p_step = 7; // all steps must be > 1 - double degree_begin = 0.01, degree_end = 2; - double g_begin = 1e-5, g_end = 0.5; - double coef_begin = 0.1, coef_end = 300; - double C_begin = 0.1, C_end = 6000; - double nu_begin = 0.01, nu_end = 0.4; - double p_begin = 0.01, p_end = 100; - - double rate = 0, gamma = 0, C = 0, degree = 0, coef = 0, p = 0, nu = 0; - - double best_rate = 0; - double best_degree = degree_begin; - double best_gamma = g_begin; - double best_coef = coef_begin; - double best_C = C_begin; - double best_nu = nu_begin; - double best_p = p_begin; - - CvSVMModelParams svm_params, *psvm_params; - CvCrossValidationParams* cv_params = (CvCrossValidationParams*)cross_valid_params; - int svm_type, kernel; - int is_regression; - - if( !model_params ) - CV_ERROR( CV_StsBadArg, "" ); - if( !cv_params ) - CV_ERROR( CV_StsBadArg, "" ); - - svm_params = *(CvSVMModelParams*)model_params; - psvm_params = (CvSVMModelParams*)model_params; - svm_type = svm_params.svm_type; - kernel = svm_params.kernel_type; - - svm_params.degree = svm_params.degree > 0 ? svm_params.degree : 1; - svm_params.gamma = svm_params.gamma > 0 ? svm_params.gamma : 1; - svm_params.coef0 = svm_params.coef0 > 0 ? svm_params.coef0 : 1e-6; - svm_params.C = svm_params.C > 0 ? svm_params.C : 1; - svm_params.nu = svm_params.nu > 0 ? svm_params.nu : 1; - svm_params.p = svm_params.p > 0 ? svm_params.p : 1; - - if( degree_grid ) - { - if( !(degree_grid->max_val == 0 && degree_grid->min_val == 0 && - degree_grid->step == 0) ) - { - if( degree_grid->min_val > degree_grid->max_val ) - CV_ERROR( CV_StsBadArg, - "low bound of grid should be less then the upper one"); - if( degree_grid->step <= 1 ) - CV_ERROR( CV_StsBadArg, "grid step should be greater 1" ); - degree_begin = degree_grid->min_val; - degree_end = degree_grid->max_val; - degree_step = degree_grid->step; + if( !params.classWeights.empty() ) + fs << "class_weights" << params.classWeights; } - } - else - degree_begin = degree_end = svm_params.degree; - if( gamma_grid ) - { - if( !(gamma_grid->max_val == 0 && gamma_grid->min_val == 0 && - gamma_grid->step == 0) ) + // write the joint collection of support vectors + int i, sv_total = sv.rows; + fs << "sv_total" << sv_total; + fs << "support_vectors" << "["; + for( i = 0; i < sv_total; i++ ) { - if( gamma_grid->min_val > gamma_grid->max_val ) - CV_ERROR( CV_StsBadArg, - "low bound of grid should be less then the upper one"); - if( gamma_grid->step <= 1 ) - CV_ERROR( CV_StsBadArg, "grid step should be greater 1" ); - g_begin = gamma_grid->min_val; - g_end = gamma_grid->max_val; - g_step = gamma_grid->step; + fs << "[:"; + fs.writeRaw("f", sv.ptr(i), sv.cols*sv.elemSize()); + fs << "]"; } - } - else - g_begin = g_end = svm_params.gamma; + fs << "]"; - if( coef_grid ) - { - if( !(coef_grid->max_val == 0 && coef_grid->min_val == 0 && - coef_grid->step == 0) ) - { - if( coef_grid->min_val > coef_grid->max_val ) - CV_ERROR( CV_StsBadArg, - "low bound of grid should be less then the upper one"); - if( coef_grid->step <= 1 ) - CV_ERROR( CV_StsBadArg, "grid step should be greater 1" ); - coef_begin = coef_grid->min_val; - coef_end = coef_grid->max_val; - coef_step = coef_grid->step; - } - } - else - coef_begin = coef_end = svm_params.coef0; + // write decision functions + int df_count = (int)decision_func.size(); - if( C_grid ) - { - if( !(C_grid->max_val == 0 && C_grid->min_val == 0 && C_grid->step == 0)) + fs << "decision_functions" << "["; + for( i = 0; i < df_count; i++ ) { - if( C_grid->min_val > C_grid->max_val ) - CV_ERROR( CV_StsBadArg, - "low bound of grid should be less then the upper one"); - if( C_grid->step <= 1 ) - CV_ERROR( CV_StsBadArg, "grid step should be greater 1" ); - C_begin = C_grid->min_val; - C_end = C_grid->max_val; - C_step = C_grid->step; + const DecisionFunc& df = decision_func[i]; + int sv_count = getSVCount(i); + fs << "{" << "sv_count" << sv_count + << "rho" << df.rho + << "alpha" << "[:"; + fs.writeRaw("d", (const uchar*)&df_alpha[df.ofs], sv_count*sizeof(df_alpha[0])); + fs << "]"; + if( class_count > 2 ) + { + fs << "index" << "[:"; + fs.writeRaw("i", (const uchar*)&df_index[df.ofs], sv_count*sizeof(df_index[0])); + fs << "]"; + } + else + CV_Assert( sv_count == sv_total ); + fs << "}"; } + fs << "]"; } - else - C_begin = C_end = svm_params.C; - if( nu_grid ) + void read_params( const FileNode& fn ) { - if(!(nu_grid->max_val == 0 && nu_grid->min_val == 0 && nu_grid->step==0)) - { - if( nu_grid->min_val > nu_grid->max_val ) - CV_ERROR( CV_StsBadArg, - "low bound of grid should be less then the upper one"); - if( nu_grid->step <= 1 ) - CV_ERROR( CV_StsBadArg, "grid step should be greater 1" ); - nu_begin = nu_grid->min_val; - nu_end = nu_grid->max_val; - nu_step = nu_grid->step; - } - } - else - nu_begin = nu_end = svm_params.nu; + Params _params; - if( p_grid ) - { - if( !(p_grid->max_val == 0 && p_grid->min_val == 0 && p_grid->step == 0)) + String svm_type_str = (String)fn["svmType"]; + int svmType = + svm_type_str == "C_SVC" ? C_SVC : + svm_type_str == "NU_SVC" ? NU_SVC : + svm_type_str == "ONE_CLASS" ? ONE_CLASS : + svm_type_str == "EPS_SVR" ? EPS_SVR : + svm_type_str == "NU_SVR" ? NU_SVR : -1; + + if( svmType < 0 ) + CV_Error( CV_StsParseError, "Missing of invalid SVM type" ); + + FileNode kernel_node = fn["kernel"]; + if( kernel_node.empty() ) + CV_Error( CV_StsParseError, "SVM kernel tag is not found" ); + + String kernel_type_str = (String)kernel_node["type"]; + int kernelType = + kernel_type_str == "LINEAR" ? LINEAR : + kernel_type_str == "POLY" ? POLY : + kernel_type_str == "RBF" ? RBF : + kernel_type_str == "SIGMOID" ? SIGMOID : -1; + + if( kernelType < 0 ) + CV_Error( CV_StsParseError, "Missing of invalid SVM kernel type" ); + + _params.svmType = svmType; + _params.kernelType = kernelType; + _params.degree = (double)kernel_node["degree"]; + _params.gamma = (double)kernel_node["gamma"]; + _params.coef0 = (double)kernel_node["coef0"]; + + _params.C = (double)fn["C"]; + _params.nu = (double)fn["nu"]; + _params.p = (double)fn["p"]; + _params.classWeights = Mat(); + + FileNode tcnode = fn["term_criteria"]; + if( !tcnode.empty() ) { - if( p_grid->min_val > p_grid->max_val ) - CV_ERROR( CV_StsBadArg, - "low bound of grid should be less then the upper one"); - if( p_grid->step <= 1 ) - CV_ERROR( CV_StsBadArg, "grid step should be greater 1" ); - p_begin = p_grid->min_val; - p_end = p_grid->max_val; - p_step = p_grid->step; + _params.termCrit.epsilon = (double)tcnode["epsilon"]; + _params.termCrit.maxCount = (int)tcnode["iterations"]; + _params.termCrit.type = (_params.termCrit.epsilon > 0 ? TermCriteria::EPS : 0) + + (_params.termCrit.maxCount > 0 ? TermCriteria::COUNT : 0); } + else + _params.termCrit = TermCriteria( TermCriteria::EPS + TermCriteria::COUNT, 1000, FLT_EPSILON ); + + setParams( _params, Ptr() ); } - else - p_begin = p_end = svm_params.p; - // these parameters are not used: - if( kernel != CvSVM::POLY ) - degree_begin = degree_end = svm_params.degree; + void read( const FileNode& fn ) + { + clear(); - if( kernel == CvSVM::LINEAR ) - g_begin = g_end = svm_params.gamma; + // read SVM parameters + read_params( fn ); - if( kernel != CvSVM::POLY && kernel != CvSVM::SIGMOID ) - coef_begin = coef_end = svm_params.coef0; + // and top-level data + int i, sv_total = (int)fn["sv_total"]; + var_count = (int)fn["var_count"]; + int class_count = (int)fn["class_count"]; - if( svm_type == CvSVM::NU_SVC || svm_type == CvSVM::ONE_CLASS ) - C_begin = C_end = svm_params.C; + if( sv_total <= 0 || var_count <= 0 ) + CV_Error( CV_StsParseError, "SVM model data is invalid, check sv_count, var_* and class_count tags" ); - if( svm_type == CvSVM::C_SVC || svm_type == CvSVM::EPS_SVR ) - nu_begin = nu_end = svm_params.nu; + FileNode m = fn["class_labels"]; + if( !m.empty() ) + m >> class_labels; + m = fn["class_weights"]; + if( !m.empty() ) + m >> params.classWeights; - if( svm_type != CvSVM::EPS_SVR ) - p_begin = p_end = svm_params.p; + if( class_count > 1 && (class_labels.empty() || (int)class_labels.total() != class_count)) + CV_Error( CV_StsParseError, "Array of class labels is missing or invalid" ); - is_regression = cv_params->is_regression; - best_rate = is_regression ? FLT_MAX : 0; + // read support vectors + FileNode sv_node = fn["support_vectors"]; - assert( g_step > 1 && degree_step > 1 && coef_step > 1); - assert( p_step > 1 && C_step > 1 && nu_step > 1 ); + CV_Assert((int)sv_node.size() == sv_total); + sv.create(sv_total, var_count, CV_32F); - for( degree = degree_begin; degree <= degree_end; degree *= degree_step ) - { - svm_params.degree = degree; - //printf("degree = %.3f\n", degree ); - for( gamma= g_begin; gamma <= g_end; gamma *= g_step ) - { - svm_params.gamma = gamma; - //printf(" gamma = %.3f\n", gamma ); - for( coef = coef_begin; coef <= coef_end; coef *= coef_step ) + FileNodeIterator sv_it = sv_node.begin(); + for( i = 0; i < sv_total; i++, ++sv_it ) { - svm_params.coef0 = coef; - //printf(" coef = %.3f\n", coef ); - for( C = C_begin; C <= C_end; C *= C_step ) - { - svm_params.C = C; - //printf(" C = %.3f\n", C ); - for( nu = nu_begin; nu <= nu_end; nu *= nu_step ) - { - svm_params.nu = nu; - //printf(" nu = %.3f\n", nu ); - for( p = p_begin; p <= p_end; p *= p_step ) - { - int well; - svm_params.p = p; - //printf(" p = %.3f\n", p ); - - CV_CALL(rate = cvCrossValidation( train_data, tflag, responses, &cvTrainSVM, - cross_valid_params, (CvStatModelParams*)&svm_params, comp_idx, sample_idx )); - - well = rate > best_rate && !is_regression || rate < best_rate && is_regression; - if( well || (rate == best_rate && C < best_C) ) - { - best_rate = rate; - best_degree = degree; - best_gamma = gamma; - best_coef = coef; - best_C = C; - best_nu = nu; - best_p = p; - } - //printf(" rate = %.2f\n", rate ); - } - } - } + (*sv_it).readRaw("f", sv.ptr(i), var_count*sv.elemSize()); } - } - } - //printf("The best:\nrate = %.2f%% degree = %f gamma = %f coef = %f c = %f nu = %f p = %f\n", - // best_rate, best_degree, best_gamma, best_coef, best_C, best_nu, best_p ); - psvm_params->C = best_C; - psvm_params->nu = best_nu; - psvm_params->p = best_p; - psvm_params->gamma = best_gamma; - psvm_params->degree = best_degree; - psvm_params->coef0 = best_coef; + // read decision functions + int df_count = class_count > 1 ? class_count*(class_count-1)/2 : 1; + FileNode df_node = fn["decision_functions"]; + + CV_Assert((int)df_node.size() == df_count); - CV_CALL(svm = cvTrainSVM( train_data, tflag, responses, model_params, comp_idx, sample_idx )); + FileNodeIterator df_it = df_node.begin(); + for( i = 0; i < df_count; i++, ++df_it ) + { + FileNode dfi = *df_it; + DecisionFunc df; + int sv_count = (int)dfi["sv_count"]; + int ofs = (int)df_index.size(); + df.rho = (double)dfi["rho"]; + df.ofs = ofs; + df_index.resize(ofs + sv_count); + df_alpha.resize(ofs + sv_count); + dfi["alpha"].readRaw("d", (uchar*)&df_alpha[ofs], sv_count*sizeof(df_alpha[0])); + if( class_count > 2 ) + dfi["index"].readRaw("i", (uchar*)&df_index[ofs], sv_count*sizeof(df_index[0])); + decision_func.push_back(df); + } + if( class_count <= 2 ) + setRangeVector(df_index, sv_total); + if( (int)fn["optimize_linear"] != 0 ) + optimize_linear_svm(); + } + + Params params; + TermCriteria termCrit; + Mat class_labels; + int var_count; + Mat sv; + vector decision_func; + vector df_alpha; + vector df_index; + + Ptr kernel; +}; - __END__; - return svm; +Ptr SVM::create(const Params& params, const Ptr& kernel) +{ + Ptr p = makePtr(); + p->setParams(params, kernel); + return p; } -#endif +} +} /* End of file. */ diff --git a/modules/ml/src/testset.cpp b/modules/ml/src/testset.cpp index 5edb3b45df83f972d2ed230697a7b01b896f9686..8b8bba5456f8c5e0c8fc622d1ba16e41502582c1 100644 --- a/modules/ml/src/testset.cpp +++ b/modules/ml/src/testset.cpp @@ -40,131 +40,74 @@ #include "precomp.hpp" -typedef struct CvDI +namespace cv { namespace ml { + +struct PairDI { double d; int i; -} CvDI; +}; -static int CV_CDECL -icvCmpDI( const void* a, const void* b, void* ) +struct CmpPairDI { - const CvDI* e1 = (const CvDI*) a; - const CvDI* e2 = (const CvDI*) b; - - return (e1->d < e2->d) ? -1 : (e1->d > e2->d); -} + bool operator ()(const PairDI& e1, const PairDI& e2) const + { + return (e1.d < e2.d) || (e1.d == e2.d && e1.i < e2.i); + } +}; -CV_IMPL void -cvCreateTestSet( int type, CvMat** samples, - int num_samples, - int num_features, - CvMat** responses, - int num_classes, ... ) +void createConcentricSpheresTestSet( int num_samples, int num_features, int num_classes, + OutputArray _samples, OutputArray _responses) { - CvMat* mean = NULL; - CvMat* cov = NULL; - CvMemStorage* storage = NULL; - - CV_FUNCNAME( "cvCreateTestSet" ); + if( num_samples < 1 ) + CV_Error( CV_StsBadArg, "num_samples parameter must be positive" ); - __BEGIN__; + if( num_features < 1 ) + CV_Error( CV_StsBadArg, "num_features parameter must be positive" ); - if( samples ) - *samples = NULL; - if( responses ) - *responses = NULL; + if( num_classes < 1 ) + CV_Error( CV_StsBadArg, "num_classes parameter must be positive" ); - if( type != CV_TS_CONCENTRIC_SPHERES ) - CV_ERROR( CV_StsBadArg, "Invalid type parameter" ); + int i, cur_class; - if( !samples ) - CV_ERROR( CV_StsNullPtr, "samples parameter must be not NULL" ); + _samples.create( num_samples, num_features, CV_32F ); + _responses.create( 1, num_samples, CV_32S ); - if( !responses ) - CV_ERROR( CV_StsNullPtr, "responses parameter must be not NULL" ); + Mat responses = _responses.getMat(); - if( num_samples < 1 ) - CV_ERROR( CV_StsBadArg, "num_samples parameter must be positive" ); + Mat mean = Mat::zeros(1, num_features, CV_32F); + Mat cov = Mat::eye(num_features, num_features, CV_32F); - if( num_features < 1 ) - CV_ERROR( CV_StsBadArg, "num_features parameter must be positive" ); + // fill the feature values matrix with random numbers drawn from standard normal distribution + randMVNormal( mean, cov, num_samples, _samples ); + Mat samples = _samples.getMat(); - if( num_classes < 1 ) - CV_ERROR( CV_StsBadArg, "num_classes parameter must be positive" ); + // calculate distances from the origin to the samples and put them + // into the sequence along with indices + std::vector dis(samples.rows); - if( type == CV_TS_CONCENTRIC_SPHERES ) + for( i = 0; i < samples.rows; i++ ) { - CvSeqWriter writer; - CvSeqReader reader; - CvMat sample; - CvDI elem; - CvSeq* seq = NULL; - int i, cur_class; - - CV_CALL( *samples = cvCreateMat( num_samples, num_features, CV_32FC1 ) ); - CV_CALL( *responses = cvCreateMat( 1, num_samples, CV_32SC1 ) ); - CV_CALL( mean = cvCreateMat( 1, num_features, CV_32FC1 ) ); - CV_CALL( cvSetZero( mean ) ); - CV_CALL( cov = cvCreateMat( num_features, num_features, CV_32FC1 ) ); - CV_CALL( cvSetIdentity( cov ) ); - - /* fill the feature values matrix with random numbers drawn from standard - normal distribution */ - CV_CALL( cvRandMVNormal( mean, cov, *samples ) ); - - /* calculate distances from the origin to the samples and put them - into the sequence along with indices */ - CV_CALL( storage = cvCreateMemStorage() ); - CV_CALL( cvStartWriteSeq( 0, sizeof( CvSeq ), sizeof( CvDI ), storage, &writer )); - for( i = 0; i < (*samples)->rows; ++i ) - { - CV_CALL( cvGetRow( *samples, &sample, i )); - elem.i = i; - CV_CALL( elem.d = cvNorm( &sample, NULL, CV_L2 )); - CV_WRITE_SEQ_ELEM( elem, writer ); - } - CV_CALL( seq = cvEndWriteSeq( &writer ) ); - - /* sort the sequence in a distance ascending order */ - CV_CALL( cvSeqSort( seq, icvCmpDI, NULL ) ); - - /* assign class labels */ - num_classes = MIN( num_samples, num_classes ); - CV_CALL( cvStartReadSeq( seq, &reader ) ); - CV_READ_SEQ_ELEM( elem, reader ); - for( i = 0, cur_class = 0; i < num_samples; ++cur_class ) - { - int last_idx; - double max_dst; - - last_idx = num_samples * (cur_class + 1) / num_classes - 1; - CV_CALL( max_dst = (*((CvDI*) cvGetSeqElem( seq, last_idx ))).d ); - max_dst = MAX( max_dst, elem.d ); - - for( ; elem.d <= max_dst && i < num_samples; ++i ) - { - CV_MAT_ELEM( **responses, int, 0, elem.i ) = cur_class; - if( i < num_samples - 1 ) - { - CV_READ_SEQ_ELEM( elem, reader ); - } - } - } + PairDI& elem = dis[i]; + elem.i = i; + elem.d = norm(samples.row(i), NORM_L2); } - __END__; + std::sort(dis.begin(), dis.end(), CmpPairDI()); - if( cvGetErrStatus() < 0 ) + // assign class labels + num_classes = std::min( num_samples, num_classes ); + for( i = 0, cur_class = 0; i < num_samples; ++cur_class ) { - if( samples ) - cvReleaseMat( samples ); - if( responses ) - cvReleaseMat( responses ); + int last_idx = num_samples * (cur_class + 1) / num_classes - 1; + double max_dst = dis[last_idx].d; + max_dst = std::max( max_dst, dis[i].d ); + + for( ; i < num_samples && dis[i].d <= max_dst; ++i ) + responses.at(i) = cur_class; } - cvReleaseMat( &mean ); - cvReleaseMat( &cov ); - cvReleaseMemStorage( &storage ); } +}} + /* End of file. */ diff --git a/modules/ml/src/tree.cpp b/modules/ml/src/tree.cpp index 41d2553a43ba04f0ba8fe1085374d6ea73e2688a..416abd9364f495058f6606d140a1aaae983ad373 100644 --- a/modules/ml/src/tree.cpp +++ b/modules/ml/src/tree.cpp @@ -7,9 +7,11 @@ // copy or use the software. // // -// Intel License Agreement +// License Agreement +// For Open Source Computer Vision Library // // Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc, all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -22,7 +24,7 @@ // this list of conditions and the following disclaimer in the documentation // and/or other materials provided with the distribution. // -// * The name of Intel Corporation may not be used to endorse or promote products +// * The name of the copyright holders may not be used to endorse or promote products // derived from this software without specific prior written permission. // // This software is provided by the copyright holders and contributors "as is" and @@ -41,2730 +43,470 @@ #include "precomp.hpp" #include -using namespace cv; +namespace cv { +namespace ml { -static const float ord_nan = FLT_MAX*0.5f; -static const int min_block_size = 1 << 16; -static const int block_size_delta = 1 << 10; +using std::vector; -CvDTreeTrainData::CvDTreeTrainData() +void DTrees::setDParams(const DTrees::Params&) { - var_idx = var_type = cat_count = cat_ofs = cat_map = - priors = priors_mult = counts = direction = split_buf = responses_copy = 0; - buf = 0; - tree_storage = temp_storage = 0; - - clear(); + CV_Error(CV_StsNotImplemented, ""); } - -CvDTreeTrainData::CvDTreeTrainData( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, - const CvMat* _sample_idx, const CvMat* _var_type, - const CvMat* _missing_mask, const CvDTreeParams& _params, - bool _shared, bool _add_labels ) +DTrees::Params DTrees::getDParams() const { - var_idx = var_type = cat_count = cat_ofs = cat_map = - priors = priors_mult = counts = direction = split_buf = responses_copy = 0; - buf = 0; - - tree_storage = temp_storage = 0; - - set_data( _train_data, _tflag, _responses, _var_idx, _sample_idx, - _var_type, _missing_mask, _params, _shared, _add_labels ); + CV_Error(CV_StsNotImplemented, ""); + return DTrees::Params(); } - -CvDTreeTrainData::~CvDTreeTrainData() +DTrees::Params::Params() { - clear(); + maxDepth = INT_MAX; + minSampleCount = 10; + regressionAccuracy = 0.01f; + useSurrogates = false; + maxCategories = 10; + CVFolds = 10; + use1SERule = true; + truncatePrunedTree = true; + priors = Mat(); } - -bool CvDTreeTrainData::set_params( const CvDTreeParams& _params ) +DTrees::Params::Params( int _maxDepth, int _minSampleCount, + double _regressionAccuracy, bool _useSurrogates, + int _maxCategories, int _CVFolds, + bool _use1SERule, bool _truncatePrunedTree, + const Mat& _priors ) { - bool ok = false; - - CV_FUNCNAME( "CvDTreeTrainData::set_params" ); - - __BEGIN__; - - // set parameters - params = _params; - - if( params.max_categories < 2 ) - CV_ERROR( CV_StsOutOfRange, "params.max_categories should be >= 2" ); - params.max_categories = MIN( params.max_categories, 15 ); - - if( params.max_depth < 0 ) - CV_ERROR( CV_StsOutOfRange, "params.max_depth should be >= 0" ); - params.max_depth = MIN( params.max_depth, 25 ); - - params.min_sample_count = MAX(params.min_sample_count,1); - - if( params.cv_folds < 0 ) - CV_ERROR( CV_StsOutOfRange, - "params.cv_folds should be =0 (the tree is not pruned) " - "or n>0 (tree is pruned using n-fold cross-validation)" ); - - if( params.cv_folds == 1 ) - params.cv_folds = 0; - - if( params.regression_accuracy < 0 ) - CV_ERROR( CV_StsOutOfRange, "params.regression_accuracy should be >= 0" ); - - ok = true; - - __END__; - - return ok; + maxDepth = _maxDepth; + minSampleCount = _minSampleCount; + regressionAccuracy = (float)_regressionAccuracy; + useSurrogates = _useSurrogates; + maxCategories = _maxCategories; + CVFolds = _CVFolds; + use1SERule = _use1SERule; + truncatePrunedTree = _truncatePrunedTree; + priors = _priors; } -template -class LessThanPtr -{ -public: - bool operator()(T* a, T* b) const { return *a < *b; } -}; - -template -class LessThanIdx +DTrees::Node::Node() { -public: - LessThanIdx( const T* _arr ) : arr(_arr) {} - bool operator()(Idx a, Idx b) const { return arr[a] < arr[b]; } - const T* arr; -}; + classIdx = 0; + value = 0; + parent = left = right = split = defaultDir = -1; +} -class LessThanPairs +DTrees::Split::Split() { -public: - bool operator()(const CvPair16u32s& a, const CvPair16u32s& b) const { return *a.i < *b.i; } -}; - -void CvDTreeTrainData::set_data( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, const CvMat* _sample_idx, - const CvMat* _var_type, const CvMat* _missing_mask, const CvDTreeParams& _params, - bool _shared, bool _add_labels, bool _update_data ) -{ - CvMat* sample_indices = 0; - CvMat* var_type0 = 0; - CvMat* tmp_map = 0; - int** int_ptr = 0; - CvPair16u32s* pair16u32s_ptr = 0; - CvDTreeTrainData* data = 0; - float *_fdst = 0; - int *_idst = 0; - unsigned short* udst = 0; - int* idst = 0; - - CV_FUNCNAME( "CvDTreeTrainData::set_data" ); - - __BEGIN__; - - int sample_all = 0, r_type, cv_n; - int total_c_count = 0; - int tree_block_size, temp_block_size, max_split_size, nv_size, cv_size = 0; - int ds_step, dv_step, ms_step = 0, mv_step = 0; // {data|mask}{sample|var}_step - int vi, i, size; - char err[100]; - const int *sidx = 0, *vidx = 0; - - uint64 effective_buf_size = 0; - int effective_buf_height = 0, effective_buf_width = 0; - - if( _update_data && data_root ) - { - data = new CvDTreeTrainData( _train_data, _tflag, _responses, _var_idx, - _sample_idx, _var_type, _missing_mask, _params, _shared, _add_labels ); - - // compare new and old train data - if( !(data->var_count == var_count && - cvNorm( data->var_type, var_type, CV_C ) < FLT_EPSILON && - cvNorm( data->cat_count, cat_count, CV_C ) < FLT_EPSILON && - cvNorm( data->cat_map, cat_map, CV_C ) < FLT_EPSILON) ) - CV_ERROR( CV_StsBadArg, - "The new training data must have the same types and the input and output variables " - "and the same categories for categorical variables" ); - - cvReleaseMat( &priors ); - cvReleaseMat( &priors_mult ); - cvReleaseMat( &buf ); - cvReleaseMat( &direction ); - cvReleaseMat( &split_buf ); - cvReleaseMemStorage( &temp_storage ); - - priors = data->priors; data->priors = 0; - priors_mult = data->priors_mult; data->priors_mult = 0; - buf = data->buf; data->buf = 0; - buf_count = data->buf_count; buf_size = data->buf_size; - sample_count = data->sample_count; - - direction = data->direction; data->direction = 0; - split_buf = data->split_buf; data->split_buf = 0; - temp_storage = data->temp_storage; data->temp_storage = 0; - nv_heap = data->nv_heap; cv_heap = data->cv_heap; - - data_root = new_node( 0, sample_count, 0, 0 ); - EXIT; - } - - clear(); - - var_all = 0; - rng = &cv::theRNG(); - - CV_CALL( set_params( _params )); - - // check parameter types and sizes - CV_CALL( cvCheckTrainData( _train_data, _tflag, _missing_mask, &var_all, &sample_all )); + varIdx = 0; + inversed = false; + quality = 0.f; + next = -1; + c = 0.f; + subsetOfs = 0; +} - train_data = _train_data; - responses = _responses; - if( _tflag == CV_ROW_SAMPLE ) +DTreesImpl::WorkData::WorkData(const Ptr& _data) +{ + data = _data; + vector subsampleIdx; + Mat sidx0 = _data->getTrainSampleIdx(); + if( !sidx0.empty() ) { - ds_step = _train_data->step/CV_ELEM_SIZE(_train_data->type); - dv_step = 1; - if( _missing_mask ) - ms_step = _missing_mask->step, mv_step = 1; + sidx0.copyTo(sidx); + std::sort(sidx.begin(), sidx.end()); } else { - dv_step = _train_data->step/CV_ELEM_SIZE(_train_data->type); - ds_step = 1; - if( _missing_mask ) - mv_step = _missing_mask->step, ms_step = 1; - } - tflag = _tflag; - - sample_count = sample_all; - var_count = var_all; - - if( _sample_idx ) - { - CV_CALL( sample_indices = cvPreprocessIndexArray( _sample_idx, sample_all )); - sidx = sample_indices->data.i; - sample_count = sample_indices->rows + sample_indices->cols - 1; - } - - if( _var_idx ) - { - CV_CALL( var_idx = cvPreprocessIndexArray( _var_idx, var_all )); - vidx = var_idx->data.i; - var_count = var_idx->rows + var_idx->cols - 1; + int n = _data->getNSamples(); + setRangeVector(sidx, n); } - is_buf_16u = false; - if ( sample_count < 65536 ) - is_buf_16u = true; + maxSubsetSize = 0; +} - if( !CV_IS_MAT(_responses) || - (CV_MAT_TYPE(_responses->type) != CV_32SC1 && - CV_MAT_TYPE(_responses->type) != CV_32FC1) || - (_responses->rows != 1 && _responses->cols != 1) || - _responses->rows + _responses->cols - 1 != sample_all ) - CV_ERROR( CV_StsBadArg, "The array of _responses must be an integer or " - "floating-point vector containing as many elements as " - "the total number of samples in the training data matrix" ); +DTreesImpl::DTreesImpl() {} +DTreesImpl::~DTreesImpl() {} +void DTreesImpl::clear() +{ + varIdx.clear(); + compVarIdx.clear(); + varType.clear(); + catOfs.clear(); + catMap.clear(); + roots.clear(); + nodes.clear(); + splits.clear(); + subsets.clear(); + classLabels.clear(); - r_type = CV_VAR_CATEGORICAL; - if( _var_type ) - CV_CALL( var_type0 = cvPreprocessVarType( _var_type, var_idx, var_count, &r_type )); + w.release(); + _isClassifier = false; +} - CV_CALL( var_type = cvCreateMat( 1, var_count+2, CV_32SC1 )); +void DTreesImpl::startTraining( const Ptr& data, int ) +{ + clear(); + w = makePtr(data); - cat_var_count = 0; - ord_var_count = -1; + Mat vtype = data->getVarType(); + vtype.copyTo(varType); - is_classifier = r_type == CV_VAR_CATEGORICAL; + data->getCatOfs().copyTo(catOfs); + data->getCatMap().copyTo(catMap); + data->getDefaultSubstValues().copyTo(missingSubst); - // step 0. calc the number of categorical vars - for( vi = 0; vi < var_count; vi++ ) - { - char vt = var_type0 ? var_type0->data.ptr[vi] : CV_VAR_ORDERED; - var_type->data.i[vi] = vt == CV_VAR_CATEGORICAL ? cat_var_count++ : ord_var_count--; - } + int nallvars = data->getNAllVars(); - ord_var_count = ~ord_var_count; - cv_n = params.cv_folds; - // set the two last elements of var_type array to be able - // to locate responses and cross-validation labels using - // the corresponding get_* functions. - var_type->data.i[var_count] = cat_var_count; - var_type->data.i[var_count+1] = cat_var_count+1; + Mat vidx0 = data->getVarIdx(); + if( !vidx0.empty() ) + vidx0.copyTo(varIdx); + else + setRangeVector(varIdx, nallvars); - // in case of single ordered predictor we need dummy cv_labels - // for safe split_node_data() operation - have_labels = cv_n > 0 || (ord_var_count == 1 && cat_var_count == 0) || _add_labels; + initCompVarIdx(); - work_var_count = var_count + (is_classifier ? 1 : 0) // for responses class_labels - + (have_labels ? 1 : 0); // for cv_labels + w->maxSubsetSize = 0; - shared = _shared; - buf_count = shared ? 2 : 1; + int i, nvars = (int)varIdx.size(); + for( i = 0; i < nvars; i++ ) + w->maxSubsetSize = std::max(w->maxSubsetSize, getCatCount(varIdx[i])); - buf_size = -1; // the member buf_size is obsolete + w->maxSubsetSize = std::max((w->maxSubsetSize + 31)/32, 1); - effective_buf_size = (uint64)(work_var_count + 1)*(uint64)sample_count * buf_count; // this is the total size of "CvMat buf" to be allocated - effective_buf_width = sample_count; - effective_buf_height = work_var_count+1; + data->getSampleWeights().copyTo(w->sample_weights); - if (effective_buf_width >= effective_buf_height) - effective_buf_height *= buf_count; - else - effective_buf_width *= buf_count; + _isClassifier = data->getResponseType() == VAR_CATEGORICAL; - if ((uint64)effective_buf_width * (uint64)effective_buf_height != effective_buf_size) + if( _isClassifier ) { - CV_Error(CV_StsBadArg, "The memory buffer cannot be allocated since its size exceeds integer fields limit"); - } + data->getNormCatResponses().copyTo(w->cat_responses); + data->getClassLabels().copyTo(classLabels); + int nclasses = (int)classLabels.size(); + Mat class_weights = params.priors; + if( !class_weights.empty() ) + { + if( class_weights.type() != CV_64F || !class_weights.isContinuous() ) + { + Mat temp; + class_weights.convertTo(temp, CV_64F); + class_weights = temp; + } + CV_Assert( class_weights.checkVector(1, CV_64F) == nclasses ); + int nsamples = (int)w->cat_responses.size(); + const double* cw = class_weights.ptr(); + CV_Assert( (int)w->sample_weights.size() == nsamples ); - if ( is_buf_16u ) - { - CV_CALL( buf = cvCreateMat( effective_buf_height, effective_buf_width, CV_16UC1 )); - CV_CALL( pair16u32s_ptr = (CvPair16u32s*)cvAlloc( sample_count*sizeof(pair16u32s_ptr[0]) )); + for( i = 0; i < nsamples; i++ ) + { + int ci = w->cat_responses[i]; + CV_Assert( 0 <= ci && ci < nclasses ); + w->sample_weights[i] *= cw[ci]; + } + } } else + data->getResponses().copyTo(w->ord_responses); +} + + +void DTreesImpl::initCompVarIdx() +{ + int nallvars = (int)varType.size(); + compVarIdx.assign(nallvars, -1); + int i, nvars = (int)varIdx.size(), prevIdx = -1; + for( i = 0; i < nvars; i++ ) { - CV_CALL( buf = cvCreateMat( effective_buf_height, effective_buf_width, CV_32SC1 )); - CV_CALL( int_ptr = (int**)cvAlloc( sample_count*sizeof(int_ptr[0]) )); + int vi = varIdx[i]; + CV_Assert( 0 <= vi && vi < nallvars && vi > prevIdx ); + prevIdx = vi; + compVarIdx[vi] = i; } +} - size = is_classifier ? (cat_var_count+1) : cat_var_count; - size = !size ? 1 : size; - CV_CALL( cat_count = cvCreateMat( 1, size, CV_32SC1 )); - CV_CALL( cat_ofs = cvCreateMat( 1, size, CV_32SC1 )); +void DTreesImpl::endTraining() +{ + w.release(); +} + +bool DTreesImpl::train( const Ptr& trainData, int flags ) +{ + startTraining(trainData, flags); + bool ok = addTree( w->sidx ) >= 0; + w.release(); + endTraining(); + return ok; +} - size = is_classifier ? (cat_var_count + 1)*params.max_categories : cat_var_count*params.max_categories; - size = !size ? 1 : size; - CV_CALL( cat_map = cvCreateMat( 1, size, CV_32SC1 )); +const vector& DTreesImpl::getActiveVars() +{ + return varIdx; +} - // now calculate the maximum size of split, - // create memory storage that will keep nodes and splits of the decision tree - // allocate root node and the buffer for the whole training data - max_split_size = cvAlign(sizeof(CvDTreeSplit) + - (MAX(0,sample_count - 33)/32)*sizeof(int),sizeof(void*)); - tree_block_size = MAX((int)sizeof(CvDTreeNode)*8, max_split_size); - tree_block_size = MAX(tree_block_size + block_size_delta, min_block_size); - CV_CALL( tree_storage = cvCreateMemStorage( tree_block_size )); - CV_CALL( node_heap = cvCreateSet( 0, sizeof(*node_heap), sizeof(CvDTreeNode), tree_storage )); +int DTreesImpl::addTree(const vector& sidx ) +{ + size_t n = (params.maxDepth > 0 ? (1 << params.maxDepth) : 1024) + w->wnodes.size(); - nv_size = var_count*sizeof(int); - nv_size = cvAlign(MAX( nv_size, (int)sizeof(CvSetElem) ), sizeof(void*)); + w->wnodes.reserve(n); + w->wsplits.reserve(n); + w->wsubsets.reserve(n*w->maxSubsetSize); + w->wnodes.clear(); + w->wsplits.clear(); + w->wsubsets.clear(); - temp_block_size = nv_size; + int cv_n = params.CVFolds; - if( cv_n ) + if( cv_n > 0 ) { - if( sample_count < cv_n*MAX(params.min_sample_count,10) ) - CV_ERROR( CV_StsOutOfRange, - "The many folds in cross-validation for such a small dataset" ); - - cv_size = cvAlign( cv_n*(sizeof(int) + sizeof(double)*2), sizeof(double) ); - temp_block_size = MAX(temp_block_size, cv_size); + w->cv_Tn.resize(n*cv_n); + w->cv_node_error.resize(n*cv_n); + w->cv_node_risk.resize(n*cv_n); } - temp_block_size = MAX( temp_block_size + block_size_delta, min_block_size ); - CV_CALL( temp_storage = cvCreateMemStorage( temp_block_size )); - CV_CALL( nv_heap = cvCreateSet( 0, sizeof(*nv_heap), nv_size, temp_storage )); - if( cv_size ) - CV_CALL( cv_heap = cvCreateSet( 0, sizeof(*cv_heap), cv_size, temp_storage )); - - CV_CALL( data_root = new_node( 0, sample_count, 0, 0 )); - - max_c_count = 1; + // build the tree recursively + int w_root = addNodeAndTrySplit(-1, sidx); + int maxdepth = INT_MAX;//pruneCV(root); - _fdst = 0; - _idst = 0; - if (ord_var_count) - _fdst = (float*)cvAlloc(sample_count*sizeof(_fdst[0])); - if (is_buf_16u && (cat_var_count || is_classifier)) - _idst = (int*)cvAlloc(sample_count*sizeof(_idst[0])); + int w_nidx = w_root, pidx = -1, depth = 0; + int root = (int)nodes.size(); - // transform the training data to convenient representation - for( vi = 0; vi <= var_count; vi++ ) + for(;;) { - int ci; - const uchar* mask = 0; - int64 m_step = 0, step; - const int* idata = 0; - const float* fdata = 0; - int num_valid = 0; - - if( vi < var_count ) // analyze i-th input variable - { - int vi0 = vidx ? vidx[vi] : vi; - ci = get_var_type(vi); - step = ds_step; m_step = ms_step; - if( CV_MAT_TYPE(_train_data->type) == CV_32SC1 ) - idata = _train_data->data.i + vi0*dv_step; - else - fdata = _train_data->data.fl + vi0*dv_step; - if( _missing_mask ) - mask = _missing_mask->data.ptr + vi0*mv_step; - } - else // analyze _responses - { - ci = cat_var_count; - step = CV_IS_MAT_CONT(_responses->type) ? - 1 : _responses->step / CV_ELEM_SIZE(_responses->type); - if( CV_MAT_TYPE(_responses->type) == CV_32SC1 ) - idata = _responses->data.i; - else - fdata = _responses->data.fl; - } + const WNode& wnode = w->wnodes[w_nidx]; + Node node; + node.parent = pidx; + node.classIdx = wnode.class_idx; + node.value = wnode.value; + node.defaultDir = wnode.defaultDir; - if( (vi < var_count && ci>=0) || - (vi == var_count && is_classifier) ) // process categorical variable or response + int wsplit_idx = wnode.split; + if( wsplit_idx >= 0 ) { - int c_count, prev_label; - int* c_map; - - if (is_buf_16u) - udst = (unsigned short*)(buf->data.s + vi*sample_count); - else - idst = buf->data.i + vi*sample_count; - - // copy data - for( i = 0; i < sample_count; i++ ) - { - int val = INT_MAX, si = sidx ? sidx[i] : i; - if( !mask || !mask[(size_t)si*m_step] ) - { - if( idata ) - val = idata[(size_t)si*step]; - else - { - float t = fdata[(size_t)si*step]; - val = cvRound(t); - if( fabs(t - val) > FLT_EPSILON ) - { - sprintf( err, "%d-th value of %d-th (categorical) " - "variable is not an integer", i, vi ); - CV_ERROR( CV_StsBadArg, err ); - } - } - - if( val == INT_MAX ) - { - sprintf( err, "%d-th value of %d-th (categorical) " - "variable is too large", i, vi ); - CV_ERROR( CV_StsBadArg, err ); - } - num_valid++; - } - if (is_buf_16u) - { - _idst[i] = val; - pair16u32s_ptr[i].u = udst + i; - pair16u32s_ptr[i].i = _idst + i; - } - else - { - idst[i] = val; - int_ptr[i] = idst + i; - } - } - - c_count = num_valid > 0; - if (is_buf_16u) - { - std::sort(pair16u32s_ptr, pair16u32s_ptr + sample_count, LessThanPairs()); - // count the categories - for( i = 1; i < num_valid; i++ ) - if (*pair16u32s_ptr[i].i != *pair16u32s_ptr[i-1].i) - c_count ++ ; - } - else - { - std::sort(int_ptr, int_ptr + sample_count, LessThanPtr()); - // count the categories - for( i = 1; i < num_valid; i++ ) - c_count += *int_ptr[i] != *int_ptr[i-1]; - } - - if( vi > 0 ) - max_c_count = MAX( max_c_count, c_count ); - cat_count->data.i[ci] = c_count; - cat_ofs->data.i[ci] = total_c_count; - - // resize cat_map, if need - if( cat_map->cols < total_c_count + c_count ) - { - tmp_map = cat_map; - CV_CALL( cat_map = cvCreateMat( 1, - MAX(cat_map->cols*3/2,total_c_count+c_count), CV_32SC1 )); - for( i = 0; i < total_c_count; i++ ) - cat_map->data.i[i] = tmp_map->data.i[i]; - cvReleaseMat( &tmp_map ); - } - - c_map = cat_map->data.i + total_c_count; - total_c_count += c_count; - - c_count = -1; - if (is_buf_16u) - { - // compact the class indices and build the map - prev_label = ~*pair16u32s_ptr[0].i; - for( i = 0; i < num_valid; i++ ) - { - int cur_label = *pair16u32s_ptr[i].i; - if( cur_label != prev_label ) - c_map[++c_count] = prev_label = cur_label; - *pair16u32s_ptr[i].u = (unsigned short)c_count; - } - // replace labels for missing values with -1 - for( ; i < sample_count; i++ ) - *pair16u32s_ptr[i].u = 65535; - } - else + const WSplit& wsplit = w->wsplits[wsplit_idx]; + Split split; + split.c = wsplit.c; + split.quality = wsplit.quality; + split.inversed = wsplit.inversed; + split.varIdx = wsplit.varIdx; + split.subsetOfs = -1; + if( wsplit.subsetOfs >= 0 ) { - // compact the class indices and build the map - prev_label = ~*int_ptr[0]; - for( i = 0; i < num_valid; i++ ) - { - int cur_label = *int_ptr[i]; - if( cur_label != prev_label ) - c_map[++c_count] = prev_label = cur_label; - *int_ptr[i] = c_count; - } - // replace labels for missing values with -1 - for( ; i < sample_count; i++ ) - *int_ptr[i] = -1; + int ssize = getSubsetSize(split.varIdx); + split.subsetOfs = (int)subsets.size(); + subsets.resize(split.subsetOfs + ssize); + memcpy(&subsets[split.subsetOfs], &w->wsubsets[wsplit.subsetOfs], ssize*sizeof(int)); } + node.split = (int)splits.size(); + splits.push_back(split); } - else if( ci < 0 ) // process ordered variable + int nidx = (int)nodes.size(); + nodes.push_back(node); + if( pidx >= 0 ) { - if (is_buf_16u) - udst = (unsigned short*)(buf->data.s + vi*sample_count); - else - idst = buf->data.i + vi*sample_count; - - for( i = 0; i < sample_count; i++ ) + int w_pidx = w->wnodes[w_nidx].parent; + if( w->wnodes[w_pidx].left == w_nidx ) { - float val = ord_nan; - int si = sidx ? sidx[i] : i; - if( !mask || !mask[(size_t)si*m_step] ) - { - if( idata ) - val = (float)idata[(size_t)si*step]; - else - val = fdata[(size_t)si*step]; - - if( fabs(val) >= ord_nan ) - { - sprintf( err, "%d-th value of %d-th (ordered) " - "variable (=%g) is too large", i, vi, val ); - CV_ERROR( CV_StsBadArg, err ); - } - num_valid++; - } - - if (is_buf_16u) - udst[i] = (unsigned short)i; // TODO: memory corruption may be here - else - idst[i] = i; - _fdst[i] = val; - + nodes[pidx].left = nidx; } - if (is_buf_16u) - std::sort(udst, udst + sample_count, LessThanIdx(_fdst)); else - std::sort(idst, idst + sample_count, LessThanIdx(_fdst)); - } - - if( vi < var_count ) - data_root->set_num_valid(vi, num_valid); - } - - // set sample labels - if (is_buf_16u) - udst = (unsigned short*)(buf->data.s + work_var_count*sample_count); - else - idst = buf->data.i + work_var_count*sample_count; - - for (i = 0; i < sample_count; i++) - { - if (udst) - udst[i] = sidx ? (unsigned short)sidx[i] : (unsigned short)i; - else - idst[i] = sidx ? sidx[i] : i; - } - - if( cv_n ) - { - unsigned short* usdst = 0; - int* idst2 = 0; - - if (is_buf_16u) - { - usdst = (unsigned short*)(buf->data.s + (get_work_var_count()-1)*sample_count); - for( i = vi = 0; i < sample_count; i++ ) { - usdst[i] = (unsigned short)vi++; - vi &= vi < cv_n ? -1 : 0; + CV_Assert(w->wnodes[w_pidx].right == w_nidx); + nodes[pidx].right = nidx; } + } - for( i = 0; i < sample_count; i++ ) - { - int a = (*rng)(sample_count); - int b = (*rng)(sample_count); - unsigned short unsh = (unsigned short)vi; - CV_SWAP( usdst[a], usdst[b], unsh ); - } + if( wnode.left >= 0 && depth+1 < maxdepth ) + { + w_nidx = wnode.left; + pidx = nidx; + depth++; } else { - idst2 = buf->data.i + (get_work_var_count()-1)*sample_count; - for( i = vi = 0; i < sample_count; i++ ) + int w_pidx = wnode.parent; + while( w_pidx >= 0 && w->wnodes[w_pidx].right == w_nidx ) { - idst2[i] = vi++; - vi &= vi < cv_n ? -1 : 0; + w_nidx = w_pidx; + w_pidx = w->wnodes[w_pidx].parent; + nidx = pidx; + pidx = nodes[pidx].parent; + depth--; } - for( i = 0; i < sample_count; i++ ) - { - int a = (*rng)(sample_count); - int b = (*rng)(sample_count); - CV_SWAP( idst2[a], idst2[b], vi ); - } - } - } - - if ( cat_map ) - cat_map->cols = MAX( total_c_count, 1 ); - - max_split_size = cvAlign(sizeof(CvDTreeSplit) + - (MAX(0,max_c_count - 33)/32)*sizeof(int),sizeof(void*)); - CV_CALL( split_heap = cvCreateSet( 0, sizeof(*split_heap), max_split_size, tree_storage )); + if( w_pidx < 0 ) + break; - have_priors = is_classifier && params.priors; - if( is_classifier ) - { - int m = get_num_classes(); - double sum = 0; - CV_CALL( priors = cvCreateMat( 1, m, CV_64F )); - for( i = 0; i < m; i++ ) - { - double val = have_priors ? params.priors[i] : 1.; - if( val <= 0 ) - CV_ERROR( CV_StsOutOfRange, "Every class weight should be positive" ); - priors->data.db[i] = val; - sum += val; + w_nidx = w->wnodes[w_pidx].right; + CV_Assert( w_nidx >= 0 ); } - - // normalize weights - if( have_priors ) - cvScale( priors, priors, 1./sum ); - - CV_CALL( priors_mult = cvCloneMat( priors )); - CV_CALL( counts = cvCreateMat( 1, m, CV_32SC1 )); } - - - CV_CALL( direction = cvCreateMat( 1, sample_count, CV_8UC1 )); - CV_CALL( split_buf = cvCreateMat( 1, sample_count, CV_32SC1 )); - - __END__; - - if( data ) - delete data; - - if (_fdst) - cvFree( &_fdst ); - if (_idst) - cvFree( &_idst ); - cvFree( &int_ptr ); - cvFree( &pair16u32s_ptr); - cvReleaseMat( &var_type0 ); - cvReleaseMat( &sample_indices ); - cvReleaseMat( &tmp_map ); + roots.push_back(root); + return root; } -void CvDTreeTrainData::do_responses_copy() +DTrees::Params DTreesImpl::getDParams() const { - responses_copy = cvCreateMat( responses->rows, responses->cols, responses->type ); - cvCopy( responses, responses_copy); - responses = responses_copy; + return params0; } -CvDTreeNode* CvDTreeTrainData::subsample_data( const CvMat* _subsample_idx ) +void DTreesImpl::setDParams(const Params& _params) { - CvDTreeNode* root = 0; - CvMat* isubsample_idx = 0; - CvMat* subsample_co = 0; + params0 = params = _params; + if( params.maxCategories < 2 ) + CV_Error( CV_StsOutOfRange, "params.max_categories should be >= 2" ); + params.maxCategories = std::min( params.maxCategories, 15 ); - bool isMakeRootCopy = true; + if( params.maxDepth < 0 ) + CV_Error( CV_StsOutOfRange, "params.max_depth should be >= 0" ); + params.maxDepth = std::min( params.maxDepth, 25 ); - CV_FUNCNAME( "CvDTreeTrainData::subsample_data" ); + params.minSampleCount = std::max(params.minSampleCount, 1); - __BEGIN__; + if( params.CVFolds < 0 ) + CV_Error( CV_StsOutOfRange, + "params.CVFolds should be =0 (the tree is not pruned) " + "or n>0 (tree is pruned using n-fold cross-validation)" ); - if( !data_root ) - CV_ERROR( CV_StsError, "No training data has been set" ); + if( params.CVFolds == 1 ) + params.CVFolds = 0; - if( _subsample_idx ) - { - CV_CALL( isubsample_idx = cvPreprocessIndexArray( _subsample_idx, sample_count )); + if( params.regressionAccuracy < 0 ) + CV_Error( CV_StsOutOfRange, "params.regression_accuracy should be >= 0" ); +} - if( isubsample_idx->cols + isubsample_idx->rows - 1 == sample_count ) - { - const int* sidx = isubsample_idx->data.i; - for( int i = 0; i < sample_count; i++ ) - { - if( sidx[i] != i ) - { - isMakeRootCopy = false; - break; - } - } - } - else - isMakeRootCopy = false; - } +int DTreesImpl::addNodeAndTrySplit( int parent, const vector& sidx ) +{ + w->wnodes.push_back(WNode()); + int nidx = (int)(w->wnodes.size() - 1); + WNode& node = w->wnodes.back(); + + node.parent = parent; + node.depth = parent >= 0 ? w->wnodes[parent].depth + 1 : 0; + int nfolds = params.CVFolds; - if( isMakeRootCopy ) + if( nfolds > 0 ) { - // make a copy of the root node - CvDTreeNode temp; - int i; - root = new_node( 0, 1, 0, 0 ); - temp = *root; - *root = *data_root; - root->num_valid = temp.num_valid; - if( root->num_valid ) - { - for( i = 0; i < var_count; i++ ) - root->num_valid[i] = data_root->num_valid[i]; - } - root->cv_Tn = temp.cv_Tn; - root->cv_node_risk = temp.cv_node_risk; - root->cv_node_error = temp.cv_node_error; + w->cv_Tn.resize((nidx+1)*nfolds); + w->cv_node_error.resize((nidx+1)*nfolds); + w->cv_node_risk.resize((nidx+1)*nfolds); } - else - { - int* sidx = isubsample_idx->data.i; - // co - array of count/offset pairs (to handle duplicated values in _subsample_idx) - int* co, cur_ofs = 0; - int vi, i; - int workVarCount = get_work_var_count(); - int count = isubsample_idx->rows + isubsample_idx->cols - 1; - - root = new_node( 0, count, 1, 0 ); - - CV_CALL( subsample_co = cvCreateMat( 1, sample_count*2, CV_32SC1 )); - cvZero( subsample_co ); - co = subsample_co->data.i; - for( i = 0; i < count; i++ ) - co[sidx[i]*2]++; - for( i = 0; i < sample_count; i++ ) - { - if( co[i*2] ) - { - co[i*2+1] = cur_ofs; - cur_ofs += co[i*2]; - } - else - co[i*2+1] = -1; - } - - cv::AutoBuffer inn_buf(sample_count*(2*sizeof(int) + sizeof(float))); - for( vi = 0; vi < workVarCount; vi++ ) - { - int ci = get_var_type(vi); - if( ci >= 0 || vi >= var_count ) - { - int num_valid = 0; - const int* src = CvDTreeTrainData::get_cat_var_data( data_root, vi, (int*)(uchar*)inn_buf ); + int i, n = node.sample_count = (int)sidx.size(); + bool can_split = true; + vector sleft, sright; - if (is_buf_16u) - { - unsigned short* udst = (unsigned short*)(buf->data.s + root->buf_idx*get_length_subbuf() + - vi*sample_count + root->offset); - for( i = 0; i < count; i++ ) - { - int val = src[sidx[i]]; - udst[i] = (unsigned short)val; - num_valid += val >= 0; - } - } - else - { - int* idst = buf->data.i + root->buf_idx*get_length_subbuf() + - vi*sample_count + root->offset; - for( i = 0; i < count; i++ ) - { - int val = src[sidx[i]]; - idst[i] = val; - num_valid += val >= 0; - } - } + calcValue( nidx, sidx ); - if( vi < var_count ) - root->set_num_valid(vi, num_valid); - } - else - { - int *src_idx_buf = (int*)(uchar*)inn_buf; - float *src_val_buf = (float*)(src_idx_buf + sample_count); - int* sample_indices_buf = (int*)(src_val_buf + sample_count); - const int* src_idx = 0; - const float* src_val = 0; - get_ord_var_data( data_root, vi, src_val_buf, src_idx_buf, &src_val, &src_idx, sample_indices_buf ); - int j = 0, idx, count_i; - int num_valid = data_root->get_num_valid(vi); - - if (is_buf_16u) - { - unsigned short* udst_idx = (unsigned short*)(buf->data.s + root->buf_idx*get_length_subbuf() + - vi*sample_count + data_root->offset); - for( i = 0; i < num_valid; i++ ) - { - idx = src_idx[i]; - count_i = co[idx*2]; - if( count_i ) - for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) - udst_idx[j] = (unsigned short)cur_ofs; - } + if( n <= params.minSampleCount || node.depth >= params.maxDepth ) + can_split = false; + else if( _isClassifier ) + { + const int* responses = &w->cat_responses[0]; + const int* s = &sidx[0]; + int first = responses[s[0]]; + for( i = 1; i < n; i++ ) + if( responses[s[i]] != first ) + break; + if( i == n ) + can_split = false; + } + else + { + if( sqrt(node.node_risk) < params.regressionAccuracy ) + can_split = false; + } - root->set_num_valid(vi, j); + if( can_split ) + node.split = findBestSplit( sidx ); - for( ; i < sample_count; i++ ) - { - idx = src_idx[i]; - count_i = co[idx*2]; - if( count_i ) - for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) - udst_idx[j] = (unsigned short)cur_ofs; - } - } - else - { - int* idst_idx = buf->data.i + root->buf_idx*get_length_subbuf() + - vi*sample_count + root->offset; - for( i = 0; i < num_valid; i++ ) - { - idx = src_idx[i]; - count_i = co[idx*2]; - if( count_i ) - for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) - idst_idx[j] = cur_ofs; - } + //printf("depth=%d, nidx=%d, parent=%d, n=%d, %s, value=%.1f, risk=%.1f\n", node.depth, nidx, node.parent, n, (node.split < 0 ? "leaf" : varType[w->wsplits[node.split].varIdx] == VAR_CATEGORICAL ? "cat" : "ord"), node.value, node.node_risk); - root->set_num_valid(vi, j); + if( node.split >= 0 ) + { + node.defaultDir = calcDir( node.split, sidx, sleft, sright ); + if( params.useSurrogates ) + CV_Error( CV_StsNotImplemented, "surrogate splits are not implemented yet"); - for( ; i < sample_count; i++ ) - { - idx = src_idx[i]; - count_i = co[idx*2]; - if( count_i ) - for( cur_ofs = co[idx*2+1]; count_i > 0; count_i--, j++, cur_ofs++ ) - idst_idx[j] = cur_ofs; - } - } - } - } - // sample indices subsampling - const int* sample_idx_src = get_sample_indices(data_root, (int*)(uchar*)inn_buf); - if (is_buf_16u) - { - unsigned short* sample_idx_dst = (unsigned short*)(buf->data.s + root->buf_idx*get_length_subbuf() + - workVarCount*sample_count + root->offset); - for (i = 0; i < count; i++) - sample_idx_dst[i] = (unsigned short)sample_idx_src[sidx[i]]; - } - else - { - int* sample_idx_dst = buf->data.i + root->buf_idx*get_length_subbuf() + - workVarCount*sample_count + root->offset; - for (i = 0; i < count; i++) - sample_idx_dst[i] = sample_idx_src[sidx[i]]; - } + int left = addNodeAndTrySplit( nidx, sleft ); + int right = addNodeAndTrySplit( nidx, sright ); + w->wnodes[nidx].left = left; + w->wnodes[nidx].right = right; + CV_Assert( w->wnodes[nidx].left > 0 && w->wnodes[nidx].right > 0 ); } - __END__; - - cvReleaseMat( &isubsample_idx ); - cvReleaseMat( &subsample_co ); - - return root; + return nidx; } - -void CvDTreeTrainData::get_vectors( const CvMat* _subsample_idx, - float* values, uchar* missing, - float* _responses, bool get_class_idx ) +int DTreesImpl::findBestSplit( const vector& _sidx ) { - CvMat* subsample_idx = 0; - CvMat* subsample_co = 0; - - CV_FUNCNAME( "CvDTreeTrainData::get_vectors" ); - - __BEGIN__; + const vector& activeVars = getActiveVars(); + int splitidx = -1; + int vi_, nv = (int)activeVars.size(); + AutoBuffer buf(w->maxSubsetSize*2); + int *subset = buf, *best_subset = subset + w->maxSubsetSize; + WSplit split, best_split; + best_split.quality = 0.; - int i, vi, total = sample_count, count = total, cur_ofs = 0; - int* sidx = 0; - int* co = 0; - - cv::AutoBuffer inn_buf(sample_count*(2*sizeof(int) + sizeof(float))); - if( _subsample_idx ) - { - CV_CALL( subsample_idx = cvPreprocessIndexArray( _subsample_idx, sample_count )); - sidx = subsample_idx->data.i; - CV_CALL( subsample_co = cvCreateMat( 1, sample_count*2, CV_32SC1 )); - co = subsample_co->data.i; - cvZero( subsample_co ); - count = subsample_idx->cols + subsample_idx->rows - 1; - for( i = 0; i < count; i++ ) - co[sidx[i]*2]++; - for( i = 0; i < total; i++ ) - { - int count_i = co[i*2]; - if( count_i ) - { - co[i*2+1] = cur_ofs*var_count; - cur_ofs += count_i; - } - } - } - - if( missing ) - memset( missing, 1, count*var_count ); - - for( vi = 0; vi < var_count; vi++ ) + for( vi_ = 0; vi_ < nv; vi_++ ) { - int ci = get_var_type(vi); - if( ci >= 0 ) // categorical + int vi = activeVars[vi_]; + if( varType[vi] == VAR_CATEGORICAL ) { - float* dst = values + vi; - uchar* m = missing ? missing + vi : 0; - const int* src = get_cat_var_data(data_root, vi, (int*)(uchar*)inn_buf); - - for( i = 0; i < count; i++, dst += var_count ) - { - int idx = sidx ? sidx[i] : i; - int val = src[idx]; - *dst = (float)val; - if( m ) - { - *m = (!is_buf_16u && val < 0) || (is_buf_16u && (val == 65535)); - m += var_count; - } - } - } - else // ordered - { - float* dst = values + vi; - uchar* m = missing ? missing + vi : 0; - int count1 = data_root->get_num_valid(vi); - float *src_val_buf = (float*)(uchar*)inn_buf; - int* src_idx_buf = (int*)(src_val_buf + sample_count); - int* sample_indices_buf = src_idx_buf + sample_count; - const float *src_val = 0; - const int* src_idx = 0; - get_ord_var_data(data_root, vi, src_val_buf, src_idx_buf, &src_val, &src_idx, sample_indices_buf); - - for( i = 0; i < count1; i++ ) - { - int idx = src_idx[i]; - int count_i = 1; - if( co ) - { - count_i = co[idx*2]; - cur_ofs = co[idx*2+1]; - } - else - cur_ofs = idx*var_count; - if( count_i ) - { - float val = src_val[i]; - for( ; count_i > 0; count_i--, cur_ofs += var_count ) - { - dst[cur_ofs] = val; - if( m ) - m[cur_ofs] = 0; - } - } - } - } - } - - // copy responses - if( _responses ) - { - if( is_classifier ) - { - const int* src = get_class_labels(data_root, (int*)(uchar*)inn_buf); - for( i = 0; i < count; i++ ) - { - int idx = sidx ? sidx[i] : i; - int val = get_class_idx ? src[idx] : - cat_map->data.i[cat_ofs->data.i[cat_var_count]+src[idx]]; - _responses[i] = (float)val; - } - } - else - { - float* val_buf = (float*)(uchar*)inn_buf; - int* sample_idx_buf = (int*)(val_buf + sample_count); - const float* _values = get_ord_responses(data_root, val_buf, sample_idx_buf); - for( i = 0; i < count; i++ ) - { - int idx = sidx ? sidx[i] : i; - _responses[i] = _values[idx]; - } - } - } - - __END__; - - cvReleaseMat( &subsample_idx ); - cvReleaseMat( &subsample_co ); -} - - -CvDTreeNode* CvDTreeTrainData::new_node( CvDTreeNode* parent, int count, - int storage_idx, int offset ) -{ - CvDTreeNode* node = (CvDTreeNode*)cvSetNew( node_heap ); - - node->sample_count = count; - node->depth = parent ? parent->depth + 1 : 0; - node->parent = parent; - node->left = node->right = 0; - node->split = 0; - node->value = 0; - node->class_idx = 0; - node->maxlr = 0.; - - node->buf_idx = storage_idx; - node->offset = offset; - if( nv_heap ) - node->num_valid = (int*)cvSetNew( nv_heap ); - else - node->num_valid = 0; - node->alpha = node->node_risk = node->tree_risk = node->tree_error = 0.; - node->complexity = 0; - - if( params.cv_folds > 0 && cv_heap ) - { - int cv_n = params.cv_folds; - node->Tn = INT_MAX; - node->cv_Tn = (int*)cvSetNew( cv_heap ); - node->cv_node_risk = (double*)cvAlignPtr(node->cv_Tn + cv_n, sizeof(double)); - node->cv_node_error = node->cv_node_risk + cv_n; - } - else - { - node->Tn = 0; - node->cv_Tn = 0; - node->cv_node_risk = 0; - node->cv_node_error = 0; - } - - return node; -} - - -CvDTreeSplit* CvDTreeTrainData::new_split_ord( int vi, float cmp_val, - int split_point, int inversed, float quality ) -{ - CvDTreeSplit* split = (CvDTreeSplit*)cvSetNew( split_heap ); - split->var_idx = vi; - split->condensed_idx = INT_MIN; - split->ord.c = cmp_val; - split->ord.split_point = split_point; - split->inversed = inversed; - split->quality = quality; - split->next = 0; - - return split; -} - - -CvDTreeSplit* CvDTreeTrainData::new_split_cat( int vi, float quality ) -{ - CvDTreeSplit* split = (CvDTreeSplit*)cvSetNew( split_heap ); - int i, n = (max_c_count + 31)/32; - - split->var_idx = vi; - split->condensed_idx = INT_MIN; - split->inversed = 0; - split->quality = quality; - for( i = 0; i < n; i++ ) - split->subset[i] = 0; - split->next = 0; - - return split; -} - - -void CvDTreeTrainData::free_node( CvDTreeNode* node ) -{ - CvDTreeSplit* split = node->split; - free_node_data( node ); - while( split ) - { - CvDTreeSplit* next = split->next; - cvSetRemoveByPtr( split_heap, split ); - split = next; - } - node->split = 0; - cvSetRemoveByPtr( node_heap, node ); -} - - -void CvDTreeTrainData::free_node_data( CvDTreeNode* node ) -{ - if( node->num_valid ) - { - cvSetRemoveByPtr( nv_heap, node->num_valid ); - node->num_valid = 0; - } - // do not free cv_* fields, as all the cross-validation related data is released at once. -} - - -void CvDTreeTrainData::free_train_data() -{ - cvReleaseMat( &counts ); - cvReleaseMat( &buf ); - cvReleaseMat( &direction ); - cvReleaseMat( &split_buf ); - cvReleaseMemStorage( &temp_storage ); - cvReleaseMat( &responses_copy ); - cv_heap = nv_heap = 0; -} - - -void CvDTreeTrainData::clear() -{ - free_train_data(); - - cvReleaseMemStorage( &tree_storage ); - - cvReleaseMat( &var_idx ); - cvReleaseMat( &var_type ); - cvReleaseMat( &cat_count ); - cvReleaseMat( &cat_ofs ); - cvReleaseMat( &cat_map ); - cvReleaseMat( &priors ); - cvReleaseMat( &priors_mult ); - - node_heap = split_heap = 0; - - sample_count = var_all = var_count = max_c_count = ord_var_count = cat_var_count = 0; - have_labels = have_priors = is_classifier = false; - - buf_count = buf_size = 0; - shared = false; - - data_root = 0; - - rng = &cv::theRNG(); -} - - -int CvDTreeTrainData::get_num_classes() const -{ - return is_classifier ? cat_count->data.i[cat_var_count] : 0; -} - - -int CvDTreeTrainData::get_var_type(int vi) const -{ - return var_type->data.i[vi]; -} - -void CvDTreeTrainData::get_ord_var_data( CvDTreeNode* n, int vi, float* ord_values_buf, int* sorted_indices_buf, - const float** ord_values, const int** sorted_indices, int* sample_indices_buf ) -{ - int vidx = var_idx ? var_idx->data.i[vi] : vi; - int node_sample_count = n->sample_count; - int td_step = train_data->step/CV_ELEM_SIZE(train_data->type); - - const int* sample_indices = get_sample_indices(n, sample_indices_buf); - - if( !is_buf_16u ) - *sorted_indices = buf->data.i + n->buf_idx*get_length_subbuf() + - vi*sample_count + n->offset; - else { - const unsigned short* short_indices = (const unsigned short*)(buf->data.s + n->buf_idx*get_length_subbuf() + - vi*sample_count + n->offset ); - for( int i = 0; i < node_sample_count; i++ ) - sorted_indices_buf[i] = short_indices[i]; - *sorted_indices = sorted_indices_buf; - } - - if( tflag == CV_ROW_SAMPLE ) - { - for( int i = 0; i < node_sample_count && - ((((*sorted_indices)[i] >= 0) && !is_buf_16u) || (((*sorted_indices)[i] != 65535) && is_buf_16u)); i++ ) - { - int idx = (*sorted_indices)[i]; - idx = sample_indices[idx]; - ord_values_buf[i] = *(train_data->data.fl + idx * td_step + vidx); - } - } - else - for( int i = 0; i < node_sample_count && - ((((*sorted_indices)[i] >= 0) && !is_buf_16u) || (((*sorted_indices)[i] != 65535) && is_buf_16u)); i++ ) - { - int idx = (*sorted_indices)[i]; - idx = sample_indices[idx]; - ord_values_buf[i] = *(train_data->data.fl + vidx* td_step + idx); - } - - *ord_values = ord_values_buf; -} - - -const int* CvDTreeTrainData::get_class_labels( CvDTreeNode* n, int* labels_buf ) -{ - if (is_classifier) - return get_cat_var_data( n, var_count, labels_buf); - return 0; -} - -const int* CvDTreeTrainData::get_sample_indices( CvDTreeNode* n, int* indices_buf ) -{ - return get_cat_var_data( n, get_work_var_count(), indices_buf ); -} - -const float* CvDTreeTrainData::get_ord_responses( CvDTreeNode* n, float* values_buf, int*sample_indices_buf ) -{ - int _sample_count = n->sample_count; - int r_step = CV_IS_MAT_CONT(responses->type) ? 1 : responses->step/CV_ELEM_SIZE(responses->type); - const int* indices = get_sample_indices(n, sample_indices_buf); - - for( int i = 0; i < _sample_count && - (((indices[i] >= 0) && !is_buf_16u) || ((indices[i] != 65535) && is_buf_16u)); i++ ) - { - int idx = indices[i]; - values_buf[i] = *(responses->data.fl + idx * r_step); - } - - return values_buf; -} - - -const int* CvDTreeTrainData::get_cv_labels( CvDTreeNode* n, int* labels_buf ) -{ - if (have_labels) - return get_cat_var_data( n, get_work_var_count()- 1, labels_buf); - return 0; -} - - -const int* CvDTreeTrainData::get_cat_var_data( CvDTreeNode* n, int vi, int* cat_values_buf) -{ - const int* cat_values = 0; - if( !is_buf_16u ) - cat_values = buf->data.i + n->buf_idx*get_length_subbuf() + - vi*sample_count + n->offset; - else { - const unsigned short* short_values = (const unsigned short*)(buf->data.s + n->buf_idx*get_length_subbuf() + - vi*sample_count + n->offset); - for( int i = 0; i < n->sample_count; i++ ) - cat_values_buf[i] = short_values[i]; - cat_values = cat_values_buf; - } - return cat_values; -} - - -int CvDTreeTrainData::get_child_buf_idx( CvDTreeNode* n ) -{ - int idx = n->buf_idx + 1; - if( idx >= buf_count ) - idx = shared ? 1 : 0; - return idx; -} - - -void CvDTreeTrainData::write_params( CvFileStorage* fs ) const -{ - CV_FUNCNAME( "CvDTreeTrainData::write_params" ); - - __BEGIN__; - - int vi, vcount = var_count; - - cvWriteInt( fs, "is_classifier", is_classifier ? 1 : 0 ); - cvWriteInt( fs, "var_all", var_all ); - cvWriteInt( fs, "var_count", var_count ); - cvWriteInt( fs, "ord_var_count", ord_var_count ); - cvWriteInt( fs, "cat_var_count", cat_var_count ); - - cvStartWriteStruct( fs, "training_params", CV_NODE_MAP ); - cvWriteInt( fs, "use_surrogates", params.use_surrogates ? 1 : 0 ); - - if( is_classifier ) - { - cvWriteInt( fs, "max_categories", params.max_categories ); - } - else - { - cvWriteReal( fs, "regression_accuracy", params.regression_accuracy ); - } - - cvWriteInt( fs, "max_depth", params.max_depth ); - cvWriteInt( fs, "min_sample_count", params.min_sample_count ); - cvWriteInt( fs, "cross_validation_folds", params.cv_folds ); - - if( params.cv_folds > 1 ) - { - cvWriteInt( fs, "use_1se_rule", params.use_1se_rule ? 1 : 0 ); - cvWriteInt( fs, "truncate_pruned_tree", params.truncate_pruned_tree ? 1 : 0 ); - } - - if( priors ) - cvWrite( fs, "priors", priors ); - - cvEndWriteStruct( fs ); - - if( var_idx ) - cvWrite( fs, "var_idx", var_idx ); - - cvStartWriteStruct( fs, "var_type", CV_NODE_SEQ+CV_NODE_FLOW ); - - for( vi = 0; vi < vcount; vi++ ) - cvWriteInt( fs, 0, var_type->data.i[vi] >= 0 ); - - cvEndWriteStruct( fs ); - - if( cat_count && (cat_var_count > 0 || is_classifier) ) - { - CV_ASSERT( cat_count != 0 ); - cvWrite( fs, "cat_count", cat_count ); - cvWrite( fs, "cat_map", cat_map ); - } - - __END__; -} - - -void CvDTreeTrainData::read_params( CvFileStorage* fs, CvFileNode* node ) -{ - CV_FUNCNAME( "CvDTreeTrainData::read_params" ); - - __BEGIN__; - - CvFileNode *tparams_node, *vartype_node; - CvSeqReader reader; - int vi, max_split_size, tree_block_size; - - is_classifier = (cvReadIntByName( fs, node, "is_classifier" ) != 0); - var_all = cvReadIntByName( fs, node, "var_all" ); - var_count = cvReadIntByName( fs, node, "var_count", var_all ); - cat_var_count = cvReadIntByName( fs, node, "cat_var_count" ); - ord_var_count = cvReadIntByName( fs, node, "ord_var_count" ); - - tparams_node = cvGetFileNodeByName( fs, node, "training_params" ); - - if( tparams_node ) // training parameters are not necessary - { - params.use_surrogates = cvReadIntByName( fs, tparams_node, "use_surrogates", 1 ) != 0; - - if( is_classifier ) - { - params.max_categories = cvReadIntByName( fs, tparams_node, "max_categories" ); - } - else - { - params.regression_accuracy = - (float)cvReadRealByName( fs, tparams_node, "regression_accuracy" ); - } - - params.max_depth = cvReadIntByName( fs, tparams_node, "max_depth" ); - params.min_sample_count = cvReadIntByName( fs, tparams_node, "min_sample_count" ); - params.cv_folds = cvReadIntByName( fs, tparams_node, "cross_validation_folds" ); - - if( params.cv_folds > 1 ) - { - params.use_1se_rule = cvReadIntByName( fs, tparams_node, "use_1se_rule" ) != 0; - params.truncate_pruned_tree = - cvReadIntByName( fs, tparams_node, "truncate_pruned_tree" ) != 0; - } - - priors = (CvMat*)cvReadByName( fs, tparams_node, "priors" ); - if( priors ) - { - if( !CV_IS_MAT(priors) ) - CV_ERROR( CV_StsParseError, "priors must stored as a matrix" ); - priors_mult = cvCloneMat( priors ); - } - } - - CV_CALL( var_idx = (CvMat*)cvReadByName( fs, node, "var_idx" )); - if( var_idx ) - { - if( !CV_IS_MAT(var_idx) || - (var_idx->cols != 1 && var_idx->rows != 1) || - var_idx->cols + var_idx->rows - 1 != var_count || - CV_MAT_TYPE(var_idx->type) != CV_32SC1 ) - CV_ERROR( CV_StsParseError, - "var_idx (if exist) must be valid 1d integer vector containing elements" ); - - for( vi = 0; vi < var_count; vi++ ) - if( (unsigned)var_idx->data.i[vi] >= (unsigned)var_all ) - CV_ERROR( CV_StsOutOfRange, "some of var_idx elements are out of range" ); - } - - ////// read var type - CV_CALL( var_type = cvCreateMat( 1, var_count + 2, CV_32SC1 )); - - cat_var_count = 0; - ord_var_count = -1; - vartype_node = cvGetFileNodeByName( fs, node, "var_type" ); - - if( vartype_node && CV_NODE_TYPE(vartype_node->tag) == CV_NODE_INT && var_count == 1 ) - var_type->data.i[0] = vartype_node->data.i ? cat_var_count++ : ord_var_count--; - else - { - if( !vartype_node || CV_NODE_TYPE(vartype_node->tag) != CV_NODE_SEQ || - vartype_node->data.seq->total != var_count ) - CV_ERROR( CV_StsParseError, "var_type must exist and be a sequence of 0's and 1's" ); - - cvStartReadSeq( vartype_node->data.seq, &reader ); - - for( vi = 0; vi < var_count; vi++ ) - { - CvFileNode* n = (CvFileNode*)reader.ptr; - if( CV_NODE_TYPE(n->tag) != CV_NODE_INT || (n->data.i & ~1) ) - CV_ERROR( CV_StsParseError, "var_type must exist and be a sequence of 0's and 1's" ); - var_type->data.i[vi] = n->data.i ? cat_var_count++ : ord_var_count--; - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); - } - } - var_type->data.i[var_count] = cat_var_count; - - ord_var_count = ~ord_var_count; - ////// - - if( cat_var_count > 0 || is_classifier ) - { - int ccount, total_c_count = 0; - CV_CALL( cat_count = (CvMat*)cvReadByName( fs, node, "cat_count" )); - CV_CALL( cat_map = (CvMat*)cvReadByName( fs, node, "cat_map" )); - - if( !CV_IS_MAT(cat_count) || !CV_IS_MAT(cat_map) || - (cat_count->cols != 1 && cat_count->rows != 1) || - CV_MAT_TYPE(cat_count->type) != CV_32SC1 || - cat_count->cols + cat_count->rows - 1 != cat_var_count + is_classifier || - (cat_map->cols != 1 && cat_map->rows != 1) || - CV_MAT_TYPE(cat_map->type) != CV_32SC1 ) - CV_ERROR( CV_StsParseError, - "Both cat_count and cat_map must exist and be valid 1d integer vectors of an appropriate size" ); - - ccount = cat_var_count + is_classifier; - - CV_CALL( cat_ofs = cvCreateMat( 1, ccount + 1, CV_32SC1 )); - cat_ofs->data.i[0] = 0; - max_c_count = 1; - - for( vi = 0; vi < ccount; vi++ ) - { - int val = cat_count->data.i[vi]; - if( val <= 0 ) - CV_ERROR( CV_StsOutOfRange, "some of cat_count elements are out of range" ); - max_c_count = MAX( max_c_count, val ); - cat_ofs->data.i[vi+1] = total_c_count += val; - } - - if( cat_map->cols + cat_map->rows - 1 != total_c_count ) - CV_ERROR( CV_StsBadSize, - "cat_map vector length is not equal to the total number of categories in all categorical vars" ); - } - - max_split_size = cvAlign(sizeof(CvDTreeSplit) + - (MAX(0,max_c_count - 33)/32)*sizeof(int),sizeof(void*)); - - tree_block_size = MAX((int)sizeof(CvDTreeNode)*8, max_split_size); - tree_block_size = MAX(tree_block_size + block_size_delta, min_block_size); - CV_CALL( tree_storage = cvCreateMemStorage( tree_block_size )); - CV_CALL( node_heap = cvCreateSet( 0, sizeof(node_heap[0]), - sizeof(CvDTreeNode), tree_storage )); - CV_CALL( split_heap = cvCreateSet( 0, sizeof(split_heap[0]), - max_split_size, tree_storage )); - - __END__; -} - -/////////////////////// Decision Tree ///////////////////////// -CvDTreeParams::CvDTreeParams() : max_categories(10), max_depth(INT_MAX), min_sample_count(10), - cv_folds(10), use_surrogates(true), use_1se_rule(true), - truncate_pruned_tree(true), regression_accuracy(0.01f), priors(0) -{} - -CvDTreeParams::CvDTreeParams( int _max_depth, int _min_sample_count, - float _regression_accuracy, bool _use_surrogates, - int _max_categories, int _cv_folds, - bool _use_1se_rule, bool _truncate_pruned_tree, - const float* _priors ) : - max_categories(_max_categories), max_depth(_max_depth), - min_sample_count(_min_sample_count), cv_folds (_cv_folds), - use_surrogates(_use_surrogates), use_1se_rule(_use_1se_rule), - truncate_pruned_tree(_truncate_pruned_tree), - regression_accuracy(_regression_accuracy), - priors(_priors) -{} - -CvDTree::CvDTree() -{ - data = 0; - var_importance = 0; - default_model_name = "my_tree"; - - clear(); -} - - -void CvDTree::clear() -{ - cvReleaseMat( &var_importance ); - if( data ) - { - if( !data->shared ) - delete data; - else - free_tree(); - data = 0; - } - root = 0; - pruned_tree_idx = -1; -} - - -CvDTree::~CvDTree() -{ - clear(); -} - - -const CvDTreeNode* CvDTree::get_root() const -{ - return root; -} - - -int CvDTree::get_pruned_tree_idx() const -{ - return pruned_tree_idx; -} - - -CvDTreeTrainData* CvDTree::get_data() -{ - return data; -} - - -bool CvDTree::train( const CvMat* _train_data, int _tflag, - const CvMat* _responses, const CvMat* _var_idx, - const CvMat* _sample_idx, const CvMat* _var_type, - const CvMat* _missing_mask, CvDTreeParams _params ) -{ - bool result = false; - - CV_FUNCNAME( "CvDTree::train" ); - - __BEGIN__; - - clear(); - data = new CvDTreeTrainData( _train_data, _tflag, _responses, - _var_idx, _sample_idx, _var_type, - _missing_mask, _params, false ); - CV_CALL( result = do_train(0) ); - - __END__; - - return result; -} - -bool CvDTree::train( const Mat& _train_data, int _tflag, - const Mat& _responses, const Mat& _var_idx, - const Mat& _sample_idx, const Mat& _var_type, - const Mat& _missing_mask, CvDTreeParams _params ) -{ - train_data_hdr = _train_data; - train_data_mat = _train_data; - responses_hdr = _responses; - responses_mat = _responses; - - CvMat vidx=_var_idx, sidx=_sample_idx, vtype=_var_type, mmask=_missing_mask; - - return train(&train_data_hdr, _tflag, &responses_hdr, vidx.data.ptr ? &vidx : 0, sidx.data.ptr ? &sidx : 0, - vtype.data.ptr ? &vtype : 0, mmask.data.ptr ? &mmask : 0, _params); -} - - -bool CvDTree::train( CvMLData* _data, CvDTreeParams _params ) -{ - bool result = false; - - CV_FUNCNAME( "CvDTree::train" ); - - __BEGIN__; - - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* missing = _data->get_missing(); - const CvMat* var_types = _data->get_var_types(); - const CvMat* train_sidx = _data->get_train_sample_idx(); - const CvMat* var_idx = _data->get_var_idx(); - - CV_CALL( result = train( values, CV_ROW_SAMPLE, response, var_idx, - train_sidx, var_types, missing, _params ) ); - - __END__; - - return result; -} - -bool CvDTree::train( CvDTreeTrainData* _data, const CvMat* _subsample_idx ) -{ - bool result = false; - - CV_FUNCNAME( "CvDTree::train" ); - - __BEGIN__; - - clear(); - data = _data; - data->shared = true; - CV_CALL( result = do_train(_subsample_idx)); - - __END__; - - return result; -} - - -bool CvDTree::do_train( const CvMat* _subsample_idx ) -{ - bool result = false; - - CV_FUNCNAME( "CvDTree::do_train" ); - - __BEGIN__; - - root = data->subsample_data( _subsample_idx ); - - CV_CALL( try_split_node(root)); - - if( root->split ) - { - CV_Assert( root->left ); - CV_Assert( root->right ); - - if( data->params.cv_folds > 0 ) - CV_CALL( prune_cv() ); - - if( !data->shared ) - data->free_train_data(); - - result = true; - } - - __END__; - - return result; -} - - -void CvDTree::try_split_node( CvDTreeNode* node ) -{ - CvDTreeSplit* best_split = 0; - int i, n = node->sample_count, vi; - bool can_split = true; - double quality_scale; - - calc_node_value( node ); - - if( node->sample_count <= data->params.min_sample_count || - node->depth >= data->params.max_depth ) - can_split = false; - - if( can_split && data->is_classifier ) - { - // check if we have a "pure" node, - // we assume that cls_count is filled by calc_node_value() - int* cls_count = data->counts->data.i; - int nz = 0, m = data->get_num_classes(); - for( i = 0; i < m; i++ ) - nz += cls_count[i] != 0; - if( nz == 1 ) // there is only one class - can_split = false; - } - else if( can_split ) - { - if( sqrt(node->node_risk)/n < data->params.regression_accuracy ) - can_split = false; - } - - if( can_split ) - { - best_split = find_best_split(node); - // TODO: check the split quality ... - node->split = best_split; - } - if( !can_split || !best_split ) - { - data->free_node_data(node); - return; - } - - quality_scale = calc_node_dir( node ); - if( data->params.use_surrogates ) - { - // find all the surrogate splits - // and sort them by their similarity to the primary one - for( vi = 0; vi < data->var_count; vi++ ) - { - CvDTreeSplit* split; - int ci = data->get_var_type(vi); - - if( vi == best_split->var_idx ) - continue; - - if( ci >= 0 ) - split = find_surrogate_split_cat( node, vi ); - else - split = find_surrogate_split_ord( node, vi ); - - if( split ) - { - // insert the split - CvDTreeSplit* prev_split = node->split; - split->quality = (float)(split->quality*quality_scale); - - while( prev_split->next && - prev_split->next->quality > split->quality ) - prev_split = prev_split->next; - split->next = prev_split->next; - prev_split->next = split; - } - } - } - split_node_data( node ); - try_split_node( node->left ); - try_split_node( node->right ); -} - - -// calculate direction (left(-1),right(1),missing(0)) -// for each sample using the best split -// the function returns scale coefficients for surrogate split quality factors. -// the scale is applied to normalize surrogate split quality relatively to the -// best (primary) split quality. That is, if a surrogate split is absolutely -// identical to the primary split, its quality will be set to the maximum value = -// quality of the primary split; otherwise, it will be lower. -// besides, the function compute node->maxlr, -// minimum possible quality (w/o considering the above mentioned scale) -// for a surrogate split. Surrogate splits with quality less than node->maxlr -// are not discarded. -double CvDTree::calc_node_dir( CvDTreeNode* node ) -{ - char* dir = (char*)data->direction->data.ptr; - int i, n = node->sample_count, vi = node->split->var_idx; - double L, R; - - assert( !node->split->inversed ); - - if( data->get_var_type(vi) >= 0 ) // split on categorical var - { - cv::AutoBuffer inn_buf(n*(!data->have_priors ? 1 : 2)); - int* labels_buf = (int*)inn_buf; - const int* labels = data->get_cat_var_data( node, vi, labels_buf ); - const int* subset = node->split->subset; - if( !data->have_priors ) - { - int sum = 0, sum_abs = 0; - - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - int d = ( ((idx >= 0)&&(!data->is_buf_16u)) || ((idx != 65535)&&(data->is_buf_16u)) ) ? - CV_DTREE_CAT_DIR(idx,subset) : 0; - sum += d; sum_abs += d & 1; - dir[i] = (char)d; - } - - R = (sum_abs + sum) >> 1; - L = (sum_abs - sum) >> 1; - } - else - { - const double* priors = data->priors_mult->data.db; - double sum = 0, sum_abs = 0; - int* responses_buf = labels_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - double w = priors[responses[i]]; - int d = idx >= 0 ? CV_DTREE_CAT_DIR(idx,subset) : 0; - sum += d*w; sum_abs += (d & 1)*w; - dir[i] = (char)d; - } - - R = (sum_abs + sum) * 0.5; - L = (sum_abs - sum) * 0.5; - } - } - else // split on ordered var - { - int split_point = node->split->ord.split_point; - int n1 = node->get_num_valid(vi); - cv::AutoBuffer inn_buf(n*(sizeof(int)*(data->have_priors ? 3 : 2) + sizeof(float))); - float* val_buf = (float*)(uchar*)inn_buf; - int* sorted_buf = (int*)(val_buf + n); - int* sample_idx_buf = sorted_buf + n; - const float* val = 0; - const int* sorted = 0; - data->get_ord_var_data( node, vi, val_buf, sorted_buf, &val, &sorted, sample_idx_buf); - - assert( 0 <= split_point && split_point < n1-1 ); - - if( !data->have_priors ) - { - for( i = 0; i <= split_point; i++ ) - dir[sorted[i]] = (char)-1; - for( ; i < n1; i++ ) - dir[sorted[i]] = (char)1; - for( ; i < n; i++ ) - dir[sorted[i]] = (char)0; - - L = split_point-1; - R = n1 - split_point + 1; - } - else - { - const double* priors = data->priors_mult->data.db; - int* responses_buf = sample_idx_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - L = R = 0; - - for( i = 0; i <= split_point; i++ ) - { - int idx = sorted[i]; - double w = priors[responses[idx]]; - dir[idx] = (char)-1; - L += w; - } - - for( ; i < n1; i++ ) - { - int idx = sorted[i]; - double w = priors[responses[idx]]; - dir[idx] = (char)1; - R += w; - } - - for( ; i < n; i++ ) - dir[sorted[i]] = (char)0; - } - } - node->maxlr = MAX( L, R ); - return node->split->quality/(L + R); -} - - -namespace cv -{ - -template<> CV_EXPORTS void DefaultDeleter::operator ()(CvDTreeSplit* obj) const -{ - fastFree(obj); -} - -DTreeBestSplitFinder::DTreeBestSplitFinder( CvDTree* _tree, CvDTreeNode* _node) -{ - tree = _tree; - node = _node; - splitSize = tree->get_data()->split_heap->elem_size; - - bestSplit.reset((CvDTreeSplit*)fastMalloc(splitSize)); - memset(bestSplit.get(), 0, splitSize); - bestSplit->quality = -1; - bestSplit->condensed_idx = INT_MIN; - split.reset((CvDTreeSplit*)fastMalloc(splitSize)); - memset(split.get(), 0, splitSize); - //haveSplit = false; -} - -DTreeBestSplitFinder::DTreeBestSplitFinder( const DTreeBestSplitFinder& finder, Split ) -{ - tree = finder.tree; - node = finder.node; - splitSize = tree->get_data()->split_heap->elem_size; - - bestSplit.reset((CvDTreeSplit*)fastMalloc(splitSize)); - memcpy(bestSplit.get(), finder.bestSplit.get(), splitSize); - split.reset((CvDTreeSplit*)fastMalloc(splitSize)); - memset(split.get(), 0, splitSize); -} - -void DTreeBestSplitFinder::operator()(const BlockedRange& range) -{ - int vi, vi1 = range.begin(), vi2 = range.end(); - int n = node->sample_count; - CvDTreeTrainData* data = tree->get_data(); - AutoBuffer inn_buf(2*n*(sizeof(int) + sizeof(float))); - - for( vi = vi1; vi < vi2; vi++ ) - { - CvDTreeSplit *res; - int ci = data->get_var_type(vi); - if( node->get_num_valid(vi) <= 1 ) - continue; - - if( data->is_classifier ) - { - if( ci >= 0 ) - res = tree->find_split_cat_class( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - else - res = tree->find_split_ord_class( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - } - else - { - if( ci >= 0 ) - res = tree->find_split_cat_reg( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - else - res = tree->find_split_ord_reg( node, vi, bestSplit->quality, split, (uchar*)inn_buf ); - } - - if( res && bestSplit->quality < split->quality ) - memcpy( bestSplit.get(), split.get(), splitSize ); - } -} - -void DTreeBestSplitFinder::join( DTreeBestSplitFinder& rhs ) -{ - if( bestSplit->quality < rhs.bestSplit->quality ) - memcpy( bestSplit.get(), rhs.bestSplit.get(), splitSize ); -} -} - - -CvDTreeSplit* CvDTree::find_best_split( CvDTreeNode* node ) -{ - DTreeBestSplitFinder finder( this, node ); - - cv::parallel_reduce(cv::BlockedRange(0, data->var_count), finder); - - CvDTreeSplit *bestSplit = 0; - if( finder.bestSplit->quality > 0 ) - { - bestSplit = data->new_split_cat( 0, -1.0f ); - memcpy( bestSplit, finder.bestSplit, finder.splitSize ); - } - - return bestSplit; -} - -CvDTreeSplit* CvDTree::find_split_ord_class( CvDTreeNode* node, int vi, - float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - int n = node->sample_count; - int n1 = node->get_num_valid(vi); - int m = data->get_num_classes(); - - int base_size = 2*m*sizeof(int); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + n*(3*sizeof(int)+sizeof(float))); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - float* values_buf = (float*)ext_buf; - int* sorted_indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = sorted_indices_buf + n; - const float* values = 0; - const int* sorted_indices = 0; - data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, - &sorted_indices, sample_indices_buf ); - int* responses_buf = sample_indices_buf + n; - const int* responses = data->get_class_labels( node, responses_buf ); - - const int* rc0 = data->counts->data.i; - int* lc = (int*)base_buf; - int* rc = lc + m; - int i, best_i = -1; - double lsum2 = 0, rsum2 = 0, best_val = init_quality; - const double* priors = data->have_priors ? data->priors_mult->data.db : 0; - - // init arrays of class instance counters on both sides of the split - for( i = 0; i < m; i++ ) - { - lc[i] = 0; - rc[i] = rc0[i]; - } - - // compensate for missing values - for( i = n1; i < n; i++ ) - { - rc[responses[sorted_indices[i]]]--; - } - - if( !priors ) - { - int L = 0, R = n1; - - for( i = 0; i < m; i++ ) - rsum2 += (double)rc[i]*rc[i]; - - for( i = 0; i < n1 - 1; i++ ) - { - int idx = responses[sorted_indices[i]]; - int lv, rv; - L++; R--; - lv = lc[idx]; rv = rc[idx]; - lsum2 += lv*2 + 1; - rsum2 -= rv*2 - 1; - lc[idx] = lv + 1; rc[idx] = rv - 1; - - if( values[i] + epsilon < values[i+1] ) - { - double val = (lsum2*R + rsum2*L)/((double)L*R); - if( best_val < val ) - { - best_val = val; - best_i = i; - } - } - } - } - else - { - double L = 0, R = 0; - for( i = 0; i < m; i++ ) - { - double wv = rc[i]*priors[i]; - R += wv; - rsum2 += wv*wv; - } - - for( i = 0; i < n1 - 1; i++ ) - { - int idx = responses[sorted_indices[i]]; - int lv, rv; - double p = priors[idx], p2 = p*p; - L += p; R -= p; - lv = lc[idx]; rv = rc[idx]; - lsum2 += p2*(lv*2 + 1); - rsum2 -= p2*(rv*2 - 1); - lc[idx] = lv + 1; rc[idx] = rv - 1; - - if( values[i] + epsilon < values[i+1] ) - { - double val = (lsum2*R + rsum2*L)/((double)L*R); - if( best_val < val ) - { - best_val = val; - best_i = i; - } - } - } - } - - CvDTreeSplit* split = 0; - if( best_i >= 0 ) - { - split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); - split->var_idx = vi; - split->ord.c = (values[best_i] + values[best_i+1])*0.5f; - split->ord.split_point = best_i; - split->inversed = 0; - split->quality = (float)best_val; - } - return split; -} - - -void CvDTree::cluster_categories( const int* vectors, int n, int m, - int* csums, int k, int* labels ) -{ - // TODO: consider adding priors (class weights) and sample weights to the clustering algorithm - int iters = 0, max_iters = 100; - int i, j, idx; - cv::AutoBuffer buf(n + k); - double *v_weights = buf, *c_weights = buf + n; - bool modified = true; - RNG* r = data->rng; - - // assign labels randomly - for( i = 0; i < n; i++ ) - { - int sum = 0; - const int* v = vectors + i*m; - labels[i] = i < k ? i : r->uniform(0, k); - - // compute weight of each vector - for( j = 0; j < m; j++ ) - sum += v[j]; - v_weights[i] = sum ? 1./sum : 0.; - } - - for( i = 0; i < n; i++ ) - { - int i1 = (*r)(n); - int i2 = (*r)(n); - CV_SWAP( labels[i1], labels[i2], j ); - } - - for( iters = 0; iters <= max_iters; iters++ ) - { - // calculate csums - for( i = 0; i < k; i++ ) - { - for( j = 0; j < m; j++ ) - csums[i*m + j] = 0; - } - - for( i = 0; i < n; i++ ) - { - const int* v = vectors + i*m; - int* s = csums + labels[i]*m; - for( j = 0; j < m; j++ ) - s[j] += v[j]; - } - - // exit the loop here, when we have up-to-date csums - if( iters == max_iters || !modified ) - break; - - modified = false; - - // calculate weight of each cluster - for( i = 0; i < k; i++ ) - { - const int* s = csums + i*m; - int sum = 0; - for( j = 0; j < m; j++ ) - sum += s[j]; - c_weights[i] = sum ? 1./sum : 0; - } - - // now for each vector determine the closest cluster - for( i = 0; i < n; i++ ) - { - const int* v = vectors + i*m; - double alpha = v_weights[i]; - double min_dist2 = DBL_MAX; - int min_idx = -1; - - for( idx = 0; idx < k; idx++ ) - { - const int* s = csums + idx*m; - double dist2 = 0., beta = c_weights[idx]; - for( j = 0; j < m; j++ ) - { - double t = v[j]*alpha - s[j]*beta; - dist2 += t*t; - } - if( min_dist2 > dist2 ) - { - min_dist2 = dist2; - min_idx = idx; - } - } - - if( min_idx != labels[i] ) - modified = true; - labels[i] = min_idx; - } - } -} - - -CvDTreeSplit* CvDTree::find_split_cat_class( CvDTreeNode* node, int vi, float init_quality, - CvDTreeSplit* _split, uchar* _ext_buf ) -{ - int ci = data->get_var_type(vi); - int n = node->sample_count; - int m = data->get_num_classes(); - int _mi = data->cat_count->data.i[ci], mi = _mi; - - int base_size = m*(3 + mi)*sizeof(int) + (mi+1)*sizeof(double); - if( m > 2 && mi > data->params.max_categories ) - base_size += (m*std::min(data->params.max_categories, n) + mi)*sizeof(int); - else - base_size += mi*sizeof(int*); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + 2*n*sizeof(int)); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - - int* lc = (int*)base_buf; - int* rc = lc + m; - int* _cjk = rc + m*2, *cjk = _cjk; - double* c_weights = (double*)alignPtr(cjk + m*mi, sizeof(double)); - - int* labels_buf = (int*)ext_buf; - const int* labels = data->get_cat_var_data(node, vi, labels_buf); - int* responses_buf = labels_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - - int* cluster_labels = 0; - int** int_ptr = 0; - int i, j, k, idx; - double L = 0, R = 0; - double best_val = init_quality; - int prevcode = 0, best_subset = -1, subset_i, subset_n, subtract = 0; - const double* priors = data->priors_mult->data.db; - - // init array of counters: - // c_{jk} - number of samples that have vi-th input variable = j and response = k. - for( j = -1; j < mi; j++ ) - for( k = 0; k < m; k++ ) - cjk[j*m + k] = 0; - - for( i = 0; i < n; i++ ) - { - j = ( labels[i] == 65535 && data->is_buf_16u) ? -1 : labels[i]; - k = responses[i]; - cjk[j*m + k]++; - } - - if( m > 2 ) - { - if( mi > data->params.max_categories ) - { - mi = MIN(data->params.max_categories, n); - cjk = (int*)(c_weights + _mi); - cluster_labels = cjk + m*mi; - cluster_categories( _cjk, _mi, m, cjk, mi, cluster_labels ); - } - subset_i = 1; - subset_n = 1 << mi; - } - else - { - assert( m == 2 ); - int_ptr = (int**)(c_weights + _mi); - for( j = 0; j < mi; j++ ) - int_ptr[j] = cjk + j*2 + 1; - std::sort(int_ptr, int_ptr + mi, LessThanPtr()); - subset_i = 0; - subset_n = mi; - } - - for( k = 0; k < m; k++ ) - { - int sum = 0; - for( j = 0; j < mi; j++ ) - sum += cjk[j*m + k]; - rc[k] = sum; - lc[k] = 0; - } - - for( j = 0; j < mi; j++ ) - { - double sum = 0; - for( k = 0; k < m; k++ ) - sum += cjk[j*m + k]*priors[k]; - c_weights[j] = sum; - R += c_weights[j]; - } - - for( ; subset_i < subset_n; subset_i++ ) - { - double weight; - int* crow; - double lsum2 = 0, rsum2 = 0; - - if( m == 2 ) - idx = (int)(int_ptr[subset_i] - cjk)/2; - else - { - int graycode = (subset_i>>1)^subset_i; - int diff = graycode ^ prevcode; - - // determine index of the changed bit. - Cv32suf u; - idx = diff >= (1 << 16) ? 16 : 0; - u.f = (float)(((diff >> 16) | diff) & 65535); - idx += (u.i >> 23) - 127; - subtract = graycode < prevcode; - prevcode = graycode; - } - - crow = cjk + idx*m; - weight = c_weights[idx]; - if( weight < FLT_EPSILON ) - continue; - - if( !subtract ) - { - for( k = 0; k < m; k++ ) - { - int t = crow[k]; - int lval = lc[k] + t; - int rval = rc[k] - t; - double p = priors[k], p2 = p*p; - lsum2 += p2*lval*lval; - rsum2 += p2*rval*rval; - lc[k] = lval; rc[k] = rval; - } - L += weight; - R -= weight; - } - else - { - for( k = 0; k < m; k++ ) - { - int t = crow[k]; - int lval = lc[k] - t; - int rval = rc[k] + t; - double p = priors[k], p2 = p*p; - lsum2 += p2*lval*lval; - rsum2 += p2*rval*rval; - lc[k] = lval; rc[k] = rval; - } - L -= weight; - R += weight; - } - - if( L > FLT_EPSILON && R > FLT_EPSILON ) - { - double val = (lsum2*R + rsum2*L)/((double)L*R); - if( best_val < val ) - { - best_val = val; - best_subset = subset_i; - } - } - } - - CvDTreeSplit* split = 0; - if( best_subset >= 0 ) - { - split = _split ? _split : data->new_split_cat( 0, -1.0f ); - split->var_idx = vi; - split->quality = (float)best_val; - memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); - if( m == 2 ) - { - for( i = 0; i <= best_subset; i++ ) - { - idx = (int)(int_ptr[i] - cjk) >> 1; - split->subset[idx >> 5] |= 1 << (idx & 31); - } + if( _isClassifier ) + split = findSplitCatClass(vi, _sidx, 0, subset); + else + split = findSplitCatReg(vi, _sidx, 0, subset); } else { - for( i = 0; i < _mi; i++ ) - { - idx = cluster_labels ? cluster_labels[i] : i; - if( best_subset & (1 << idx) ) - split->subset[i >> 5] |= 1 << (i & 31); - } - } - } - return split; -} - - -CvDTreeSplit* CvDTree::find_split_ord_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - int n = node->sample_count; - int n1 = node->get_num_valid(vi); - - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate(2*n*(sizeof(int) + sizeof(float))); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - float* values_buf = (float*)ext_buf; - int* sorted_indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = sorted_indices_buf + n; - const float* values = 0; - const int* sorted_indices = 0; - data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); - float* responses_buf = (float*)(sample_indices_buf + n); - const float* responses = data->get_ord_responses( node, responses_buf, sample_indices_buf ); - - int i, best_i = -1; - double best_val = init_quality, lsum = 0, rsum = node->value*n; - int L = 0, R = n1; - - // compensate for missing values - for( i = n1; i < n; i++ ) - rsum -= responses[sorted_indices[i]]; - - // find the optimal split - for( i = 0; i < n1 - 1; i++ ) - { - float t = responses[sorted_indices[i]]; - L++; R--; - lsum += t; - rsum -= t; - - if( values[i] + epsilon < values[i+1] ) - { - double val = (lsum*lsum*R + rsum*rsum*L)/((double)L*R); - if( best_val < val ) - { - best_val = val; - best_i = i; - } - } - } - - CvDTreeSplit* split = 0; - if( best_i >= 0 ) - { - split = _split ? _split : data->new_split_ord( 0, 0.0f, 0, 0, 0.0f ); - split->var_idx = vi; - split->ord.c = (values[best_i] + values[best_i+1])*0.5f; - split->ord.split_point = best_i; - split->inversed = 0; - split->quality = (float)best_val; - } - return split; -} - -CvDTreeSplit* CvDTree::find_split_cat_reg( CvDTreeNode* node, int vi, float init_quality, CvDTreeSplit* _split, uchar* _ext_buf ) -{ - int ci = data->get_var_type(vi); - int n = node->sample_count; - int mi = data->cat_count->data.i[ci]; - - int base_size = (mi+2)*sizeof(double) + (mi+1)*(sizeof(int) + sizeof(double*)); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + n*(2*sizeof(int) + sizeof(float))); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - int* labels_buf = (int*)ext_buf; - const int* labels = data->get_cat_var_data(node, vi, labels_buf); - float* responses_buf = (float*)(labels_buf + n); - int* sample_indices_buf = (int*)(responses_buf + n); - const float* responses = data->get_ord_responses(node, responses_buf, sample_indices_buf); - - double* sum = (double*)cv::alignPtr(base_buf,sizeof(double)) + 1; - int* counts = (int*)(sum + mi) + 1; - double** sum_ptr = (double**)(counts + mi); - int i, L = 0, R = 0; - double best_val = init_quality, lsum = 0, rsum = 0; - int best_subset = -1, subset_i; - - for( i = -1; i < mi; i++ ) - sum[i] = counts[i] = 0; - - // calculate sum response and weight of each category of the input var - for( i = 0; i < n; i++ ) - { - int idx = ( (labels[i] == 65535) && data->is_buf_16u ) ? -1 : labels[i]; - double s = sum[idx] + responses[i]; - int nc = counts[idx] + 1; - sum[idx] = s; - counts[idx] = nc; - } - - // calculate average response in each category - for( i = 0; i < mi; i++ ) - { - R += counts[i]; - rsum += sum[i]; - sum[i] /= MAX(counts[i],1); - sum_ptr[i] = sum + i; - } - - std::sort(sum_ptr, sum_ptr + mi, LessThanPtr()); - - // revert back to unnormalized sums - // (there should be a very little loss of accuracy) - for( i = 0; i < mi; i++ ) - sum[i] *= counts[i]; - - for( subset_i = 0; subset_i < mi-1; subset_i++ ) - { - int idx = (int)(sum_ptr[subset_i] - sum); - int ni = counts[idx]; - - if( ni ) - { - double s = sum[idx]; - lsum += s; L += ni; - rsum -= s; R -= ni; - - if( L && R ) - { - double val = (lsum*lsum*R + rsum*rsum*L)/((double)L*R); - if( best_val < val ) - { - best_val = val; - best_subset = subset_i; - } - } - } - } - - CvDTreeSplit* split = 0; - if( best_subset >= 0 ) - { - split = _split ? _split : data->new_split_cat( 0, -1.0f); - split->var_idx = vi; - split->quality = (float)best_val; - memset( split->subset, 0, (data->max_c_count + 31)/32 * sizeof(int)); - for( i = 0; i <= best_subset; i++ ) - { - int idx = (int)(sum_ptr[i] - sum); - split->subset[idx >> 5] |= 1 << (idx & 31); - } - } - return split; -} - -CvDTreeSplit* CvDTree::find_surrogate_split_ord( CvDTreeNode* node, int vi, uchar* _ext_buf ) -{ - const float epsilon = FLT_EPSILON*2; - const char* dir = (char*)data->direction->data.ptr; - int n = node->sample_count, n1 = node->get_num_valid(vi); - cv::AutoBuffer inn_buf; - if( !_ext_buf ) - inn_buf.allocate( n*(sizeof(int)*(data->have_priors ? 3 : 2) + sizeof(float)) ); - uchar* ext_buf = _ext_buf ? _ext_buf : (uchar*)inn_buf; - float* values_buf = (float*)ext_buf; - int* sorted_indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = sorted_indices_buf + n; - const float* values = 0; - const int* sorted_indices = 0; - data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); - // LL - number of samples that both the primary and the surrogate splits send to the left - // LR - ... primary split sends to the left and the surrogate split sends to the right - // RL - ... primary split sends to the right and the surrogate split sends to the left - // RR - ... both send to the right - int i, best_i = -1, best_inversed = 0; - double best_val; - - if( !data->have_priors ) - { - int LL = 0, RL = 0, LR, RR; - int worst_val = cvFloor(node->maxlr), _best_val = worst_val; - int sum = 0, sum_abs = 0; - - for( i = 0; i < n1; i++ ) - { - int d = dir[sorted_indices[i]]; - sum += d; sum_abs += d & 1; - } - - // sum_abs = R + L; sum = R - L - RR = (sum_abs + sum) >> 1; - LR = (sum_abs - sum) >> 1; - - // initially all the samples are sent to the right by the surrogate split, - // LR of them are sent to the left by primary split, and RR - to the right. - // now iteratively compute LL, LR, RL and RR for every possible surrogate split value. - for( i = 0; i < n1 - 1; i++ ) - { - int d = dir[sorted_indices[i]]; - - if( d < 0 ) - { - LL++; LR--; - if( LL + RR > _best_val && values[i] + epsilon < values[i+1] ) - { - best_val = LL + RR; - best_i = i; best_inversed = 0; - } - } - else if( d > 0 ) - { - RL++; RR--; - if( RL + LR > _best_val && values[i] + epsilon < values[i+1] ) - { - best_val = RL + LR; - best_i = i; best_inversed = 1; - } - } - } - best_val = _best_val; - } - else - { - double LL = 0, RL = 0, LR, RR; - double worst_val = node->maxlr; - double sum = 0, sum_abs = 0; - const double* priors = data->priors_mult->data.db; - int* responses_buf = sample_indices_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - best_val = worst_val; - - for( i = 0; i < n1; i++ ) - { - int idx = sorted_indices[i]; - double w = priors[responses[idx]]; - int d = dir[idx]; - sum += d*w; sum_abs += (d & 1)*w; - } - - // sum_abs = R + L; sum = R - L - RR = (sum_abs + sum)*0.5; - LR = (sum_abs - sum)*0.5; - - // initially all the samples are sent to the right by the surrogate split, - // LR of them are sent to the left by primary split, and RR - to the right. - // now iteratively compute LL, LR, RL and RR for every possible surrogate split value. - for( i = 0; i < n1 - 1; i++ ) - { - int idx = sorted_indices[i]; - double w = priors[responses[idx]]; - int d = dir[idx]; - - if( d < 0 ) - { - LL += w; LR -= w; - if( LL + RR > best_val && values[i] + epsilon < values[i+1] ) - { - best_val = LL + RR; - best_i = i; best_inversed = 0; - } - } - else if( d > 0 ) - { - RL += w; RR -= w; - if( RL + LR > best_val && values[i] + epsilon < values[i+1] ) - { - best_val = RL + LR; - best_i = i; best_inversed = 1; - } - } - } - } - return best_i >= 0 && best_val > node->maxlr ? data->new_split_ord( vi, - (values[best_i] + values[best_i+1])*0.5f, best_i, best_inversed, (float)best_val ) : 0; -} - - -CvDTreeSplit* CvDTree::find_surrogate_split_cat( CvDTreeNode* node, int vi, uchar* _ext_buf ) -{ - const char* dir = (char*)data->direction->data.ptr; - int n = node->sample_count; - int i, mi = data->cat_count->data.i[data->get_var_type(vi)], l_win = 0; - - int base_size = (2*(mi+1)+1)*sizeof(double) + (!data->have_priors ? 2*(mi+1)*sizeof(int) : 0); - cv::AutoBuffer inn_buf(base_size); - if( !_ext_buf ) - inn_buf.allocate(base_size + n*(sizeof(int) + (data->have_priors ? sizeof(int) : 0))); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = _ext_buf ? _ext_buf : base_buf + base_size; - - int* labels_buf = (int*)ext_buf; - const int* labels = data->get_cat_var_data(node, vi, labels_buf); - // LL - number of samples that both the primary and the surrogate splits send to the left - // LR - ... primary split sends to the left and the surrogate split sends to the right - // RL - ... primary split sends to the right and the surrogate split sends to the left - // RR - ... both send to the right - CvDTreeSplit* split = data->new_split_cat( vi, 0 ); - double best_val = 0; - double* lc = (double*)cv::alignPtr(base_buf,sizeof(double)) + 1; - double* rc = lc + mi + 1; - - for( i = -1; i < mi; i++ ) - lc[i] = rc[i] = 0; - - // for each category calculate the weight of samples - // sent to the left (lc) and to the right (rc) by the primary split - if( !data->have_priors ) - { - int* _lc = (int*)rc + 1; - int* _rc = _lc + mi + 1; - - for( i = -1; i < mi; i++ ) - _lc[i] = _rc[i] = 0; - - for( i = 0; i < n; i++ ) - { - int idx = ( (labels[i] == 65535) && (data->is_buf_16u) ) ? -1 : labels[i]; - int d = dir[i]; - int sum = _lc[idx] + d; - int sum_abs = _rc[idx] + (d & 1); - _lc[idx] = sum; _rc[idx] = sum_abs; - } - - for( i = 0; i < mi; i++ ) - { - int sum = _lc[i]; - int sum_abs = _rc[i]; - lc[i] = (sum_abs - sum) >> 1; - rc[i] = (sum_abs + sum) >> 1; - } - } - else - { - const double* priors = data->priors_mult->data.db; - int* responses_buf = labels_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - - for( i = 0; i < n; i++ ) - { - int idx = ( (labels[i] == 65535) && (data->is_buf_16u) ) ? -1 : labels[i]; - double w = priors[responses[i]]; - int d = dir[i]; - double sum = lc[idx] + d*w; - double sum_abs = rc[idx] + (d & 1)*w; - lc[idx] = sum; rc[idx] = sum_abs; + if( _isClassifier ) + split = findSplitOrdClass(vi, _sidx, 0); + else + split = findSplitOrdReg(vi, _sidx, 0); } - - for( i = 0; i < mi; i++ ) + if( split.quality > best_split.quality ) { - double sum = lc[i]; - double sum_abs = rc[i]; - lc[i] = (sum_abs - sum) * 0.5; - rc[i] = (sum_abs + sum) * 0.5; + best_split = split; + std::swap(subset, best_subset); } } - // 2. now form the split. - // in each category send all the samples to the same direction as majority - for( i = 0; i < mi; i++ ) + if( best_split.quality > 0 ) { - double lval = lc[i], rval = rc[i]; - if( lval > rval ) - { - split->subset[i >> 5] |= 1 << (i & 31); - best_val += lval; - l_win++; - } - else - best_val += rval; + int best_vi = best_split.varIdx; + CV_Assert( compVarIdx[best_split.varIdx] >= 0 && best_vi >= 0 ); + int i, prevsz = (int)w->wsubsets.size(), ssize = getSubsetSize(best_vi); + w->wsubsets.resize(prevsz + ssize); + for( i = 0; i < ssize; i++ ) + w->wsubsets[prevsz + i] = best_subset[i]; + best_split.subsetOfs = prevsz; + w->wsplits.push_back(best_split); + splitidx = (int)(w->wsplits.size()-1); } - split->quality = (float)best_val; - if( split->quality <= node->maxlr || l_win == 0 || l_win == mi ) - cvSetRemoveByPtr( data->split_heap, split ), split = 0; - - return split; + return splitidx; } - -void CvDTree::calc_node_value( CvDTreeNode* node ) +void DTreesImpl::calcValue( int nidx, const vector& _sidx ) { - int i, j, k, n = node->sample_count, cv_n = data->params.cv_folds; - int m = data->get_num_classes(); + WNode* node = &w->wnodes[nidx]; + int i, j, k, n = (int)_sidx.size(), cv_n = params.CVFolds; + int m = (int)classLabels.size(); - int base_size = data->is_classifier ? m*cv_n*sizeof(int) : 2*cv_n*sizeof(double)+cv_n*sizeof(int); - int ext_size = n*(sizeof(int) + (data->is_classifier ? sizeof(int) : sizeof(int)+sizeof(float))); - cv::AutoBuffer inn_buf(base_size + ext_size); - uchar* base_buf = (uchar*)inn_buf; - uchar* ext_buf = base_buf + base_size; + cv::AutoBuffer buf(std::max(m, 3)*(cv_n+1)); - int* cv_labels_buf = (int*)ext_buf; - const int* cv_labels = data->get_cv_labels(node, cv_labels_buf); + if( cv_n > 0 ) + { + size_t sz = w->cv_Tn.size(); + w->cv_Tn.resize(sz + cv_n); + w->cv_node_risk.resize(sz + cv_n); + w->cv_node_error.resize(sz + cv_n); + } - if( data->is_classifier ) + if( _isClassifier ) { // in case of classification tree: // * node value is the label of the class that has the largest weight in the node. @@ -2775,13 +517,11 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) // misclassified samples with cv_labels(*)==j. // compute the number of instances of each class - int* cls_count = data->counts->data.i; - int* responses_buf = cv_labels_buf + n; - const int* responses = data->get_class_labels(node, responses_buf); - int* cv_cls_count = (int*)base_buf; + double* cls_count = buf; + double* cv_cls_count = cls_count + m; + double max_val = -1, total_weight = 0; int max_k = -1; - double* priors = data->priors_mult->data.db; for( k = 0; k < m; k++ ) cls_count[k] = 0; @@ -2789,7 +529,10 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) if( cv_n == 0 ) { for( i = 0; i < n; i++ ) - cls_count[responses[i]]++; + { + int si = _sidx[i]; + cls_count[w->cat_responses[si]] += w->sample_weights[si]; + } } else { @@ -2799,8 +542,9 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) for( i = 0; i < n; i++ ) { - j = cv_labels[i]; k = responses[i]; - cv_cls_count[j*m + k]++; + int si = _sidx[i]; + j = w->cv_labels[si]; k = w->cat_responses[si]; + cv_cls_count[j*m + k] += w->sample_weights[si]; } for( j = 0; j < cv_n; j++ ) @@ -2808,24 +552,9 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) cls_count[k] += cv_cls_count[j*m + k]; } - if( data->have_priors && node->parent == 0 ) - { - // compute priors_mult from priors, take the sample ratio into account. - double sum = 0; - for( k = 0; k < m; k++ ) - { - int n_k = cls_count[k]; - priors[k] = data->priors->data.db[k]*(n_k ? 1./n_k : 0.); - sum += priors[k]; - } - sum = 1./sum; - for( k = 0; k < m; k++ ) - priors[k] *= sum; - } - for( k = 0; k < m; k++ ) { - double val = cls_count[k]*priors[k]; + double val = cls_count[k]; total_weight += val; if( max_val < val ) { @@ -2835,8 +564,7 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) } node->class_idx = max_k; - node->value = data->cat_map->data.i[ - data->cat_ofs->data.i[data->cat_var_count] + max_k]; + node->value = classLabels[max_k]; node->node_risk = total_weight - max_val; for( j = 0; j < cv_n; j++ ) @@ -2846,9 +574,8 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) for( k = 0; k < m; k++ ) { - double w = priors[k]; - double val_k = cv_cls_count[j*m + k]*w; - double val = cls_count[k]*w - val_k; + double val_k = cv_cls_count[j*m + k]; + double val = cls_count[k] - val_k; sum_k += val_k; sum += val; if( max_val < val ) @@ -2859,9 +586,9 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) } } - node->cv_Tn[j] = INT_MAX; - node->cv_node_risk[j] = sum - max_val; - node->cv_node_error[j] = sum_k - max_val_k; + w->cv_Tn[nidx*cv_n + j] = INT_MAX; + w->cv_node_risk[nidx*cv_n + j] = sum - max_val; + w->cv_node_error[nidx*cv_n + j] = sum_k - max_val_k; } } else @@ -2878,28 +605,24 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) // where node_value_j is the node value calculated // as described in the previous bullet, and summation is done // over the samples with cv_labels(*)==j. - - double sum = 0, sum2 = 0; - float* values_buf = (float*)(cv_labels_buf + n); - int* sample_indices_buf = (int*)(values_buf + n); - const float* values = data->get_ord_responses(node, values_buf, sample_indices_buf); - double *cv_sum = 0, *cv_sum2 = 0; - int* cv_count = 0; + double sum = 0, sum2 = 0, sumw = 0; if( cv_n == 0 ) { for( i = 0; i < n; i++ ) { - double t = values[i]; - sum += t; - sum2 += t*t; + int si = _sidx[i]; + double wval = w->sample_weights[si]; + double t = w->ord_responses[si]; + sum += t*wval; + sum2 += t*t*wval; + sumw += wval; } } else { - cv_sum = (double*)base_buf; - cv_sum2 = cv_sum + cv_n; - cv_count = (int*)(cv_sum2 + cv_n); + double *cv_sum = buf, *cv_sum2 = cv_sum + cv_n; + double* cv_count = (double*)(cv_sum2 + cv_n); for( j = 0; j < cv_n; j++ ) { @@ -2909,537 +632,642 @@ void CvDTree::calc_node_value( CvDTreeNode* node ) for( i = 0; i < n; i++ ) { - j = cv_labels[i]; - double t = values[i]; - double s = cv_sum[j] + t; - double s2 = cv_sum2[j] + t*t; - int nc = cv_count[j] + 1; - cv_sum[j] = s; - cv_sum2[j] = s2; - cv_count[j] = nc; + int si = _sidx[i]; + j = w->cv_labels[si]; + double wval = w->sample_weights[si]; + double t = w->ord_responses[si]; + cv_sum[j] += t*wval; + cv_sum2[j] += t*t*wval; + cv_count[j] += wval; } for( j = 0; j < cv_n; j++ ) { sum += cv_sum[j]; sum2 += cv_sum2[j]; + sumw += cv_count[j]; + } + + for( j = 0; j < cv_n; j++ ) + { + double s = sum - cv_sum[j], si = sum - s; + double s2 = sum2 - cv_sum2[j], s2i = sum2 - s2; + double c = cv_count[j], ci = sumw - c; + double r = si/std::max(ci, DBL_EPSILON); + w->cv_node_risk[nidx*cv_n + j] = s2i - r*r*ci; + w->cv_node_error[nidx*cv_n + j] = s2 - 2*r*s + c*r*r; + w->cv_Tn[nidx*cv_n + j] = INT_MAX; } } - node->node_risk = sum2 - (sum/n)*sum; - node->value = sum/n; + node->node_risk = sum2 - (sum/sumw)*sum; + node->value = sum/sumw; + } +} + +DTreesImpl::WSplit DTreesImpl::findSplitOrdClass( int vi, const vector& _sidx, double initQuality ) +{ + const double epsilon = FLT_EPSILON*2; + int n = (int)_sidx.size(); + int m = (int)classLabels.size(); - for( j = 0; j < cv_n; j++ ) + cv::AutoBuffer buf(n*(sizeof(float) + sizeof(int)) + m*2*sizeof(double)); + const int* sidx = &_sidx[0]; + const int* responses = &w->cat_responses[0]; + const double* weights = &w->sample_weights[0]; + double* lcw = (double*)(uchar*)buf; + double* rcw = lcw + m; + float* values = (float*)(rcw + m); + int* sorted_idx = (int*)(values + n); + int i, best_i = -1; + double best_val = initQuality; + + for( i = 0; i < m; i++ ) + lcw[i] = rcw[i] = 0.; + + w->data->getValues( vi, _sidx, values ); + + for( i = 0; i < n; i++ ) + { + sorted_idx[i] = i; + int si = sidx[i]; + rcw[responses[si]] += weights[si]; + } + + std::sort(sorted_idx, sorted_idx + n, cmp_lt_idx(values)); + + double L = 0, R = 0, lsum2 = 0, rsum2 = 0; + for( i = 0; i < m; i++ ) + { + double wval = rcw[i]; + R += wval; + rsum2 += wval*wval; + } + + for( i = 0; i < n - 1; i++ ) + { + int curr = sorted_idx[i]; + int next = sorted_idx[i+1]; + int si = sidx[curr]; + double wval = weights[si], w2 = wval*wval; + L += wval; R -= wval; + int idx = responses[si]; + double lv = lcw[idx], rv = rcw[idx]; + lsum2 += 2*lv*wval + w2; + rsum2 -= 2*rv*wval - w2; + lcw[idx] = lv + wval; rcw[idx] = rv - wval; + + if( values[curr] + epsilon < values[next] ) { - double s = cv_sum[j], si = sum - s; - double s2 = cv_sum2[j], s2i = sum2 - s2; - int c = cv_count[j], ci = n - c; - double r = si/MAX(ci,1); - node->cv_node_risk[j] = s2i - r*r*ci; - node->cv_node_error[j] = s2 - 2*r*s + c*r*r; - node->cv_Tn[j] = INT_MAX; + double val = (lsum2*R + rsum2*L)/(L*R); + if( best_val < val ) + { + best_val = val; + best_i = i; + } } } -} + WSplit split; + if( best_i >= 0 ) + { + split.varIdx = vi; + split.c = (values[sorted_idx[best_i]] + values[sorted_idx[best_i+1]])*0.5f; + split.inversed = false; + split.quality = (float)best_val; + } + return split; +} -void CvDTree::complete_node_dir( CvDTreeNode* node ) +// simple k-means, slightly modified to take into account the "weight" (L1-norm) of each vector. +void DTreesImpl::clusterCategories( const double* vectors, int n, int m, double* csums, int k, int* labels ) { - int vi, i, n = node->sample_count, nl, nr, d0 = 0, d1 = -1; - int nz = n - node->get_num_valid(node->split->var_idx); - char* dir = (char*)data->direction->data.ptr; + int iters = 0, max_iters = 100; + int i, j, idx; + cv::AutoBuffer buf(n + k); + double *v_weights = buf, *c_weights = buf + n; + bool modified = true; + RNG r((uint64)-1); - // try to complete direction using surrogate splits - if( nz && data->params.use_surrogates ) + // assign labels randomly + for( i = 0; i < n; i++ ) + { + double sum = 0; + const double* v = vectors + i*m; + labels[i] = i < k ? i : r.uniform(0, k); + + // compute weight of each vector + for( j = 0; j < m; j++ ) + sum += v[j]; + v_weights[i] = sum ? 1./sum : 0.; + } + + for( i = 0; i < n; i++ ) + { + int i1 = r.uniform(0, n); + int i2 = r.uniform(0, n); + std::swap( labels[i1], labels[i2] ); + } + + for( iters = 0; iters <= max_iters; iters++ ) { - cv::AutoBuffer inn_buf(n*(2*sizeof(int)+sizeof(float))); - CvDTreeSplit* split = node->split->next; - for( ; split != 0 && nz; split = split->next ) + // calculate csums + for( i = 0; i < k; i++ ) { - int inversed_mask = split->inversed ? -1 : 0; - vi = split->var_idx; + for( j = 0; j < m; j++ ) + csums[i*m + j] = 0; + } - if( data->get_var_type(vi) >= 0 ) // split on categorical var - { - int* labels_buf = (int*)(uchar*)inn_buf; - const int* labels = data->get_cat_var_data(node, vi, labels_buf); - const int* subset = split->subset; + for( i = 0; i < n; i++ ) + { + const double* v = vectors + i*m; + double* s = csums + labels[i]*m; + for( j = 0; j < m; j++ ) + s[j] += v[j]; + } - for( i = 0; i < n; i++ ) - { - int idx = labels[i]; - if( !dir[i] && ( ((idx >= 0)&&(!data->is_buf_16u)) || ((idx != 65535)&&(data->is_buf_16u)) )) + // exit the loop here, when we have up-to-date csums + if( iters == max_iters || !modified ) + break; - { - int d = CV_DTREE_CAT_DIR(idx,subset); - dir[i] = (char)((d ^ inversed_mask) - inversed_mask); - if( --nz ) - break; - } - } - } - else // split on ordered var + modified = false; + + // calculate weight of each cluster + for( i = 0; i < k; i++ ) + { + const double* s = csums + i*m; + double sum = 0; + for( j = 0; j < m; j++ ) + sum += s[j]; + c_weights[i] = sum ? 1./sum : 0; + } + + // now for each vector determine the closest cluster + for( i = 0; i < n; i++ ) + { + const double* v = vectors + i*m; + double alpha = v_weights[i]; + double min_dist2 = DBL_MAX; + int min_idx = -1; + + for( idx = 0; idx < k; idx++ ) { - float* values_buf = (float*)(uchar*)inn_buf; - int* sorted_indices_buf = (int*)(values_buf + n); - int* sample_indices_buf = sorted_indices_buf + n; - const float* values = 0; - const int* sorted_indices = 0; - data->get_ord_var_data( node, vi, values_buf, sorted_indices_buf, &values, &sorted_indices, sample_indices_buf ); - int split_point = split->ord.split_point; - int n1 = node->get_num_valid(vi); - - assert( 0 <= split_point && split_point < n-1 ); - - for( i = 0; i < n1; i++ ) + const double* s = csums + idx*m; + double dist2 = 0., beta = c_weights[idx]; + for( j = 0; j < m; j++ ) { - int idx = sorted_indices[i]; - if( !dir[idx] ) - { - int d = i <= split_point ? -1 : 1; - dir[idx] = (char)((d ^ inversed_mask) - inversed_mask); - if( --nz ) - break; - } + double t = v[j]*alpha - s[j]*beta; + dist2 += t*t; + } + if( min_dist2 > dist2 ) + { + min_dist2 = dist2; + min_idx = idx; } } + + if( min_idx != labels[i] ) + modified = true; + labels[i] = min_idx; } } +} + +DTreesImpl::WSplit DTreesImpl::findSplitCatClass( int vi, const vector& _sidx, + double initQuality, int* subset ) +{ + int _mi = getCatCount(vi), mi = _mi; + int n = (int)_sidx.size(); + int m = (int)classLabels.size(); + + int base_size = m*(3 + mi) + mi + 1; + if( m > 2 && mi > params.maxCategories ) + base_size += m*std::min(params.maxCategories, n) + mi; + else + base_size += mi; + AutoBuffer buf(base_size + n); + + double* lc = (double*)buf; + double* rc = lc + m; + double* _cjk = rc + m*2, *cjk = _cjk; + double* c_weights = cjk + m*mi; + + int* labels = (int*)(buf + base_size); + w->data->getNormCatValues(vi, _sidx, labels); + const int* responses = &w->cat_responses[0]; + const double* weights = &w->sample_weights[0]; + + int* cluster_labels = 0; + double** dbl_ptr = 0; + int i, j, k, si, idx; + double L = 0, R = 0; + double best_val = initQuality; + int prevcode = 0, best_subset = -1, subset_i, subset_n, subtract = 0; - // find the default direction for the rest - if( nz ) + // init array of counters: + // c_{jk} - number of samples that have vi-th input variable = j and response = k. + for( j = -1; j < mi; j++ ) + for( k = 0; k < m; k++ ) + cjk[j*m + k] = 0; + + for( i = 0; i < n; i++ ) { - for( i = nr = 0; i < n; i++ ) - nr += dir[i] > 0; - nl = n - nr - nz; - d0 = nl > nr ? -1 : nr > nl; + si = _sidx[i]; + j = labels[i]; + k = responses[si]; + cjk[j*m + k] += weights[si]; } - // make sure that every sample is directed either to the left or to the right - for( i = 0; i < n; i++ ) + if( m > 2 ) { - int d = dir[i]; - if( !d ) + if( mi > params.maxCategories ) { - d = d0; - if( !d ) - d = d1, d1 = -d1; + mi = std::min(params.maxCategories, n); + cjk = c_weights + _mi; + cluster_labels = (int*)(cjk + m*mi); + clusterCategories( _cjk, _mi, m, cjk, mi, cluster_labels ); } - d = d > 0; - dir[i] = (char)d; // remap (-1,1) to (0,1) + subset_i = 1; + subset_n = 1 << mi; } -} - - -void CvDTree::split_node_data( CvDTreeNode* node ) -{ - int vi, i, n = node->sample_count, nl, nr, scount = data->sample_count; - char* dir = (char*)data->direction->data.ptr; - CvDTreeNode *left = 0, *right = 0; - int* new_idx = data->split_buf->data.i; - int new_buf_idx = data->get_child_buf_idx( node ); - int work_var_count = data->get_work_var_count(); - CvMat* buf = data->buf; - size_t length_buf_row = data->get_length_subbuf(); - cv::AutoBuffer inn_buf(n*(3*sizeof(int) + sizeof(float))); - int* temp_buf = (int*)(uchar*)inn_buf; - - complete_node_dir(node); - - for( i = nl = nr = 0; i < n; i++ ) + else { - int d = dir[i]; - // initialize new indices for splitting ordered variables - new_idx[i] = (nl & (d-1)) | (nr & -d); // d ? ri : li - nr += d; - nl += d^1; + assert( m == 2 ); + dbl_ptr = (double**)(c_weights + _mi); + for( j = 0; j < mi; j++ ) + dbl_ptr[j] = cjk + j*2 + 1; + std::sort(dbl_ptr, dbl_ptr + mi, cmp_lt_ptr()); + subset_i = 0; + subset_n = mi; } - bool split_input_data; - node->left = left = data->new_node( node, nl, new_buf_idx, node->offset ); - node->right = right = data->new_node( node, nr, new_buf_idx, node->offset + nl ); + for( k = 0; k < m; k++ ) + { + double sum = 0; + for( j = 0; j < mi; j++ ) + sum += cjk[j*m + k]; + CV_Assert(sum > 0); + rc[k] = sum; + lc[k] = 0; + } - split_input_data = node->depth + 1 < data->params.max_depth && - (node->left->sample_count > data->params.min_sample_count || - node->right->sample_count > data->params.min_sample_count); + for( j = 0; j < mi; j++ ) + { + double sum = 0; + for( k = 0; k < m; k++ ) + sum += cjk[j*m + k]; + c_weights[j] = sum; + R += c_weights[j]; + } - // split ordered variables, keep both halves sorted. - for( vi = 0; vi < data->var_count; vi++ ) + for( ; subset_i < subset_n; subset_i++ ) { - int ci = data->get_var_type(vi); + double lsum2 = 0, rsum2 = 0; - if( ci >= 0 || !split_input_data ) - continue; + if( m == 2 ) + idx = (int)(dbl_ptr[subset_i] - cjk)/2; + else + { + int graycode = (subset_i>>1)^subset_i; + int diff = graycode ^ prevcode; - int n1 = node->get_num_valid(vi); - float* src_val_buf = (float*)(uchar*)(temp_buf + n); - int* src_sorted_idx_buf = (int*)(src_val_buf + n); - int* src_sample_idx_buf = src_sorted_idx_buf + n; - const float* src_val = 0; - const int* src_sorted_idx = 0; - data->get_ord_var_data(node, vi, src_val_buf, src_sorted_idx_buf, &src_val, &src_sorted_idx, src_sample_idx_buf); + // determine index of the changed bit. + Cv32suf u; + idx = diff >= (1 << 16) ? 16 : 0; + u.f = (float)(((diff >> 16) | diff) & 65535); + idx += (u.i >> 23) - 127; + subtract = graycode < prevcode; + prevcode = graycode; + } - for(i = 0; i < n; i++) - temp_buf[i] = src_sorted_idx[i]; + double* crow = cjk + idx*m; + double weight = c_weights[idx]; + if( weight < FLT_EPSILON ) + continue; - if (data->is_buf_16u) + if( !subtract ) { - unsigned short *ldst, *rdst, *ldst0, *rdst0; - //unsigned short tl, tr; - ldst0 = ldst = (unsigned short*)(buf->data.s + left->buf_idx*length_buf_row + - vi*scount + left->offset); - rdst0 = rdst = (unsigned short*)(ldst + nl); - - // split sorted - for( i = 0; i < n1; i++ ) + for( k = 0; k < m; k++ ) { - int idx = temp_buf[i]; - int d = dir[idx]; - idx = new_idx[idx]; - if (d) - { - *rdst = (unsigned short)idx; - rdst++; - } - else - { - *ldst = (unsigned short)idx; - ldst++; - } + double t = crow[k]; + double lval = lc[k] + t; + double rval = rc[k] - t; + lsum2 += lval*lval; + rsum2 += rval*rval; + lc[k] = lval; rc[k] = rval; + } + L += weight; + R -= weight; + } + else + { + for( k = 0; k < m; k++ ) + { + double t = crow[k]; + double lval = lc[k] - t; + double rval = rc[k] + t; + lsum2 += lval*lval; + rsum2 += rval*rval; + lc[k] = lval; rc[k] = rval; } + L -= weight; + R += weight; + } - left->set_num_valid(vi, (int)(ldst - ldst0)); - right->set_num_valid(vi, (int)(rdst - rdst0)); + if( L > FLT_EPSILON && R > FLT_EPSILON ) + { + double val = (lsum2*R + rsum2*L)/(L*R); + if( best_val < val ) + { + best_val = val; + best_subset = subset_i; + } + } + } - // split missing - for( ; i < n; i++ ) + WSplit split; + if( best_subset >= 0 ) + { + split.varIdx = vi; + split.quality = (float)best_val; + memset( subset, 0, getSubsetSize(vi) * sizeof(int) ); + if( m == 2 ) + { + for( i = 0; i <= best_subset; i++ ) { - int idx = temp_buf[i]; - int d = dir[idx]; - idx = new_idx[idx]; - if (d) - { - *rdst = (unsigned short)idx; - rdst++; - } - else - { - *ldst = (unsigned short)idx; - ldst++; - } + idx = (int)(dbl_ptr[i] - cjk) >> 1; + subset[idx >> 5] |= 1 << (idx & 31); } } else { - int *ldst0, *ldst, *rdst0, *rdst; - ldst0 = ldst = buf->data.i + left->buf_idx*length_buf_row + - vi*scount + left->offset; - rdst0 = rdst = buf->data.i + right->buf_idx*length_buf_row + - vi*scount + right->offset; - - // split sorted - for( i = 0; i < n1; i++ ) + for( i = 0; i < _mi; i++ ) { - int idx = temp_buf[i]; - int d = dir[idx]; - idx = new_idx[idx]; - if (d) - { - *rdst = idx; - rdst++; - } - else - { - *ldst = idx; - ldst++; - } + idx = cluster_labels ? cluster_labels[i] : i; + if( best_subset & (1 << idx) ) + subset[i >> 5] |= 1 << (i & 31); } + } + } + return split; +} + +DTreesImpl::WSplit DTreesImpl::findSplitOrdReg( int vi, const vector& _sidx, double initQuality ) +{ + const float epsilon = FLT_EPSILON*2; + const double* weights = &w->sample_weights[0]; + int n = (int)_sidx.size(); + + AutoBuffer buf(n*(sizeof(int) + sizeof(float))); + + float* values = (float*)(uchar*)buf; + int* sorted_idx = (int*)(values + n); + w->data->getValues(vi, _sidx, values); + const double* responses = &w->ord_responses[0]; + + int i, si, best_i = -1; + double L = 0, R = 0; + double best_val = initQuality, lsum = 0, rsum = 0; - left->set_num_valid(vi, (int)(ldst - ldst0)); - right->set_num_valid(vi, (int)(rdst - rdst0)); + for( i = 0; i < n; i++ ) + { + sorted_idx[i] = i; + si = _sidx[i]; + R += weights[si]; + rsum += weights[si]*responses[si]; + } + + std::sort(sorted_idx, sorted_idx + n, cmp_lt_idx(values)); - // split missing - for( ; i < n; i++ ) + // find the optimal split + for( i = 0; i < n - 1; i++ ) + { + int curr = sorted_idx[i]; + int next = sorted_idx[i+1]; + si = _sidx[curr]; + double wval = weights[si]; + double t = responses[si]*wval; + L += wval; R -= wval; + lsum += t; rsum -= t; + + if( values[curr] + epsilon < values[next] ) + { + double val = (lsum*lsum*R + rsum*rsum*L)/(L*R); + if( best_val < val ) { - int idx = temp_buf[i]; - int d = dir[idx]; - idx = new_idx[idx]; - if (d) - { - *rdst = idx; - rdst++; - } - else - { - *ldst = idx; - ldst++; - } + best_val = val; + best_i = i; } } } - // split categorical vars, responses and cv_labels using new_idx relocation table - for( vi = 0; vi < work_var_count; vi++ ) + WSplit split; + if( best_i >= 0 ) { - int ci = data->get_var_type(vi); - int n1 = node->get_num_valid(vi), nr1 = 0; + split.varIdx = vi; + split.c = (values[sorted_idx[best_i]] + values[sorted_idx[best_i+1]])*0.5f; + split.inversed = false; + split.quality = (float)best_val; + } + return split; +} - if( ci < 0 || (vi < data->var_count && !split_input_data) ) - continue; +DTreesImpl::WSplit DTreesImpl::findSplitCatReg( int vi, const vector& _sidx, + double initQuality, int* subset ) +{ + const double* weights = &w->sample_weights[0]; + const double* responses = &w->ord_responses[0]; + int n = (int)_sidx.size(); + int mi = getCatCount(vi); - int *src_lbls_buf = temp_buf + n; - const int* src_lbls = data->get_cat_var_data(node, vi, src_lbls_buf); + AutoBuffer buf(3*mi + 3 + n); + double* sum = (double*)buf + 1; + double* counts = sum + mi + 1; + double** sum_ptr = (double**)(counts + mi); + int* cat_labels = (int*)(sum_ptr + mi); - for(i = 0; i < n; i++) - temp_buf[i] = src_lbls[i]; + w->data->getNormCatValues(vi, _sidx, cat_labels); - if (data->is_buf_16u) - { - unsigned short *ldst = (unsigned short *)(buf->data.s + left->buf_idx*length_buf_row + - vi*scount + left->offset); - unsigned short *rdst = (unsigned short *)(buf->data.s + right->buf_idx*length_buf_row + - vi*scount + right->offset); + double L = 0, R = 0, best_val = initQuality, lsum = 0, rsum = 0; + int i, si, best_subset = -1, subset_i; - for( i = 0; i < n; i++ ) - { - int d = dir[i]; - int idx = temp_buf[i]; - if (d) - { - *rdst = (unsigned short)idx; - rdst++; - nr1 += (idx != 65535 )&d; - } - else - { - *ldst = (unsigned short)idx; - ldst++; - } - } + for( i = -1; i < mi; i++ ) + sum[i] = counts[i] = 0; - if( vi < data->var_count ) - { - left->set_num_valid(vi, n1 - nr1); - right->set_num_valid(vi, nr1); - } - } - else + // calculate sum response and weight of each category of the input var + for( i = 0; i < n; i++ ) + { + int idx = cat_labels[i]; + si = _sidx[i]; + double wval = weights[si]; + sum[idx] += responses[si]*wval; + counts[idx] += wval; + } + + // calculate average response in each category + for( i = 0; i < mi; i++ ) + { + R += counts[i]; + rsum += sum[i]; + sum[i] = fabs(counts[i]) > DBL_EPSILON ? sum[i]/counts[i] : 0; + sum_ptr[i] = sum + i; + } + + std::sort(sum_ptr, sum_ptr + mi, cmp_lt_ptr()); + + // revert back to unnormalized sums + // (there should be a very little loss in accuracy) + for( i = 0; i < mi; i++ ) + sum[i] *= counts[i]; + + for( subset_i = 0; subset_i < mi-1; subset_i++ ) + { + int idx = (int)(sum_ptr[subset_i] - sum); + double ni = counts[idx]; + + if( ni > FLT_EPSILON ) { - int *ldst = buf->data.i + left->buf_idx*length_buf_row + - vi*scount + left->offset; - int *rdst = buf->data.i + right->buf_idx*length_buf_row + - vi*scount + right->offset; + double s = sum[idx]; + lsum += s; L += ni; + rsum -= s; R -= ni; - for( i = 0; i < n; i++ ) + if( L > FLT_EPSILON && R > FLT_EPSILON ) { - int d = dir[i]; - int idx = temp_buf[i]; - if (d) - { - *rdst = idx; - rdst++; - nr1 += (idx >= 0)&d; - } - else + double val = (lsum*lsum*R + rsum*rsum*L)/(L*R); + if( best_val < val ) { - *ldst = idx; - ldst++; + best_val = val; + best_subset = subset_i; } - - } - - if( vi < data->var_count ) - { - left->set_num_valid(vi, n1 - nr1); - right->set_num_valid(vi, nr1); } } } + WSplit split; + if( best_subset >= 0 ) + { + split.varIdx = vi; + split.quality = (float)best_val; + memset( subset, 0, getSubsetSize(vi) * sizeof(int)); + for( i = 0; i <= best_subset; i++ ) + { + int idx = (int)(sum_ptr[i] - sum); + subset[idx >> 5] |= 1 << (idx & 31); + } + } + return split; +} - // split sample indices - int *sample_idx_src_buf = temp_buf + n; - const int* sample_idx_src = data->get_sample_indices(node, sample_idx_src_buf); +int DTreesImpl::calcDir( int splitidx, const vector& _sidx, + vector& _sleft, vector& _sright ) +{ + WSplit split = w->wsplits[splitidx]; + int i, si, n = (int)_sidx.size(), vi = split.varIdx; + _sleft.reserve(n); + _sright.reserve(n); + _sleft.clear(); + _sright.clear(); - for(i = 0; i < n; i++) - temp_buf[i] = sample_idx_src[i]; + AutoBuffer buf(n); + int mi = getCatCount(vi); + double wleft = 0, wright = 0; + const double* weights = &w->sample_weights[0]; - int pos = data->get_work_var_count(); - if (data->is_buf_16u) + if( mi <= 0 ) // split on an ordered variable { - unsigned short* ldst = (unsigned short*)(buf->data.s + left->buf_idx*length_buf_row + - pos*scount + left->offset); - unsigned short* rdst = (unsigned short*)(buf->data.s + right->buf_idx*length_buf_row + - pos*scount + right->offset); - for (i = 0; i < n; i++) + float c = split.c; + float* values = buf; + w->data->getValues(vi, _sidx, values); + + for( i = 0; i < n; i++ ) { - int d = dir[i]; - unsigned short idx = (unsigned short)temp_buf[i]; - if (d) + si = _sidx[i]; + if( values[i] <= c ) { - *rdst = idx; - rdst++; + _sleft.push_back(si); + wleft += weights[si]; } else { - *ldst = idx; - ldst++; + _sright.push_back(si); + wright += weights[si]; } } } else { - int* ldst = buf->data.i + left->buf_idx*length_buf_row + - pos*scount + left->offset; - int* rdst = buf->data.i + right->buf_idx*length_buf_row + - pos*scount + right->offset; - for (i = 0; i < n; i++) + const int* subset = &w->wsubsets[split.subsetOfs]; + int* cat_labels = (int*)(float*)buf; + w->data->getNormCatValues(vi, _sidx, cat_labels); + + for( i = 0; i < n; i++ ) { - int d = dir[i]; - int idx = temp_buf[i]; - if (d) + si = _sidx[i]; + unsigned u = cat_labels[i]; + if( CV_DTREE_CAT_DIR(u, subset) < 0 ) { - *rdst = idx; - rdst++; + _sleft.push_back(si); + wleft += weights[si]; } else { - *ldst = idx; - ldst++; + _sright.push_back(si); + wright += weights[si]; } } } - - // deallocate the parent node data that is not needed anymore - data->free_node_data(node); -} - -float CvDTree::calc_error( CvMLData* _data, int type, std::vector *resp ) -{ - float err = 0; - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* missing = _data->get_missing(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - const CvMat* var_types = _data->get_var_types(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(response->type) ? - 1 : response->step / CV_ELEM_SIZE(response->type); - bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; - float* pred_resp = 0; - if( resp && (sample_count > 0) ) - { - resp->resize( sample_count ); - pred_resp = &((*resp)[0]); - } - - if ( is_classifier ) - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample, miss; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - if( missing ) - cvGetRow( missing, &miss, si ); - float r = (float)predict( &sample, missing ? &miss : 0 )->value; - if( pred_resp ) - pred_resp[i] = r; - int d = fabs((double)r - response->data.fl[(size_t)si*r_step]) <= FLT_EPSILON ? 0 : 1; - err += d; - } - err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; - } - else - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample, miss; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - if( missing ) - cvGetRow( missing, &miss, si ); - float r = (float)predict( &sample, missing ? &miss : 0 )->value; - if( pred_resp ) - pred_resp[i] = r; - float d = r - response->data.fl[(size_t)si*r_step]; - err += d*d; - } - err = sample_count ? err / (float)sample_count : -FLT_MAX; - } - return err; + CV_Assert( (int)_sleft.size() < n && (int)_sright.size() < n ); + return wleft > wright ? -1 : 1; } -void CvDTree::prune_cv() +int DTreesImpl::pruneCV( int root ) { - CvMat* ab = 0; - CvMat* temp = 0; - CvMat* err_jk = 0; + vector ab; // 1. build tree sequence for each cv fold, calculate error_{Tj,beta_k}. // 2. choose the best tree index (if need, apply 1SE rule). // 3. store the best index and cut the branches. - CV_FUNCNAME( "CvDTree::prune_cv" ); - - __BEGIN__; - - int ti, j, tree_count = 0, cv_n = data->params.cv_folds, n = root->sample_count; + int ti, tree_count = 0, j, cv_n = params.CVFolds, n = w->wnodes[root].sample_count; // currently, 1SE for regression is not implemented - bool use_1se = data->params.use_1se_rule != 0 && data->is_classifier; - double* err; + bool use_1se = params.use1SERule != 0 && _isClassifier; double min_err = 0, min_err_se = 0; int min_idx = -1; - CV_CALL( ab = cvCreateMat( 1, 256, CV_64F )); - // build the main tree sequence, calculate alpha's for(;;tree_count++) { - double min_alpha = update_tree_rnc(tree_count, -1); - if( cut_tree(tree_count, -1, min_alpha) ) + double min_alpha = updateTreeRNC(root, tree_count, -1); + if( cutTree(root, tree_count, -1, min_alpha) ) break; - if( ab->cols <= tree_count ) - { - CV_CALL( temp = cvCreateMat( 1, ab->cols*3/2, CV_64F )); - for( ti = 0; ti < ab->cols; ti++ ) - temp->data.db[ti] = ab->data.db[ti]; - cvReleaseMat( &ab ); - ab = temp; - temp = 0; - } - - ab->data.db[tree_count] = min_alpha; + ab.push_back(min_alpha); } - ab->data.db[0] = 0.; - if( tree_count > 0 ) { + ab[0] = 0.; + for( ti = 1; ti < tree_count-1; ti++ ) - ab->data.db[ti] = sqrt(ab->data.db[ti]*ab->data.db[ti+1]); - ab->data.db[tree_count-1] = DBL_MAX*0.5; + ab[ti] = std::sqrt(ab[ti]*ab[ti+1]); + ab[tree_count-1] = DBL_MAX*0.5; - CV_CALL( err_jk = cvCreateMat( cv_n, tree_count, CV_64F )); - err = err_jk->data.db; + Mat err_jk(cv_n, tree_count, CV_64F); for( j = 0; j < cv_n; j++ ) { int tj = 0, tk = 0; - for( ; tk < tree_count; tj++ ) + for( ; tj < tree_count; tj++ ) { - double min_alpha = update_tree_rnc(tj, j); - if( cut_tree(tj, j, min_alpha) ) + double min_alpha = updateTreeRNC(root, tj, j); + if( cutTree(root, tj, j, min_alpha) ) min_alpha = DBL_MAX; for( ; tk < tree_count; tk++ ) { - if( ab->data.db[tk] > min_alpha ) + if( ab[tk] > min_alpha ) break; - err[j*tree_count + tk] = root->tree_error; + err_jk.at(j, tk) = w->wnodes[root].tree_error; } } } @@ -3448,7 +1276,7 @@ void CvDTree::prune_cv() { double sum_err = 0; for( j = 0; j < cv_n; j++ ) - sum_err += err[j*tree_count + ti]; + sum_err += err_jk.at(j, ti); if( ti == 0 || sum_err < min_err ) { min_err = sum_err; @@ -3461,242 +1289,190 @@ void CvDTree::prune_cv() } } - pruned_tree_idx = min_idx; - free_prune_data(data->params.truncate_pruned_tree != 0); - - __END__; - - cvReleaseMat( &err_jk ); - cvReleaseMat( &ab ); - cvReleaseMat( &temp ); + return min_idx; } - -double CvDTree::update_tree_rnc( int T, int fold ) +double DTreesImpl::updateTreeRNC( int root, double T, int fold ) { - CvDTreeNode* node = root; + int nidx = root, pidx = -1, cv_n = params.CVFolds; double min_alpha = DBL_MAX; for(;;) { - CvDTreeNode* parent; + WNode *node = 0, *parent = 0; + for(;;) { - int t = fold >= 0 ? node->cv_Tn[fold] : node->Tn; - if( t <= T || !node->left ) + node = &w->wnodes[nidx]; + double t = fold >= 0 ? w->cv_Tn[nidx*cv_n + fold] : node->Tn; + if( t <= T || node->left < 0 ) { node->complexity = 1; node->tree_risk = node->node_risk; node->tree_error = 0.; if( fold >= 0 ) { - node->tree_risk = node->cv_node_risk[fold]; - node->tree_error = node->cv_node_error[fold]; + node->tree_risk = w->cv_node_risk[nidx*cv_n + fold]; + node->tree_error = w->cv_node_error[nidx*cv_n + fold]; } break; } - node = node->left; + nidx = node->left; } - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) + for( pidx = node->parent; pidx >= 0 && w->wnodes[pidx].right == nidx; + nidx = pidx, pidx = w->wnodes[pidx].parent ) { + node = &w->wnodes[nidx]; + parent = &w->wnodes[pidx]; parent->complexity += node->complexity; parent->tree_risk += node->tree_risk; parent->tree_error += node->tree_error; - parent->alpha = ((fold >= 0 ? parent->cv_node_risk[fold] : parent->node_risk) - - parent->tree_risk)/(parent->complexity - 1); - min_alpha = MIN( min_alpha, parent->alpha ); + parent->alpha = ((fold >= 0 ? w->cv_node_risk[pidx*cv_n + fold] : parent->node_risk) + - parent->tree_risk)/(parent->complexity - 1); + min_alpha = std::min( min_alpha, parent->alpha ); } - if( !parent ) + if( pidx < 0 ) break; + node = &w->wnodes[nidx]; + parent = &w->wnodes[pidx]; parent->complexity = node->complexity; parent->tree_risk = node->tree_risk; parent->tree_error = node->tree_error; - node = parent->right; + nidx = parent->right; } return min_alpha; } - -int CvDTree::cut_tree( int T, int fold, double min_alpha ) +bool DTreesImpl::cutTree( int root, double T, int fold, double min_alpha ) { - CvDTreeNode* node = root; - if( !node->left ) - return 1; + int cv_n = params.CVFolds, nidx = root, pidx = -1; + WNode* node = &w->wnodes[root]; + if( node->left < 0 ) + return true; for(;;) { - CvDTreeNode* parent; for(;;) { - int t = fold >= 0 ? node->cv_Tn[fold] : node->Tn; - if( t <= T || !node->left ) + node = &w->wnodes[nidx]; + double t = fold >= 0 ? w->cv_Tn[nidx*cv_n + fold] : node->Tn; + if( t <= T || node->left < 0 ) break; if( node->alpha <= min_alpha + FLT_EPSILON ) { if( fold >= 0 ) - node->cv_Tn[fold] = T; + w->cv_Tn[nidx*cv_n + fold] = T; else node->Tn = T; - if( node == root ) - return 1; + if( nidx == root ) + return true; break; } - node = node->left; + nidx = node->left; } - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) + for( pidx = node->parent; pidx >= 0 && w->wnodes[pidx].right == nidx; + nidx = pidx, pidx = w->wnodes[pidx].parent ) ; - if( !parent ) + if( pidx < 0 ) break; - node = parent->right; + nidx = w->wnodes[pidx].right; } - return 0; + return false; } - -void CvDTree::free_prune_data(bool _cut_tree) +float DTreesImpl::predictTrees( const Range& range, const Mat& sample, int flags ) const { - CvDTreeNode* node = root; - - for(;;) - { - CvDTreeNode* parent; - for(;;) - { - // do not call cvSetRemoveByPtr( cv_heap, node->cv_Tn ) - // as we will clear the whole cross-validation heap at the end - node->cv_Tn = 0; - node->cv_node_error = node->cv_node_risk = 0; - if( !node->left ) - break; - node = node->left; - } - - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) - { - if( _cut_tree && parent->Tn <= pruned_tree_idx ) - { - data->free_node( parent->left ); - data->free_node( parent->right ); - parent->left = parent->right = 0; - } - } - - if( !parent ) - break; + CV_Assert( sample.type() == CV_32F ); - node = parent->right; - } - - if( data->cv_heap ) - cvClearSet( data->cv_heap ); -} + int predictType = flags & PREDICT_MASK; + int nvars = (int)varIdx.size(); + if( nvars == 0 ) + nvars = (int)varType.size(); + int i, ncats = (int)catOfs.size(), nclasses = (int)classLabels.size(); + int catbufsize = ncats > 0 ? nvars : 0; + AutoBuffer buf(nclasses + catbufsize + 1); + int* votes = buf; + int* catbuf = votes + nclasses; + const int* cvidx = (flags & (COMPRESSED_INPUT|PREPROCESSED_INPUT)) == 0 && !varIdx.empty() ? &compVarIdx[0] : 0; + const uchar* vtype = &varType[0]; + const Vec2i* cofs = !catOfs.empty() ? &catOfs[0] : 0; + const int* cmap = !catMap.empty() ? &catMap[0] : 0; + const float* psample = sample.ptr(); + const float* missingSubstPtr = !missingSubst.empty() ? &missingSubst[0] : 0; + size_t sstep = sample.isContinuous() ? 1 : sample.step/sizeof(float); + double sum = 0.; + int lastClassIdx = -1; + const float MISSED_VAL = TrainData::missingValue(); + for( i = 0; i < catbufsize; i++ ) + catbuf[i] = -1; -void CvDTree::free_tree() -{ - if( root && data && data->shared ) + if( predictType == PREDICT_AUTO ) { - pruned_tree_idx = INT_MIN; - free_prune_data(true); - data->free_node(root); - root = 0; + predictType = !_isClassifier || (classLabels.size() == 2 && (flags & RAW_OUTPUT) != 0) ? + PREDICT_SUM : PREDICT_MAX_VOTE; } -} - -CvDTreeNode* CvDTree::predict( const CvMat* _sample, - const CvMat* _missing, bool preprocessed_input ) const -{ - cv::AutoBuffer catbuf; - - int i, mstep = 0; - const uchar* m = 0; - CvDTreeNode* node = root; - if( !node ) - CV_Error( CV_StsError, "The tree has not been trained yet" ); - - if( !CV_IS_MAT(_sample) || CV_MAT_TYPE(_sample->type) != CV_32FC1 || - (_sample->cols != 1 && _sample->rows != 1) || - (_sample->cols + _sample->rows - 1 != data->var_all && !preprocessed_input) || - (_sample->cols + _sample->rows - 1 != data->var_count && preprocessed_input) ) - CV_Error( CV_StsBadArg, - "the input sample must be 1d floating-point vector with the same " - "number of elements as the total number of variables used for training" ); - - const float* sample = _sample->data.fl; - int step = CV_IS_MAT_CONT(_sample->type) ? 1 : _sample->step/sizeof(sample[0]); - - if( data->cat_count && !preprocessed_input ) // cache for categorical variables + if( predictType == PREDICT_MAX_VOTE ) { - int n = data->cat_count->cols; - catbuf.allocate(n); - for( i = 0; i < n; i++ ) - catbuf[i] = -1; + for( i = 0; i < nclasses; i++ ) + votes[i] = 0; } - if( _missing ) + for( int ridx = range.start; ridx < range.end; ridx++ ) { - if( !CV_IS_MAT(_missing) || !CV_IS_MASK_ARR(_missing) || - !CV_ARE_SIZES_EQ(_missing, _sample) ) - CV_Error( CV_StsBadArg, - "the missing data mask must be 8-bit vector of the same size as input sample" ); - m = _missing->data.ptr; - mstep = CV_IS_MAT_CONT(_missing->type) ? 1 : _missing->step/sizeof(m[0]); - } - - const int* vtype = data->var_type->data.i; - const int* vidx = data->var_idx && !preprocessed_input ? data->var_idx->data.i : 0; - const int* cmap = data->cat_map ? data->cat_map->data.i : 0; - const int* cofs = data->cat_ofs ? data->cat_ofs->data.i : 0; + int nidx = roots[ridx], prev = nidx, c = 0; - while( node->Tn > pruned_tree_idx && node->left ) - { - CvDTreeSplit* split = node->split; - int dir = 0; - for( ; !dir && split != 0; split = split->next ) + for(;;) { - int vi = split->var_idx; - int ci = vtype[vi]; - i = vidx ? vidx[vi] : vi; - float val = sample[(size_t)i*step]; - if( m && m[(size_t)i*mstep] ) - continue; - if( ci < 0 ) // ordered - dir = val <= split->ord.c ? -1 : 1; - else // categorical + prev = nidx; + const Node& node = nodes[nidx]; + if( node.split < 0 ) + break; + const Split& split = splits[node.split]; + int vi = split.varIdx; + int ci = cvidx ? cvidx[vi] : vi; + float val = psample[ci*sstep]; + if( val == MISSED_VAL ) + { + if( !missingSubstPtr ) + { + nidx = node.defaultDir < 0 ? node.left : node.right; + continue; + } + val = missingSubstPtr[vi]; + } + + if( vtype[vi] == VAR_ORDERED ) + nidx = val <= split.c ? node.left : node.right; + else { - int c; - if( preprocessed_input ) + if( flags & PREPROCESSED_INPUT ) c = cvRound(val); else { c = catbuf[ci]; if( c < 0 ) { - int a = c = cofs[ci]; - int b = (ci+1 >= data->cat_ofs->cols) ? data->cat_map->cols : cofs[ci+1]; + int a = c = cofs[vi][0]; + int b = cofs[vi][1]; int ival = cvRound(val); if( ival != val ) CV_Error( CV_StsBadArg, - "one of input categorical variable is not an integer" ); + "one of input categorical variable is not an integer" ); - int sh = 0; while( a < b ) { - sh++; c = (a + b) >> 1; if( ival < cmap[c] ) b = c; @@ -3706,446 +1482,423 @@ CvDTreeNode* CvDTree::predict( const CvMat* _sample, break; } - if( c < 0 || ival != cmap[c] ) - continue; + CV_Assert( c >= 0 && ival == cmap[c] ); - catbuf[ci] = c -= cofs[ci]; + c -= cofs[vi][0]; + catbuf[ci] = c; } + const int* subset = &subsets[split.subsetOfs]; + unsigned u = c; + nidx = CV_DTREE_CAT_DIR(u, subset) < 0 ? node.left : node.right; } - c = ( (c == 65535) && data->is_buf_16u ) ? -1 : c; - dir = CV_DTREE_CAT_DIR(c, split->subset); } + } - if( split->inversed ) - dir = -dir; + if( predictType == PREDICT_SUM ) + sum += nodes[prev].value; + else + { + lastClassIdx = nodes[prev].classIdx; + votes[lastClassIdx]++; } + } - if( !dir ) + if( predictType == PREDICT_MAX_VOTE ) + { + int best_idx = lastClassIdx; + if( range.end - range.start > 1 ) { - double diff = node->right->sample_count - node->left->sample_count; - dir = diff < 0 ? -1 : 1; + best_idx = 0; + for( i = 1; i < nclasses; i++ ) + if( votes[best_idx] < votes[i] ) + best_idx = i; } - node = dir < 0 ? node->left : node->right; + sum = (flags & RAW_OUTPUT) ? (float)best_idx : classLabels[best_idx]; } - return node; + return (float)sum; } -CvDTreeNode* CvDTree::predict( const Mat& _sample, const Mat& _missing, bool preprocessed_input ) const +float DTreesImpl::predict( InputArray _samples, OutputArray _results, int flags ) const { - CvMat sample = _sample, mmask = _missing; - return predict(&sample, mmask.data.ptr ? &mmask : 0, preprocessed_input); -} + CV_Assert( !roots.empty() ); + Mat samples = _samples.getMat(), results; + int i, nsamples = samples.rows; + int rtype = CV_32F; + bool needresults = _results.needed(); + float retval = 0.f; + bool iscls = isClassifier(); + float scale = !iscls ? 1.f/(int)roots.size() : 1.f; + if( iscls && (flags & PREDICT_MASK) == PREDICT_MAX_VOTE ) + rtype = CV_32S; -const CvMat* CvDTree::get_var_importance() -{ - if( !var_importance ) + if( needresults ) { - CvDTreeNode* node = root; - double* importance; - if( !node ) - return 0; - var_importance = cvCreateMat( 1, data->var_count, CV_64F ); - cvZero( var_importance ); - importance = var_importance->data.db; + _results.create(nsamples, 1, rtype); + results = _results.getMat(); + } + else + nsamples = std::min(nsamples, 1); - for(;;) + for( i = 0; i < nsamples; i++ ) + { + float val = predictTrees( Range(0, (int)roots.size()), samples.row(i), flags )*scale; + if( needresults ) { - CvDTreeNode* parent; - for( ;; node = node->left ) - { - CvDTreeSplit* split = node->split; + if( rtype == CV_32F ) + results.at(i) = val; + else + results.at(i) = cvRound(val); + } + if( i == 0 ) + retval = val; + } + return retval; +} - if( !node->left || node->Tn <= pruned_tree_idx ) - break; +void DTreesImpl::writeTrainingParams(FileStorage& fs) const +{ + fs << "use_surrogates" << (params0.useSurrogates ? 1 : 0); + fs << "max_categories" << params0.maxCategories; + fs << "regression_accuracy" << params0.regressionAccuracy; - for( ; split != 0; split = split->next ) - importance[split->var_idx] += split->quality; - } + fs << "max_depth" << params0.maxDepth; + fs << "min_sample_count" << params0.minSampleCount; + fs << "cross_validation_folds" << params0.CVFolds; - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) - ; + if( params0.CVFolds > 1 ) + fs << "use_1se_rule" << (params0.use1SERule ? 1 : 0); - if( !parent ) - break; + if( !params0.priors.empty() ) + fs << "priors" << params0.priors; +} - node = parent->right; - } +void DTreesImpl::writeParams(FileStorage& fs) const +{ + fs << "is_classifier" << isClassifier(); + fs << "var_all" << (int)varType.size(); + fs << "var_count" << getVarCount(); - cvNormalize( var_importance, var_importance, 1., 0, CV_L1 ); - } + int ord_var_count = 0, cat_var_count = 0; + int i, n = (int)varType.size(); + for( i = 0; i < n; i++ ) + if( varType[i] == VAR_ORDERED ) + ord_var_count++; + else + cat_var_count++; + fs << "ord_var_count" << ord_var_count; + fs << "cat_var_count" << cat_var_count; - return var_importance; -} + fs << "training_params" << "{"; + writeTrainingParams(fs); + + fs << "}"; + + if( !varIdx.empty() ) + fs << "var_idx" << varIdx; + + fs << "var_type" << varType; + if( !catOfs.empty() ) + fs << "cat_ofs" << catOfs; + if( !catMap.empty() ) + fs << "cat_map" << catMap; + if( !classLabels.empty() ) + fs << "class_labels" << classLabels; + if( !missingSubst.empty() ) + fs << "missing_subst" << missingSubst; +} -void CvDTree::write_split( CvFileStorage* fs, CvDTreeSplit* split ) const +void DTreesImpl::writeSplit( FileStorage& fs, int splitidx ) const { - int ci; + const Split& split = splits[splitidx]; + + fs << "{:"; - cvStartWriteStruct( fs, 0, CV_NODE_MAP + CV_NODE_FLOW ); - cvWriteInt( fs, "var", split->var_idx ); - cvWriteReal( fs, "quality", split->quality ); + int vi = split.varIdx; + fs << "var" << vi; + fs << "quality" << split.quality; - ci = data->get_var_type(split->var_idx); - if( ci >= 0 ) // split on a categorical var + if( varType[vi] == VAR_CATEGORICAL ) // split on a categorical var { - int i, n = data->cat_count->data.i[ci], to_right = 0, default_dir; + int i, n = getCatCount(vi), to_right = 0; + const int* subset = &subsets[split.subsetOfs]; for( i = 0; i < n; i++ ) - to_right += CV_DTREE_CAT_DIR(i,split->subset) > 0; + to_right += CV_DTREE_CAT_DIR(i, subset) > 0; // ad-hoc rule when to use inverse categorical split notation // to achieve more compact and clear representation - default_dir = to_right <= 1 || to_right <= MIN(3, n/2) || to_right <= n/3 ? -1 : 1; + int default_dir = to_right <= 1 || to_right <= std::min(3, n/2) || to_right <= n/3 ? -1 : 1; - cvStartWriteStruct( fs, default_dir*(split->inversed ? -1 : 1) > 0 ? - "in" : "not_in", CV_NODE_SEQ+CV_NODE_FLOW ); + fs << (default_dir*(split.inversed ? -1 : 1) > 0 ? "in" : "not_in") << "[:"; for( i = 0; i < n; i++ ) { - int dir = CV_DTREE_CAT_DIR(i,split->subset); + int dir = CV_DTREE_CAT_DIR(i, subset); if( dir*default_dir < 0 ) - cvWriteInt( fs, 0, i ); + fs << i; } - cvEndWriteStruct( fs ); + + fs << "]"; } else - cvWriteReal( fs, !split->inversed ? "le" : "gt", split->ord.c ); + fs << (!split.inversed ? "le" : "gt") << split.c; - cvEndWriteStruct( fs ); + fs << "}"; } - -void CvDTree::write_node( CvFileStorage* fs, CvDTreeNode* node ) const +void DTreesImpl::writeNode( FileStorage& fs, int nidx, int depth ) const { - CvDTreeSplit* split; + const Node& node = nodes[nidx]; + fs << "{"; + fs << "depth" << depth; + fs << "value" << node.value; - cvStartWriteStruct( fs, 0, CV_NODE_MAP ); + if( _isClassifier ) + fs << "norm_class_idx" << node.classIdx; - cvWriteInt( fs, "depth", node->depth ); - cvWriteInt( fs, "sample_count", node->sample_count ); - cvWriteReal( fs, "value", node->value ); - - if( data->is_classifier ) - cvWriteInt( fs, "norm_class_idx", node->class_idx ); - - cvWriteInt( fs, "Tn", node->Tn ); - cvWriteInt( fs, "complexity", node->complexity ); - cvWriteReal( fs, "alpha", node->alpha ); - cvWriteReal( fs, "node_risk", node->node_risk ); - cvWriteReal( fs, "tree_risk", node->tree_risk ); - cvWriteReal( fs, "tree_error", node->tree_error ); - - if( node->left ) + if( node.split >= 0 ) { - cvStartWriteStruct( fs, "splits", CV_NODE_SEQ ); + fs << "splits" << "["; - for( split = node->split; split != 0; split = split->next ) - write_split( fs, split ); + for( int splitidx = node.split; splitidx >= 0; splitidx = splits[splitidx].next ) + writeSplit( fs, splitidx ); - cvEndWriteStruct( fs ); + fs << "]"; } - cvEndWriteStruct( fs ); + fs << "}"; } - -void CvDTree::write_tree_nodes( CvFileStorage* fs ) const +void DTreesImpl::writeTree( FileStorage& fs, int root ) const { - //CV_FUNCNAME( "CvDTree::write_tree_nodes" ); + fs << "nodes" << "["; - __BEGIN__; - - CvDTreeNode* node = root; + int nidx = root, pidx = 0, depth = 0; + const Node *node = 0; // traverse the tree and save all the nodes in depth-first order for(;;) { - CvDTreeNode* parent; for(;;) { - write_node( fs, node ); - if( !node->left ) + writeNode( fs, nidx, depth ); + node = &nodes[nidx]; + if( node->left < 0 ) break; - node = node->left; + nidx = node->left; + depth++; } - for( parent = node->parent; parent && parent->right == node; - node = parent, parent = parent->parent ) - ; + for( pidx = node->parent; pidx >= 0 && nodes[pidx].right == nidx; + nidx = pidx, pidx = nodes[pidx].parent ) + depth--; - if( !parent ) + if( pidx < 0 ) break; - node = parent->right; + nidx = nodes[pidx].right; } - __END__; + fs << "]"; } - -void CvDTree::write( CvFileStorage* fs, const char* name ) const +void DTreesImpl::write( FileStorage& fs ) const { - //CV_FUNCNAME( "CvDTree::write" ); - - __BEGIN__; - - cvStartWriteStruct( fs, name, CV_NODE_MAP, CV_TYPE_NAME_ML_TREE ); - - //get_var_importance(); - data->write_params( fs ); - //if( var_importance ) - //cvWrite( fs, "var_importance", var_importance ); - write( fs ); - - cvEndWriteStruct( fs ); - - __END__; + writeParams(fs); + writeTree(fs, roots[0]); } - -void CvDTree::write( CvFileStorage* fs ) const +void DTreesImpl::readParams( const FileNode& fn ) { - //CV_FUNCNAME( "CvDTree::write" ); + _isClassifier = (int)fn["is_classifier"] != 0; + /*int var_all = (int)fn["var_all"]; + int var_count = (int)fn["var_count"]; + int cat_var_count = (int)fn["cat_var_count"]; + int ord_var_count = (int)fn["ord_var_count"];*/ - __BEGIN__; + FileNode tparams_node = fn["training_params"]; - cvWriteInt( fs, "best_tree_idx", pruned_tree_idx ); + params0 = Params(); - cvStartWriteStruct( fs, "nodes", CV_NODE_SEQ ); - write_tree_nodes( fs ); - cvEndWriteStruct( fs ); + if( !tparams_node.empty() ) // training parameters are not necessary + { + params0.useSurrogates = (int)tparams_node["use_surrogates"] != 0; + params0.maxCategories = (int)tparams_node["max_categories"]; + params0.regressionAccuracy = (float)tparams_node["regression_accuracy"]; - __END__; -} + params0.maxDepth = (int)tparams_node["max_depth"]; + params0.minSampleCount = (int)tparams_node["min_sample_count"]; + params0.CVFolds = (int)tparams_node["cross_validation_folds"]; + if( params0.CVFolds > 1 ) + { + params.use1SERule = (int)tparams_node["use_1se_rule"] != 0; + } -CvDTreeSplit* CvDTree::read_split( CvFileStorage* fs, CvFileNode* fnode ) -{ - CvDTreeSplit* split = 0; + tparams_node["priors"] >> params0.priors; + } - CV_FUNCNAME( "CvDTree::read_split" ); + fn["var_idx"] >> varIdx; + fn["var_type"] >> varType; - __BEGIN__; + fn["cat_ofs"] >> catOfs; + fn["cat_map"] >> catMap; + fn["missing_subst"] >> missingSubst; + fn["class_labels"] >> classLabels; - int vi, ci; + initCompVarIdx(); + setDParams(params0); +} - if( !fnode || CV_NODE_TYPE(fnode->tag) != CV_NODE_MAP ) - CV_ERROR( CV_StsParseError, "some of the splits are not stored properly" ); +int DTreesImpl::readSplit( const FileNode& fn ) +{ + Split split; - vi = cvReadIntByName( fs, fnode, "var", -1 ); - if( (unsigned)vi >= (unsigned)data->var_count ) - CV_ERROR( CV_StsOutOfRange, "Split variable index is out of range" ); + int vi = (int)fn["var"]; + CV_Assert( 0 <= vi && vi <= (int)varType.size() ); + split.varIdx = vi; - ci = data->get_var_type(vi); - if( ci >= 0 ) // split on categorical var + if( varType[vi] == VAR_CATEGORICAL ) // split on categorical var { - int i, n = data->cat_count->data.i[ci], inversed = 0, val; - CvSeqReader reader; - CvFileNode* inseq; - split = data->new_split_cat( vi, 0 ); - inseq = cvGetFileNodeByName( fs, fnode, "in" ); - if( !inseq ) + int i, val, ssize = getSubsetSize(vi); + split.subsetOfs = (int)subsets.size(); + for( i = 0; i < ssize; i++ ) + subsets.push_back(0); + int* subset = &subsets[split.subsetOfs]; + FileNode fns = fn["in"]; + if( fns.empty() ) { - inseq = cvGetFileNodeByName( fs, fnode, "not_in" ); - inversed = 1; + fns = fn["not_in"]; + split.inversed = true; } - if( !inseq || - (CV_NODE_TYPE(inseq->tag) != CV_NODE_SEQ && CV_NODE_TYPE(inseq->tag) != CV_NODE_INT)) - CV_ERROR( CV_StsParseError, - "Either 'in' or 'not_in' tags should be inside a categorical split data" ); - if( CV_NODE_TYPE(inseq->tag) == CV_NODE_INT ) + if( fns.isInt() ) { - val = inseq->data.i; - if( (unsigned)val >= (unsigned)n ) - CV_ERROR( CV_StsOutOfRange, "some of in/not_in elements are out of range" ); - - split->subset[val >> 5] |= 1 << (val & 31); + val = (int)fns; + subset[val >> 5] |= 1 << (val & 31); } else { - cvStartReadSeq( inseq->data.seq, &reader ); - - for( i = 0; i < reader.seq->total; i++ ) + FileNodeIterator it = fns.begin(); + int n = (int)fns.size(); + for( i = 0; i < n; i++, ++it ) { - CvFileNode* inode = (CvFileNode*)reader.ptr; - val = inode->data.i; - if( CV_NODE_TYPE(inode->tag) != CV_NODE_INT || (unsigned)val >= (unsigned)n ) - CV_ERROR( CV_StsOutOfRange, "some of in/not_in elements are out of range" ); - - split->subset[val >> 5] |= 1 << (val & 31); - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + val = (int)*it; + subset[val >> 5] |= 1 << (val & 31); } } // for categorical splits we do not use inversed splits, // instead we inverse the variable set in the split - if( inversed ) - for( i = 0; i < (n + 31) >> 5; i++ ) - split->subset[i] ^= -1; + if( split.inversed ) + { + for( i = 0; i < ssize; i++ ) + subset[i] ^= -1; + split.inversed = false; + } } else { - CvFileNode* cmp_node; - split = data->new_split_ord( vi, 0, 0, 0, 0 ); - - cmp_node = cvGetFileNodeByName( fs, fnode, "le" ); - if( !cmp_node ) + FileNode cmpNode = fn["le"]; + if( cmpNode.empty() ) { - cmp_node = cvGetFileNodeByName( fs, fnode, "gt" ); - split->inversed = 1; + cmpNode = fn["gt"]; + split.inversed = true; } - - split->ord.c = (float)cvReadReal( cmp_node ); + split.c = (float)cmpNode; } - split->quality = (float)cvReadRealByName( fs, fnode, "quality" ); + split.quality = (float)fn["quality"]; + splits.push_back(split); - __END__; - - return split; + return (int)(splits.size() - 1); } - -CvDTreeNode* CvDTree::read_node( CvFileStorage* fs, CvFileNode* fnode, CvDTreeNode* parent ) +int DTreesImpl::readNode( const FileNode& fn ) { - CvDTreeNode* node = 0; - - CV_FUNCNAME( "CvDTree::read_node" ); - - __BEGIN__; - - CvFileNode* splits; - int i, depth; - - if( !fnode || CV_NODE_TYPE(fnode->tag) != CV_NODE_MAP ) - CV_ERROR( CV_StsParseError, "some of the tree elements are not stored properly" ); - - CV_CALL( node = data->new_node( parent, 0, 0, 0 )); - depth = cvReadIntByName( fs, fnode, "depth", -1 ); - if( depth != node->depth ) - CV_ERROR( CV_StsParseError, "incorrect node depth" ); + Node node; + node.value = (double)fn["value"]; - node->sample_count = cvReadIntByName( fs, fnode, "sample_count" ); - node->value = cvReadRealByName( fs, fnode, "value" ); - if( data->is_classifier ) - node->class_idx = cvReadIntByName( fs, fnode, "norm_class_idx" ); + if( _isClassifier ) + node.classIdx = (int)fn["norm_class_idx"]; - node->Tn = cvReadIntByName( fs, fnode, "Tn" ); - node->complexity = cvReadIntByName( fs, fnode, "complexity" ); - node->alpha = cvReadRealByName( fs, fnode, "alpha" ); - node->node_risk = cvReadRealByName( fs, fnode, "node_risk" ); - node->tree_risk = cvReadRealByName( fs, fnode, "tree_risk" ); - node->tree_error = cvReadRealByName( fs, fnode, "tree_error" ); - - splits = cvGetFileNodeByName( fs, fnode, "splits" ); - if( splits ) + FileNode sfn = fn["splits"]; + if( !sfn.empty() ) { - CvSeqReader reader; - CvDTreeSplit* last_split = 0; - - if( CV_NODE_TYPE(splits->tag) != CV_NODE_SEQ ) - CV_ERROR( CV_StsParseError, "splits tag must stored as a sequence" ); + int i, n = (int)sfn.size(), prevsplit = -1; + FileNodeIterator it = sfn.begin(); - cvStartReadSeq( splits->data.seq, &reader ); - for( i = 0; i < reader.seq->total; i++ ) + for( i = 0; i < n; i++, ++it ) { - CvDTreeSplit* split; - CV_CALL( split = read_split( fs, (CvFileNode*)reader.ptr )); - if( !last_split ) - node->split = last_split = split; + int splitidx = readSplit(*it); + if( splitidx < 0 ) + break; + if( prevsplit < 0 ) + node.split = splitidx; else - last_split = last_split->next = split; - - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); + splits[prevsplit].next = splitidx; + prevsplit = splitidx; } } - - __END__; - - return node; + nodes.push_back(node); + return (int)(nodes.size() - 1); } - -void CvDTree::read_tree_nodes( CvFileStorage* fs, CvFileNode* fnode ) +int DTreesImpl::readTree( const FileNode& fn ) { - CV_FUNCNAME( "CvDTree::read_tree_nodes" ); - - __BEGIN__; + int i, n = (int)fn.size(), root = -1, pidx = -1; + FileNodeIterator it = fn.begin(); - CvSeqReader reader; - CvDTreeNode _root; - CvDTreeNode* parent = &_root; - int i; - parent->left = parent->right = parent->parent = 0; - - cvStartReadSeq( fnode->data.seq, &reader ); - - for( i = 0; i < reader.seq->total; i++ ) + for( i = 0; i < n; i++, ++it ) { - CvDTreeNode* node; - - CV_CALL( node = read_node( fs, (CvFileNode*)reader.ptr, parent != &_root ? parent : 0 )); - if( !parent->left ) - parent->left = node; + int nidx = readNode(*it); + if( nidx < 0 ) + break; + Node& node = nodes[nidx]; + node.parent = pidx; + if( pidx < 0 ) + root = nidx; else - parent->right = node; - if( node->split ) - parent = node; + { + Node& parent = nodes[pidx]; + if( parent.left < 0 ) + parent.left = nidx; + else + parent.right = nidx; + } + if( node.split >= 0 ) + pidx = nidx; else { - while( parent && parent->right ) - parent = parent->parent; + while( pidx >= 0 && nodes[pidx].right >= 0 ) + pidx = nodes[pidx].parent; } - - CV_NEXT_SEQ_ELEM( reader.seq->elem_size, reader ); } - - root = _root.left; - - __END__; + roots.push_back(root); + return root; } - -void CvDTree::read( CvFileStorage* fs, CvFileNode* fnode ) +void DTreesImpl::read( const FileNode& fn ) { - CvDTreeTrainData* _data = new CvDTreeTrainData(); - _data->read_params( fs, fnode ); + clear(); + readParams(fn); - read( fs, fnode, _data ); - get_var_importance(); + FileNode fnodes = fn["nodes"]; + CV_Assert( !fnodes.empty() ); + readTree(fnodes); } - -// a special entry point for reading weak decision trees from the tree ensembles -void CvDTree::read( CvFileStorage* fs, CvFileNode* node, CvDTreeTrainData* _data ) +Ptr DTrees::create(const DTrees::Params& params) { - CV_FUNCNAME( "CvDTree::read" ); - - __BEGIN__; - - CvFileNode* tree_nodes; - - clear(); - data = _data; - - tree_nodes = cvGetFileNodeByName( fs, node, "nodes" ); - if( !tree_nodes || CV_NODE_TYPE(tree_nodes->tag) != CV_NODE_SEQ ) - CV_ERROR( CV_StsParseError, "nodes tag is missing" ); - - pruned_tree_idx = cvReadIntByName( fs, node, "best_tree_idx", -1 ); - read_tree_nodes( fs, tree_nodes ); - - __END__; + Ptr p = makePtr(); + p->setDParams(params); + return p; } -Mat CvDTree::getVarImportance() -{ - return cvarrToMat(get_var_importance()); +} } /* End of file. */ diff --git a/modules/ml/test/test_emknearestkmeans.cpp b/modules/ml/test/test_emknearestkmeans.cpp index a14b636061970326c4f68a44b6496aefa64712cb..98b88c70111310cf225afebf82df44cce25b6353 100644 --- a/modules/ml/test/test_emknearestkmeans.cpp +++ b/modules/ml/test/test_emknearestkmeans.cpp @@ -43,6 +43,9 @@ using namespace std; using namespace cv; +using cv::ml::TrainData; +using cv::ml::EM; +using cv::ml::KNearest; static void defaultDistribs( Mat& means, vector& covs, int type=CV_32FC1 ) @@ -309,9 +312,9 @@ void CV_KNearestTest::run( int /*start_from*/ ) generateData( testData, testLabels, sizes, means, covs, CV_32FC1, CV_32FC1 ); int code = cvtest::TS::OK; - KNearest knearest; - knearest.train( trainData, trainLabels ); - knearest.find_nearest( testData, 4, &bestLabels ); + Ptr knearest = KNearest::create(true); + knearest->train(trainData, cv::ml::ROW_SAMPLE, trainLabels); + knearest->findNearest( testData, 4, bestLabels); float err; if( !calcErr( bestLabels, testLabels, sizes, err, true ) ) { @@ -373,13 +376,16 @@ int CV_EMTest::runCase( int caseIndex, const EM_Params& params, cv::Mat labels; float err; - cv::EM em(params.nclusters, params.covMatType, params.termCrit); + Ptr em; + EM::Params emp(params.nclusters, params.covMatType, params.termCrit); if( params.startStep == EM::START_AUTO_STEP ) - em.train( trainData, noArray(), labels ); + em = EM::train( trainData, noArray(), labels, noArray(), emp ); else if( params.startStep == EM::START_E_STEP ) - em.trainE( trainData, *params.means, *params.covs, *params.weights, noArray(), labels ); + em = EM::train_startWithE( trainData, *params.means, *params.covs, + *params.weights, noArray(), labels, noArray(), emp ); else if( params.startStep == EM::START_M_STEP ) - em.trainM( trainData, *params.probs, noArray(), labels ); + em = EM::train_startWithM( trainData, *params.probs, + noArray(), labels, noArray(), emp ); // check train error if( !calcErr( labels, trainLabels, sizes, err , false, false ) ) @@ -399,7 +405,7 @@ int CV_EMTest::runCase( int caseIndex, const EM_Params& params, { Mat sample = testData.row(i); Mat probs; - labels.at(i) = static_cast(em.predict( sample, probs )[1]); + labels.at(i) = static_cast(em->predict2( sample, probs )[1]); } if( !calcErr( labels, testLabels, sizes, err, false, false ) ) { @@ -446,56 +452,56 @@ void CV_EMTest::run( int /*start_from*/ ) int code = cvtest::TS::OK; int caseIndex = 0; { - params.startStep = cv::EM::START_AUTO_STEP; - params.covMatType = cv::EM::COV_MAT_GENERIC; + params.startStep = EM::START_AUTO_STEP; + params.covMatType = EM::COV_MAT_GENERIC; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_AUTO_STEP; - params.covMatType = cv::EM::COV_MAT_DIAGONAL; + params.startStep = EM::START_AUTO_STEP; + params.covMatType = EM::COV_MAT_DIAGONAL; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_AUTO_STEP; - params.covMatType = cv::EM::COV_MAT_SPHERICAL; + params.startStep = EM::START_AUTO_STEP; + params.covMatType = EM::COV_MAT_SPHERICAL; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_M_STEP; - params.covMatType = cv::EM::COV_MAT_GENERIC; + params.startStep = EM::START_M_STEP; + params.covMatType = EM::COV_MAT_GENERIC; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_M_STEP; - params.covMatType = cv::EM::COV_MAT_DIAGONAL; + params.startStep = EM::START_M_STEP; + params.covMatType = EM::COV_MAT_DIAGONAL; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_M_STEP; - params.covMatType = cv::EM::COV_MAT_SPHERICAL; + params.startStep = EM::START_M_STEP; + params.covMatType = EM::COV_MAT_SPHERICAL; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_E_STEP; - params.covMatType = cv::EM::COV_MAT_GENERIC; + params.startStep = EM::START_E_STEP; + params.covMatType = EM::COV_MAT_GENERIC; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_E_STEP; - params.covMatType = cv::EM::COV_MAT_DIAGONAL; + params.startStep = EM::START_E_STEP; + params.covMatType = EM::COV_MAT_DIAGONAL; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } { - params.startStep = cv::EM::START_E_STEP; - params.covMatType = cv::EM::COV_MAT_SPHERICAL; + params.startStep = EM::START_E_STEP; + params.covMatType = EM::COV_MAT_SPHERICAL; int currCode = runCase(caseIndex++, params, trainData, trainLabels, testData, testLabels, sizes); code = currCode == cvtest::TS::OK ? code : currCode; } @@ -511,7 +517,6 @@ protected: { int code = cvtest::TS::OK; const int nclusters = 2; - cv::EM em(nclusters); Mat samples = Mat(3,1,CV_64FC1); samples.at(0,0) = 1; @@ -520,11 +525,11 @@ protected: Mat labels; - em.train(samples, labels); + Ptr em = EM::train(samples, noArray(), labels, noArray(), EM::Params(nclusters)); Mat firstResult(samples.rows, 1, CV_32SC1); for( int i = 0; i < samples.rows; i++) - firstResult.at(i) = static_cast(em.predict(samples.row(i))[1]); + firstResult.at(i) = static_cast(em->predict2(samples.row(i), noArray())[1]); // Write out string filename = cv::tempfile(".xml"); @@ -533,7 +538,7 @@ protected: try { fs << "em" << "{"; - em.write(fs); + em->write(fs); fs << "}"; } catch(...) @@ -543,29 +548,24 @@ protected: } } - em.clear(); + em.release(); // Read in + try { - FileStorage fs = FileStorage(filename, FileStorage::READ); - CV_Assert(fs.isOpened()); - FileNode fn = fs["em"]; - try - { - em.read(fn); - } - catch(...) - { - ts->printf( cvtest::TS::LOG, "Crash in read method.\n" ); - ts->set_failed_test_info( cvtest::TS::FAIL_EXCEPTION ); - } + em = StatModel::load(filename); + } + catch(...) + { + ts->printf( cvtest::TS::LOG, "Crash in read method.\n" ); + ts->set_failed_test_info( cvtest::TS::FAIL_EXCEPTION ); } remove( filename.c_str() ); int errCaseCount = 0; for( int i = 0; i < samples.rows; i++) - errCaseCount = std::abs(em.predict(samples.row(i))[1] - firstResult.at(i)) < FLT_EPSILON ? 0 : 1; + errCaseCount = std::abs(em->predict2(samples.row(i), noArray())[1] - firstResult.at(i)) < FLT_EPSILON ? 0 : 1; if( errCaseCount > 0 ) { @@ -588,21 +588,18 @@ protected: // 1. estimates distributions of "spam" / "not spam" // 2. predict classID using Bayes classifier for estimated distributions. - CvMLData data; string dataFilename = string(ts->get_data_path()) + "spambase.data"; + Ptr data = TrainData::loadFromCSV(dataFilename, 0); - if(data.read_csv(dataFilename.c_str()) != 0) + if( data.empty() ) { ts->printf(cvtest::TS::LOG, "File with spambase dataset cann't be read.\n"); ts->set_failed_test_info(cvtest::TS::FAIL_INVALID_TEST_DATA); } - Mat values = cv::cvarrToMat(data.get_values()); - CV_Assert(values.cols == 58); - int responseIndex = 57; - - Mat samples = values.colRange(0, responseIndex); - Mat responses = values.col(responseIndex); + Mat samples = data->getSamples(); + CV_Assert(samples.cols == 57); + Mat responses = data->getResponses(); vector trainSamplesMask(samples.rows, 0); int trainSamplesCount = (int)(0.5f * samples.rows); @@ -616,7 +613,6 @@ protected: std::swap(trainSamplesMask[i1], trainSamplesMask[i2]); } - EM model0(3), model1(3); Mat samples0, samples1; for(int i = 0; i < samples.rows; i++) { @@ -630,8 +626,8 @@ protected: samples1.push_back(sample); } } - model0.train(samples0); - model1.train(samples1); + Ptr model0 = EM::train(samples0, noArray(), noArray(), noArray(), EM::Params(3)); + Ptr model1 = EM::train(samples1, noArray(), noArray(), noArray(), EM::Params(3)); Mat trainConfusionMat(2, 2, CV_32SC1, Scalar(0)), testConfusionMat(2, 2, CV_32SC1, Scalar(0)); @@ -639,8 +635,8 @@ protected: for(int i = 0; i < samples.rows; i++) { Mat sample = samples.row(i); - double sampleLogLikelihoods0 = model0.predict(sample)[0]; - double sampleLogLikelihoods1 = model1.predict(sample)[0]; + double sampleLogLikelihoods0 = model0->predict2(sample, noArray())[0]; + double sampleLogLikelihoods1 = model1->predict2(sample, noArray())[0]; int classID = sampleLogLikelihoods0 >= lambda * sampleLogLikelihoods1 ? 0 : 1; diff --git a/modules/ml/test/test_gbttest.cpp b/modules/ml/test/test_gbttest.cpp index 1e6d0fb20eac57d4f252db83934f88371edc8959..df19489f607f4c10283817dc2dbf062f8eab6f9b 100644 --- a/modules/ml/test/test_gbttest.cpp +++ b/modules/ml/test/test_gbttest.cpp @@ -1,6 +1,8 @@ #include "test_precomp.hpp" +#if 0 + #include #include #include @@ -284,3 +286,5 @@ void CV_GBTreesTest::run(int) ///////////////////////////////////////////////////////////////////////////// TEST(ML_GBTrees, regression) { CV_GBTreesTest test; test.safe_run(); } + +#endif diff --git a/modules/ml/test/test_mltests.cpp b/modules/ml/test/test_mltests.cpp index e04ca98fe27b4225716dec933aa018efbe662e2d..2ffa531ece6ad66be8c2790447bee76416e58009 100644 --- a/modules/ml/test/test_mltests.cpp +++ b/modules/ml/test/test_mltests.cpp @@ -65,7 +65,7 @@ int CV_AMLTest::run_test_case( int testCaseIdx ) for (int k = 0; k < icount; k++) { #endif - data.mix_train_and_test_idx(); + data->shuffleTrainTest(); code = train( testCaseIdx ); #ifdef GET_STAT float case_result = get_error(); @@ -101,9 +101,10 @@ int CV_AMLTest::validate_test_results( int testCaseIdx ) { resultNode["mean"] >> mean; resultNode["sigma"] >> sigma; - float curErr = get_error( testCaseIdx, CV_TEST_ERROR ); + model->save(format("/Users/vp/tmp/dtree/testcase_%02d.cur.yml", testCaseIdx)); + float curErr = get_test_error( testCaseIdx ); const int coeff = 4; - ts->printf( cvtest::TS::LOG, "Test case = %d; test error = %f; mean error = %f (diff=%f), %d*sigma = %f", + ts->printf( cvtest::TS::LOG, "Test case = %d; test error = %f; mean error = %f (diff=%f), %d*sigma = %f\n", testCaseIdx, curErr, mean, abs( curErr - mean), coeff, coeff*sigma ); if ( abs( curErr - mean) > coeff*sigma ) { @@ -125,6 +126,6 @@ int CV_AMLTest::validate_test_results( int testCaseIdx ) TEST(ML_DTree, regression) { CV_AMLTest test( CV_DTREE ); test.safe_run(); } TEST(ML_Boost, regression) { CV_AMLTest test( CV_BOOST ); test.safe_run(); } TEST(ML_RTrees, regression) { CV_AMLTest test( CV_RTREES ); test.safe_run(); } -TEST(ML_ERTrees, regression) { CV_AMLTest test( CV_ERTREES ); test.safe_run(); } +TEST(DISABLED_ML_ERTrees, regression) { CV_AMLTest test( CV_ERTREES ); test.safe_run(); } /* End of file. */ diff --git a/modules/ml/test/test_mltests2.cpp b/modules/ml/test/test_mltests2.cpp index 560c449321d1fd31a884e52a445580e6120671ac..b7c5f46c6ea84ad3ec9ff51338186d444420138d 100644 --- a/modules/ml/test/test_mltests2.cpp +++ b/modules/ml/test/test_mltests2.cpp @@ -44,257 +44,49 @@ using namespace cv; using namespace std; -// auxiliary functions -// 1. nbayes -void nbayes_check_data( CvMLData* _data ) -{ - if( _data->get_missing() ) - CV_Error( CV_StsBadArg, "missing values are not supported" ); - const CvMat* var_types = _data->get_var_types(); - bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; - - Mat _var_types = cvarrToMat(var_types); - if( ( fabs( cvtest::norm( _var_types, Mat::zeros(_var_types.dims, _var_types.size, _var_types.type()), CV_L1 ) - - (var_types->rows + var_types->cols - 2)*CV_VAR_ORDERED - CV_VAR_CATEGORICAL ) > FLT_EPSILON ) || - !is_classifier ) - CV_Error( CV_StsBadArg, "incorrect types of predictors or responses" ); -} -bool nbayes_train( CvNormalBayesClassifier* nbayes, CvMLData* _data ) -{ - nbayes_check_data( _data ); - const CvMat* values = _data->get_values(); - const CvMat* responses = _data->get_responses(); - const CvMat* train_sidx = _data->get_train_sample_idx(); - const CvMat* var_idx = _data->get_var_idx(); - return nbayes->train( values, responses, var_idx, train_sidx ); -} -float nbayes_calc_error( CvNormalBayesClassifier* nbayes, CvMLData* _data, int type, vector *resp ) -{ - float err = 0; - nbayes_check_data( _data ); - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(response->type) ? - 1 : response->step / CV_ELEM_SIZE(response->type); - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; - float* pred_resp = 0; - if( resp && (sample_count > 0) ) - { - resp->resize( sample_count ); - pred_resp = &((*resp)[0]); - } - - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - float r = (float)nbayes->predict( &sample, 0 ); - if( pred_resp ) - pred_resp[i] = r; - int d = fabs((double)r - response->data.fl[si*r_step]) <= FLT_EPSILON ? 0 : 1; - err += d; - } - err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; - return err; -} - -// 2. knearest -void knearest_check_data_and_get_predictors( CvMLData* _data, CvMat* _predictors ) -{ - const CvMat* values = _data->get_values(); - const CvMat* var_idx = _data->get_var_idx(); - if( var_idx->cols + var_idx->rows != values->cols ) - CV_Error( CV_StsBadArg, "var_idx is not supported" ); - if( _data->get_missing() ) - CV_Error( CV_StsBadArg, "missing values are not supported" ); - int resp_idx = _data->get_response_idx(); - if( resp_idx == 0) - cvGetCols( values, _predictors, 1, values->cols ); - else if( resp_idx == values->cols - 1 ) - cvGetCols( values, _predictors, 0, values->cols - 1 ); - else - CV_Error( CV_StsBadArg, "responses must be in the first or last column; other cases are not supported" ); -} -bool knearest_train( CvKNearest* knearest, CvMLData* _data ) -{ - const CvMat* responses = _data->get_responses(); - const CvMat* train_sidx = _data->get_train_sample_idx(); - bool is_regression = _data->get_var_type( _data->get_response_idx() ) == CV_VAR_ORDERED; - CvMat predictors; - knearest_check_data_and_get_predictors( _data, &predictors ); - return knearest->train( &predictors, responses, train_sidx, is_regression ); -} -float knearest_calc_error( CvKNearest* knearest, CvMLData* _data, int k, int type, vector *resp ) -{ - float err = 0; - const CvMat* response = _data->get_responses(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(response->type) ? - 1 : response->step / CV_ELEM_SIZE(response->type); - bool is_regression = _data->get_var_type( _data->get_response_idx() ) == CV_VAR_ORDERED; - CvMat predictors; - knearest_check_data_and_get_predictors( _data, &predictors ); - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? predictors.rows : sample_count; - float* pred_resp = 0; - if( resp && (sample_count > 0) ) - { - resp->resize( sample_count ); - pred_resp = &((*resp)[0]); - } - if ( !is_regression ) - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample; - int si = sidx ? sidx[i] : i; - cvGetRow( &predictors, &sample, si ); - float r = knearest->find_nearest( &sample, k ); - if( pred_resp ) - pred_resp[i] = r; - int d = fabs((double)r - response->data.fl[si*r_step]) <= FLT_EPSILON ? 0 : 1; - err += d; - } - err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; - } - else - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample; - int si = sidx ? sidx[i] : i; - cvGetRow( &predictors, &sample, si ); - float r = knearest->find_nearest( &sample, k ); - if( pred_resp ) - pred_resp[i] = r; - float d = r - response->data.fl[si*r_step]; - err += d*d; - } - err = sample_count ? err / (float)sample_count : -FLT_MAX; - } - return err; -} - -// 3. svm int str_to_svm_type(String& str) { if( !str.compare("C_SVC") ) - return CvSVM::C_SVC; + return SVM::C_SVC; if( !str.compare("NU_SVC") ) - return CvSVM::NU_SVC; + return SVM::NU_SVC; if( !str.compare("ONE_CLASS") ) - return CvSVM::ONE_CLASS; + return SVM::ONE_CLASS; if( !str.compare("EPS_SVR") ) - return CvSVM::EPS_SVR; + return SVM::EPS_SVR; if( !str.compare("NU_SVR") ) - return CvSVM::NU_SVR; + return SVM::NU_SVR; CV_Error( CV_StsBadArg, "incorrect svm type string" ); return -1; } int str_to_svm_kernel_type( String& str ) { if( !str.compare("LINEAR") ) - return CvSVM::LINEAR; + return SVM::LINEAR; if( !str.compare("POLY") ) - return CvSVM::POLY; + return SVM::POLY; if( !str.compare("RBF") ) - return CvSVM::RBF; + return SVM::RBF; if( !str.compare("SIGMOID") ) - return CvSVM::SIGMOID; + return SVM::SIGMOID; CV_Error( CV_StsBadArg, "incorrect svm type string" ); return -1; } -void svm_check_data( CvMLData* _data ) -{ - if( _data->get_missing() ) - CV_Error( CV_StsBadArg, "missing values are not supported" ); - const CvMat* var_types = _data->get_var_types(); - for( int i = 0; i < var_types->cols-1; i++ ) - if (var_types->data.ptr[i] == CV_VAR_CATEGORICAL) - { - char msg[50]; - sprintf( msg, "incorrect type of %d-predictor", i ); - CV_Error( CV_StsBadArg, msg ); - } -} -bool svm_train( CvSVM* svm, CvMLData* _data, CvSVMParams _params ) -{ - svm_check_data(_data); - const CvMat* _train_data = _data->get_values(); - const CvMat* _responses = _data->get_responses(); - const CvMat* _var_idx = _data->get_var_idx(); - const CvMat* _sample_idx = _data->get_train_sample_idx(); - return svm->train( _train_data, _responses, _var_idx, _sample_idx, _params ); -} -bool svm_train_auto( CvSVM* svm, CvMLData* _data, CvSVMParams _params, - int k_fold, CvParamGrid C_grid, CvParamGrid gamma_grid, - CvParamGrid p_grid, CvParamGrid nu_grid, CvParamGrid coef_grid, - CvParamGrid degree_grid ) -{ - svm_check_data(_data); - const CvMat* _train_data = _data->get_values(); - const CvMat* _responses = _data->get_responses(); - const CvMat* _var_idx = _data->get_var_idx(); - const CvMat* _sample_idx = _data->get_train_sample_idx(); - return svm->train_auto( _train_data, _responses, _var_idx, - _sample_idx, _params, k_fold, C_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid ); -} -float svm_calc_error( CvSVM* svm, CvMLData* _data, int type, vector *resp ) + +Ptr svm_train_auto( Ptr _data, SVM::Params _params, + int k_fold, ParamGrid C_grid, ParamGrid gamma_grid, + ParamGrid p_grid, ParamGrid nu_grid, ParamGrid coef_grid, + ParamGrid degree_grid ) { - svm_check_data(_data); - float err = 0; - const CvMat* values = _data->get_values(); - const CvMat* response = _data->get_responses(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - const CvMat* var_types = _data->get_var_types(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(response->type) ? - 1 : response->step / CV_ELEM_SIZE(response->type); - bool is_classifier = var_types->data.ptr[var_types->cols-1] == CV_VAR_CATEGORICAL; - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? values->rows : sample_count; - float* pred_resp = 0; - if( resp && (sample_count > 0) ) - { - resp->resize( sample_count ); - pred_resp = &((*resp)[0]); - } - if ( is_classifier ) - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - float r = svm->predict( &sample ); - if( pred_resp ) - pred_resp[i] = r; - int d = fabs((double)r - response->data.fl[si*r_step]) <= FLT_EPSILON ? 0 : 1; - err += d; - } - err = sample_count ? err / (float)sample_count * 100 : -FLT_MAX; - } - else - { - for( int i = 0; i < sample_count; i++ ) - { - CvMat sample; - int si = sidx ? sidx[i] : i; - cvGetRow( values, &sample, si ); - float r = svm->predict( &sample ); - if( pred_resp ) - pred_resp[i] = r; - float d = r - response->data.fl[si*r_step]; - err += d*d; - } - err = sample_count ? err / (float)sample_count : -FLT_MAX; - } - return err; + Mat _train_data = _data->getSamples(); + Mat _responses = _data->getResponses(); + Mat _var_idx = _data->getVarIdx(); + Mat _sample_idx = _data->getTrainSampleIdx(); + + Ptr svm = SVM::create(_params); + if( svm->trainAuto( _data, k_fold, C_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid ) ) + return svm; + return Ptr(); } // 4. em @@ -302,79 +94,66 @@ float svm_calc_error( CvSVM* svm, CvMLData* _data, int type, vector *resp int str_to_ann_train_method( String& str ) { if( !str.compare("BACKPROP") ) - return CvANN_MLP_TrainParams::BACKPROP; + return ANN_MLP::Params::BACKPROP; if( !str.compare("RPROP") ) - return CvANN_MLP_TrainParams::RPROP; + return ANN_MLP::Params::RPROP; CV_Error( CV_StsBadArg, "incorrect ann train method string" ); return -1; } -void ann_check_data_and_get_predictors( CvMLData* _data, CvMat* _inputs ) + +void ann_check_data( Ptr _data ) { - const CvMat* values = _data->get_values(); - const CvMat* var_idx = _data->get_var_idx(); - if( var_idx->cols + var_idx->rows != values->cols ) + Mat values = _data->getSamples(); + Mat var_idx = _data->getVarIdx(); + int nvars = (int)var_idx.total(); + if( nvars != 0 && nvars != values.cols ) CV_Error( CV_StsBadArg, "var_idx is not supported" ); - if( _data->get_missing() ) + if( !_data->getMissing().empty() ) CV_Error( CV_StsBadArg, "missing values are not supported" ); - int resp_idx = _data->get_response_idx(); - if( resp_idx == 0) - cvGetCols( values, _inputs, 1, values->cols ); - else if( resp_idx == values->cols - 1 ) - cvGetCols( values, _inputs, 0, values->cols - 1 ); - else - CV_Error( CV_StsBadArg, "outputs must be in the first or last column; other cases are not supported" ); } -void ann_get_new_responses( CvMLData* _data, Mat& new_responses, map& cls_map ) + +// unroll the categorical responses to binary vectors +Mat ann_get_new_responses( Ptr _data, map& cls_map ) { - const CvMat* train_sidx = _data->get_train_sample_idx(); - int* train_sidx_ptr = train_sidx->data.i; - const CvMat* responses = _data->get_responses(); - float* responses_ptr = responses->data.fl; - int r_step = CV_IS_MAT_CONT(responses->type) ? - 1 : responses->step / CV_ELEM_SIZE(responses->type); + Mat train_sidx = _data->getTrainSampleIdx(); + int* train_sidx_ptr = train_sidx.ptr(); + Mat responses = _data->getResponses(); int cls_count = 0; // construct cls_map cls_map.clear(); - for( int si = 0; si < train_sidx->cols; si++ ) + int nresponses = (int)responses.total(); + int si, n = !train_sidx.empty() ? (int)train_sidx.total() : nresponses; + + for( si = 0; si < n; si++ ) { - int sidx = train_sidx_ptr[si]; - int r = cvRound(responses_ptr[sidx*r_step]); - CV_DbgAssert( fabs(responses_ptr[sidx*r_step]-r) < FLT_EPSILON ); - int cls_map_size = (int)cls_map.size(); - cls_map[r]; - if ( (int)cls_map.size() > cls_map_size ) + int sidx = train_sidx_ptr ? train_sidx_ptr[si] : si; + int r = cvRound(responses.at(sidx)); + CV_DbgAssert( fabs(responses.at(sidx) - r) < FLT_EPSILON ); + map::iterator it = cls_map.find(r); + if( it == cls_map.end() ) cls_map[r] = cls_count++; } - new_responses.create( responses->rows, cls_count, CV_32F ); - new_responses.setTo( 0 ); - for( int si = 0; si < train_sidx->cols; si++ ) + Mat new_responses = Mat::zeros( nresponses, cls_count, CV_32F ); + for( si = 0; si < n; si++ ) { - int sidx = train_sidx_ptr[si]; - int r = cvRound(responses_ptr[sidx*r_step]); + int sidx = train_sidx_ptr ? train_sidx_ptr[si] : si; + int r = cvRound(responses.at(sidx)); int cidx = cls_map[r]; - new_responses.ptr(sidx)[cidx] = 1; + new_responses.at(sidx, cidx) = 1.f; } + return new_responses; } -int ann_train( CvANN_MLP* ann, CvMLData* _data, Mat& new_responses, CvANN_MLP_TrainParams _params, int flags = 0 ) -{ - const CvMat* train_sidx = _data->get_train_sample_idx(); - CvMat predictors; - ann_check_data_and_get_predictors( _data, &predictors ); - CvMat _new_responses = CvMat( new_responses ); - return ann->train( &predictors, &_new_responses, 0, train_sidx, _params, flags ); -} -float ann_calc_error( CvANN_MLP* ann, CvMLData* _data, map& cls_map, int type , vector *resp_labels ) + +float ann_calc_error( Ptr ann, Ptr _data, map& cls_map, int type, vector *resp_labels ) { float err = 0; - const CvMat* responses = _data->get_responses(); - const CvMat* sample_idx = (type == CV_TEST_ERROR) ? _data->get_test_sample_idx() : _data->get_train_sample_idx(); - int* sidx = sample_idx ? sample_idx->data.i : 0; - int r_step = CV_IS_MAT_CONT(responses->type) ? - 1 : responses->step / CV_ELEM_SIZE(responses->type); - CvMat predictors; - ann_check_data_and_get_predictors( _data, &predictors ); - int sample_count = sample_idx ? sample_idx->cols : 0; - sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? predictors.rows : sample_count; + Mat samples = _data->getSamples(); + Mat responses = _data->getResponses(); + Mat sample_idx = (type == CV_TEST_ERROR) ? _data->getTestSampleIdx() : _data->getTrainSampleIdx(); + int* sidx = !sample_idx.empty() ? sample_idx.ptr() : 0; + ann_check_data( _data ); + int sample_count = (int)sample_idx.total(); + sample_count = (type == CV_TRAIN_ERROR && sample_count == 0) ? samples.rows : sample_count; float* pred_resp = 0; vector innresp; if( sample_count > 0 ) @@ -392,17 +171,16 @@ float ann_calc_error( CvANN_MLP* ann, CvMLData* _data, map& cls_map, i } int cls_count = (int)cls_map.size(); Mat output( 1, cls_count, CV_32FC1 ); - CvMat _output = CvMat(output); + for( int i = 0; i < sample_count; i++ ) { - CvMat sample; int si = sidx ? sidx[i] : i; - cvGetRow( &predictors, &sample, si ); - ann->predict( &sample, &_output ); - CvPoint best_cls; - cvMinMaxLoc( &_output, 0, 0, 0, &best_cls, 0 ); - int r = cvRound(responses->data.fl[si*r_step]); - CV_DbgAssert( fabs(responses->data.fl[si*r_step]-r) < FLT_EPSILON ); + Mat sample = samples.row(si); + ann->predict( sample, output ); + Point best_cls; + minMaxLoc(output, 0, 0, 0, &best_cls, 0); + int r = cvRound(responses.at(si)); + CV_DbgAssert( fabs(responses.at(si) - r) < FLT_EPSILON ); r = cls_map[r]; int d = best_cls.x == r ? 0 : 1; err += d; @@ -417,13 +195,13 @@ float ann_calc_error( CvANN_MLP* ann, CvMLData* _data, map& cls_map, i int str_to_boost_type( String& str ) { if ( !str.compare("DISCRETE") ) - return CvBoost::DISCRETE; + return Boost::DISCRETE; if ( !str.compare("REAL") ) - return CvBoost::REAL; + return Boost::REAL; if ( !str.compare("LOGIT") ) - return CvBoost::LOGIT; + return Boost::LOGIT; if ( !str.compare("GENTLE") ) - return CvBoost::GENTLE; + return Boost::GENTLE; CV_Error( CV_StsBadArg, "incorrect boost type string" ); return -1; } @@ -446,76 +224,37 @@ CV_MLBaseTest::CV_MLBaseTest(const char* _modelName) RNG& rng = theRNG(); initSeed = rng.state; - rng.state = seeds[rng(seedCount)]; modelName = _modelName; - nbayes = 0; - knearest = 0; - svm = 0; - ann = 0; - dtree = 0; - boost = 0; - rtrees = 0; - ertrees = 0; - if( !modelName.compare(CV_NBAYES) ) - nbayes = new CvNormalBayesClassifier; - else if( !modelName.compare(CV_KNEAREST) ) - knearest = new CvKNearest; - else if( !modelName.compare(CV_SVM) ) - svm = new CvSVM; - else if( !modelName.compare(CV_ANN) ) - ann = new CvANN_MLP; - else if( !modelName.compare(CV_DTREE) ) - dtree = new CvDTree; - else if( !modelName.compare(CV_BOOST) ) - boost = new CvBoost; - else if( !modelName.compare(CV_RTREES) ) - rtrees = new CvRTrees; - else if( !modelName.compare(CV_ERTREES) ) - ertrees = new CvERTrees; } CV_MLBaseTest::~CV_MLBaseTest() { if( validationFS.isOpened() ) validationFS.release(); - if( nbayes ) - delete nbayes; - if( knearest ) - delete knearest; - if( svm ) - delete svm; - if( ann ) - delete ann; - if( dtree ) - delete dtree; - if( boost ) - delete boost; - if( rtrees ) - delete rtrees; - if( ertrees ) - delete ertrees; theRNG().state = initSeed; } -int CV_MLBaseTest::read_params( CvFileStorage* _fs ) +int CV_MLBaseTest::read_params( CvFileStorage* __fs ) { - if( !_fs ) + FileStorage _fs(__fs, false); + if( !_fs.isOpened() ) test_case_count = -1; else { - CvFileNode* fn = cvGetRootFileNode( _fs, 0 ); - fn = (CvFileNode*)cvGetSeqElem( fn->data.seq, 0 ); - fn = cvGetFileNodeByName( _fs, fn, "run_params" ); - CvSeq* dataSetNamesSeq = cvGetFileNodeByName( _fs, fn, modelName.c_str() )->data.seq; - test_case_count = dataSetNamesSeq ? dataSetNamesSeq->total : -1; + FileNode fn = _fs.getFirstTopLevelNode()["run_params"][modelName]; + test_case_count = (int)fn.size(); + if( test_case_count <= 0 ) + test_case_count = -1; if( test_case_count > 0 ) { dataSetNames.resize( test_case_count ); - vector::iterator it = dataSetNames.begin(); - for( int i = 0; i < test_case_count; i++, it++ ) - *it = ((CvFileNode*)cvGetSeqElem( dataSetNamesSeq, i ))->data.str.ptr; + FileNodeIterator it = fn.begin(); + for( int i = 0; i < test_case_count; i++, ++it ) + { + dataSetNames[i] = (string)*it; + } } } return cvtest::TS::OK;; @@ -547,8 +286,6 @@ void CV_MLBaseTest::run( int ) int CV_MLBaseTest::prepare_test_case( int test_case_idx ) { - int trainSampleCount, respIdx; - String varTypes; clear(); string dataPath = ts->get_data_path(); @@ -560,30 +297,27 @@ int CV_MLBaseTest::prepare_test_case( int test_case_idx ) string dataName = dataSetNames[test_case_idx], filename = dataPath + dataName + ".data"; - if ( data.read_csv( filename.c_str() ) != 0) - { - char msg[100]; - sprintf( msg, "file %s can not be read", filename.c_str() ); - ts->printf( cvtest::TS::LOG, msg ); - return cvtest::TS::FAIL_INVALID_TEST_DATA; - } FileNode dataParamsNode = validationFS.getFirstTopLevelNode()["validation"][modelName][dataName]["data_params"]; CV_DbgAssert( !dataParamsNode.empty() ); CV_DbgAssert( !dataParamsNode["LS"].empty() ); - dataParamsNode["LS"] >> trainSampleCount; - CvTrainTestSplit spl( trainSampleCount ); - data.set_train_test_split( &spl ); + int trainSampleCount = (int)dataParamsNode["LS"]; CV_DbgAssert( !dataParamsNode["resp_idx"].empty() ); - dataParamsNode["resp_idx"] >> respIdx; - data.set_response_idx( respIdx ); + int respIdx = (int)dataParamsNode["resp_idx"]; CV_DbgAssert( !dataParamsNode["types"].empty() ); - dataParamsNode["types"] >> varTypes; - data.set_var_types( varTypes.c_str() ); + String varTypes = (String)dataParamsNode["types"]; + data = TrainData::loadFromCSV(filename, 0, respIdx, respIdx+1, varTypes); + if( data.empty() ) + { + ts->printf( cvtest::TS::LOG, "file %s can not be read\n", filename.c_str() ); + return cvtest::TS::FAIL_INVALID_TEST_DATA; + } + + data->setTrainTestSplit(trainSampleCount); return cvtest::TS::OK; } @@ -598,114 +332,98 @@ int CV_MLBaseTest::train( int testCaseIdx ) FileNode modelParamsNode = validationFS.getFirstTopLevelNode()["validation"][modelName][dataSetNames[testCaseIdx]]["model_params"]; - if( !modelName.compare(CV_NBAYES) ) - is_trained = nbayes_train( nbayes, &data ); - else if( !modelName.compare(CV_KNEAREST) ) + if( modelName == CV_NBAYES ) + model = NormalBayesClassifier::create(); + else if( modelName == CV_KNEAREST ) { - assert( 0 ); - //is_trained = knearest->train( &data ); + model = KNearest::create(); } - else if( !modelName.compare(CV_SVM) ) + else if( modelName == CV_SVM ) { String svm_type_str, kernel_type_str; modelParamsNode["svm_type"] >> svm_type_str; modelParamsNode["kernel_type"] >> kernel_type_str; - CvSVMParams params; - params.svm_type = str_to_svm_type( svm_type_str ); - params.kernel_type = str_to_svm_kernel_type( kernel_type_str ); + SVM::Params params; + params.svmType = str_to_svm_type( svm_type_str ); + params.kernelType = str_to_svm_kernel_type( kernel_type_str ); modelParamsNode["degree"] >> params.degree; modelParamsNode["gamma"] >> params.gamma; modelParamsNode["coef0"] >> params.coef0; modelParamsNode["C"] >> params.C; modelParamsNode["nu"] >> params.nu; modelParamsNode["p"] >> params.p; - is_trained = svm_train( svm, &data, params ); + model = SVM::create(params); } - else if( !modelName.compare(CV_EM) ) + else if( modelName == CV_EM ) { assert( 0 ); } - else if( !modelName.compare(CV_ANN) ) + else if( modelName == CV_ANN ) { String train_method_str; double param1, param2; modelParamsNode["train_method"] >> train_method_str; modelParamsNode["param1"] >> param1; modelParamsNode["param2"] >> param2; - Mat new_responses; - ann_get_new_responses( &data, new_responses, cls_map ); - int layer_sz[] = { data.get_values()->cols - 1, 100, 100, (int)cls_map.size() }; - CvMat layer_sizes = - cvMat( 1, (int)(sizeof(layer_sz)/sizeof(layer_sz[0])), CV_32S, layer_sz ); - ann->create( &layer_sizes ); - is_trained = ann_train( ann, &data, new_responses, CvANN_MLP_TrainParams(cvTermCriteria(CV_TERMCRIT_ITER,300,0.01), - str_to_ann_train_method(train_method_str), param1, param2) ) >= 0; + Mat new_responses = ann_get_new_responses( data, cls_map ); + // binarize the responses + data = TrainData::create(data->getSamples(), data->getLayout(), new_responses, + data->getVarIdx(), data->getTrainSampleIdx()); + int layer_sz[] = { data->getNAllVars(), 100, 100, (int)cls_map.size() }; + Mat layer_sizes( 1, (int)(sizeof(layer_sz)/sizeof(layer_sz[0])), CV_32S, layer_sz ); + model = ANN_MLP::create(ANN_MLP::Params(layer_sizes, ANN_MLP::SIGMOID_SYM, 0, 0, + TermCriteria(TermCriteria::COUNT,300,0.01), + str_to_ann_train_method(train_method_str), param1, param2)); } - else if( !modelName.compare(CV_DTREE) ) + else if( modelName == CV_DTREE ) { int MAX_DEPTH, MIN_SAMPLE_COUNT, MAX_CATEGORIES, CV_FOLDS; float REG_ACCURACY = 0; - bool USE_SURROGATE, IS_PRUNED; + bool USE_SURROGATE = false, IS_PRUNED; modelParamsNode["max_depth"] >> MAX_DEPTH; modelParamsNode["min_sample_count"] >> MIN_SAMPLE_COUNT; - modelParamsNode["use_surrogate"] >> USE_SURROGATE; + //modelParamsNode["use_surrogate"] >> USE_SURROGATE; modelParamsNode["max_categories"] >> MAX_CATEGORIES; modelParamsNode["cv_folds"] >> CV_FOLDS; modelParamsNode["is_pruned"] >> IS_PRUNED; - is_trained = dtree->train( &data, - CvDTreeParams(MAX_DEPTH, MIN_SAMPLE_COUNT, REG_ACCURACY, USE_SURROGATE, - MAX_CATEGORIES, CV_FOLDS, false, IS_PRUNED, 0 )) != 0; + model = DTrees::create(DTrees::Params(MAX_DEPTH, MIN_SAMPLE_COUNT, REG_ACCURACY, USE_SURROGATE, + MAX_CATEGORIES, CV_FOLDS, false, IS_PRUNED, Mat() )); } - else if( !modelName.compare(CV_BOOST) ) + else if( modelName == CV_BOOST ) { int BOOST_TYPE, WEAK_COUNT, MAX_DEPTH; float WEIGHT_TRIM_RATE; - bool USE_SURROGATE; + bool USE_SURROGATE = false; String typeStr; modelParamsNode["type"] >> typeStr; BOOST_TYPE = str_to_boost_type( typeStr ); modelParamsNode["weak_count"] >> WEAK_COUNT; modelParamsNode["weight_trim_rate"] >> WEIGHT_TRIM_RATE; modelParamsNode["max_depth"] >> MAX_DEPTH; - modelParamsNode["use_surrogate"] >> USE_SURROGATE; - is_trained = boost->train( &data, - CvBoostParams(BOOST_TYPE, WEAK_COUNT, WEIGHT_TRIM_RATE, MAX_DEPTH, USE_SURROGATE, 0) ) != 0; + //modelParamsNode["use_surrogate"] >> USE_SURROGATE; + model = Boost::create( Boost::Params(BOOST_TYPE, WEAK_COUNT, WEIGHT_TRIM_RATE, MAX_DEPTH, USE_SURROGATE, Mat()) ); } - else if( !modelName.compare(CV_RTREES) ) + else if( modelName == CV_RTREES ) { int MAX_DEPTH, MIN_SAMPLE_COUNT, MAX_CATEGORIES, CV_FOLDS, NACTIVE_VARS, MAX_TREES_NUM; float REG_ACCURACY = 0, OOB_EPS = 0.0; - bool USE_SURROGATE, IS_PRUNED; + bool USE_SURROGATE = false, IS_PRUNED; modelParamsNode["max_depth"] >> MAX_DEPTH; modelParamsNode["min_sample_count"] >> MIN_SAMPLE_COUNT; - modelParamsNode["use_surrogate"] >> USE_SURROGATE; + //modelParamsNode["use_surrogate"] >> USE_SURROGATE; modelParamsNode["max_categories"] >> MAX_CATEGORIES; modelParamsNode["cv_folds"] >> CV_FOLDS; modelParamsNode["is_pruned"] >> IS_PRUNED; modelParamsNode["nactive_vars"] >> NACTIVE_VARS; modelParamsNode["max_trees_num"] >> MAX_TREES_NUM; - is_trained = rtrees->train( &data, CvRTParams( MAX_DEPTH, MIN_SAMPLE_COUNT, REG_ACCURACY, - USE_SURROGATE, MAX_CATEGORIES, 0, true, // (calc_var_importance == true) <=> RF processes variable importance - NACTIVE_VARS, MAX_TREES_NUM, OOB_EPS, CV_TERMCRIT_ITER)) != 0; - } - else if( !modelName.compare(CV_ERTREES) ) - { - int MAX_DEPTH, MIN_SAMPLE_COUNT, MAX_CATEGORIES, CV_FOLDS, NACTIVE_VARS, MAX_TREES_NUM; - float REG_ACCURACY = 0, OOB_EPS = 0.0; - bool USE_SURROGATE, IS_PRUNED; - modelParamsNode["max_depth"] >> MAX_DEPTH; - modelParamsNode["min_sample_count"] >> MIN_SAMPLE_COUNT; - modelParamsNode["use_surrogate"] >> USE_SURROGATE; - modelParamsNode["max_categories"] >> MAX_CATEGORIES; - modelParamsNode["cv_folds"] >> CV_FOLDS; - modelParamsNode["is_pruned"] >> IS_PRUNED; - modelParamsNode["nactive_vars"] >> NACTIVE_VARS; - modelParamsNode["max_trees_num"] >> MAX_TREES_NUM; - is_trained = ertrees->train( &data, CvRTParams( MAX_DEPTH, MIN_SAMPLE_COUNT, REG_ACCURACY, - USE_SURROGATE, MAX_CATEGORIES, 0, false, // (calc_var_importance == true) <=> RF processes variable importance - NACTIVE_VARS, MAX_TREES_NUM, OOB_EPS, CV_TERMCRIT_ITER)) != 0; + model = RTrees::create(RTrees::Params( MAX_DEPTH, MIN_SAMPLE_COUNT, REG_ACCURACY, + USE_SURROGATE, MAX_CATEGORIES, Mat(), true, // (calc_var_importance == true) <=> RF processes variable importance + NACTIVE_VARS, TermCriteria(TermCriteria::COUNT, MAX_TREES_NUM, OOB_EPS))); } + if( !model.empty() ) + is_trained = model->train(data, 0); + if( !is_trained ) { ts->printf( cvtest::TS::LOG, "in test case %d model training was failed", testCaseIdx ); @@ -714,78 +432,46 @@ int CV_MLBaseTest::train( int testCaseIdx ) return cvtest::TS::OK; } -float CV_MLBaseTest::get_error( int /*testCaseIdx*/, int type, vector *resp ) +float CV_MLBaseTest::get_test_error( int /*testCaseIdx*/, vector *resp ) { + int type = CV_TEST_ERROR; float err = 0; - if( !modelName.compare(CV_NBAYES) ) - err = nbayes_calc_error( nbayes, &data, type, resp ); - else if( !modelName.compare(CV_KNEAREST) ) - { - assert( 0 ); - /*testCaseIdx = 0; - int k = 2; - validationFS.getFirstTopLevelNode()["validation"][modelName][dataSetNames[testCaseIdx]]["model_params"]["k"] >> k; - err = knearest->calc_error( &data, k, type, resp );*/ - } - else if( !modelName.compare(CV_SVM) ) - err = svm_calc_error( svm, &data, type, resp ); - else if( !modelName.compare(CV_EM) ) + Mat _resp; + if( modelName == CV_EM ) assert( 0 ); - else if( !modelName.compare(CV_ANN) ) - err = ann_calc_error( ann, &data, cls_map, type, resp ); - else if( !modelName.compare(CV_DTREE) ) - err = dtree->calc_error( &data, type, resp ); - else if( !modelName.compare(CV_BOOST) ) - err = boost->calc_error( &data, type, resp ); - else if( !modelName.compare(CV_RTREES) ) - err = rtrees->calc_error( &data, type, resp ); - else if( !modelName.compare(CV_ERTREES) ) - err = ertrees->calc_error( &data, type, resp ); + else if( modelName == CV_ANN ) + err = ann_calc_error( model, data, cls_map, type, resp ); + else if( modelName == CV_DTREE || modelName == CV_BOOST || modelName == CV_RTREES || + modelName == CV_SVM || modelName == CV_NBAYES || modelName == CV_KNEAREST ) + err = model->calcError( data, true, _resp ); + if( !_resp.empty() && resp ) + _resp.convertTo(*resp, CV_32F); return err; } void CV_MLBaseTest::save( const char* filename ) { - if( !modelName.compare(CV_NBAYES) ) - nbayes->save( filename ); - else if( !modelName.compare(CV_KNEAREST) ) - knearest->save( filename ); - else if( !modelName.compare(CV_SVM) ) - svm->save( filename ); - else if( !modelName.compare(CV_ANN) ) - ann->save( filename ); - else if( !modelName.compare(CV_DTREE) ) - dtree->save( filename ); - else if( !modelName.compare(CV_BOOST) ) - boost->save( filename ); - else if( !modelName.compare(CV_RTREES) ) - rtrees->save( filename ); - else if( !modelName.compare(CV_ERTREES) ) - ertrees->save( filename ); + model->save( filename ); } void CV_MLBaseTest::load( const char* filename ) { - if( !modelName.compare(CV_NBAYES) ) - nbayes->load( filename ); - else if( !modelName.compare(CV_KNEAREST) ) - knearest->load( filename ); - else if( !modelName.compare(CV_SVM) ) - { - delete svm; - svm = new CvSVM; - svm->load( filename ); - } - else if( !modelName.compare(CV_ANN) ) - ann->load( filename ); - else if( !modelName.compare(CV_DTREE) ) - dtree->load( filename ); - else if( !modelName.compare(CV_BOOST) ) - boost->load( filename ); - else if( !modelName.compare(CV_RTREES) ) - rtrees->load( filename ); - else if( !modelName.compare(CV_ERTREES) ) - ertrees->load( filename ); + if( modelName == CV_NBAYES ) + model = StatModel::load( filename ); + else if( modelName == CV_KNEAREST ) + model = StatModel::load( filename ); + else if( modelName == CV_SVM ) + model = StatModel::load( filename ); + else if( modelName == CV_ANN ) + model = StatModel::load( filename ); + else if( modelName == CV_DTREE ) + model = StatModel::load( filename ); + else if( modelName == CV_BOOST ) + model = StatModel::load( filename ); + else if( modelName == CV_RTREES ) + model = StatModel::load( filename ); + else + CV_Error( CV_StsNotImplemented, "invalid stat model name"); } /* End of file. */ diff --git a/modules/ml/test/test_precomp.hpp b/modules/ml/test/test_precomp.hpp index e68e5513ba7bf99897164cb3a918c84ceeec06c2..329b9bd6c0b8c582adc363021568c72c05eab0bd 100644 --- a/modules/ml/test/test_precomp.hpp +++ b/modules/ml/test/test_precomp.hpp @@ -25,6 +25,20 @@ #define CV_RTREES "rtrees" #define CV_ERTREES "ertrees" +enum { CV_TRAIN_ERROR=0, CV_TEST_ERROR=1 }; + +using cv::Ptr; +using cv::ml::StatModel; +using cv::ml::TrainData; +using cv::ml::NormalBayesClassifier; +using cv::ml::SVM; +using cv::ml::KNearest; +using cv::ml::ParamGrid; +using cv::ml::ANN_MLP; +using cv::ml::DTrees; +using cv::ml::Boost; +using cv::ml::RTrees; + class CV_MLBaseTest : public cvtest::BaseTest { public: @@ -39,24 +53,16 @@ protected: virtual int validate_test_results( int testCaseIdx ) = 0; int train( int testCaseIdx ); - float get_error( int testCaseIdx, int type, std::vector *resp = 0 ); + float get_test_error( int testCaseIdx, std::vector *resp = 0 ); void save( const char* filename ); void load( const char* filename ); - CvMLData data; + Ptr data; std::string modelName, validationFN; std::vector dataSetNames; cv::FileStorage validationFS; - // MLL models - CvNormalBayesClassifier* nbayes; - CvKNearest* knearest; - CvSVM* svm; - CvANN_MLP* ann; - CvDTree* dtree; - CvBoost* boost; - CvRTrees* rtrees; - CvERTrees* ertrees; + Ptr model; std::map cls_map; @@ -67,6 +73,7 @@ class CV_AMLTest : public CV_MLBaseTest { public: CV_AMLTest( const char* _modelName ); + virtual ~CV_AMLTest() {} protected: virtual int run_test_case( int testCaseIdx ); virtual int validate_test_results( int testCaseIdx ); @@ -76,6 +83,7 @@ class CV_SLMLTest : public CV_MLBaseTest { public: CV_SLMLTest( const char* _modelName ); + virtual ~CV_SLMLTest() {} protected: virtual int run_test_case( int testCaseIdx ); virtual int validate_test_results( int testCaseIdx ); diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index 8b58ce534a24a2dd1c02e2aaaeef2d353805cf62..bef2fd0e1c6a1fbc02faed7e5bb43fc20ca689ea 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -59,20 +59,20 @@ int CV_SLMLTest::run_test_case( int testCaseIdx ) if( code == cvtest::TS::OK ) { - data.mix_train_and_test_idx(); - code = train( testCaseIdx ); - if( code == cvtest::TS::OK ) - { - get_error( testCaseIdx, CV_TEST_ERROR, &test_resps1 ); - fname1 = tempfile(".yml.gz"); - save( fname1.c_str() ); - load( fname1.c_str() ); - get_error( testCaseIdx, CV_TEST_ERROR, &test_resps2 ); - fname2 = tempfile(".yml.gz"); - save( fname2.c_str() ); - } - else - ts->printf( cvtest::TS::LOG, "model can not be trained" ); + data->setTrainTestSplit(data->getNTrainSamples(), true); + code = train( testCaseIdx ); + if( code == cvtest::TS::OK ) + { + get_test_error( testCaseIdx, &test_resps1 ); + fname1 = tempfile(".yml.gz"); + save( fname1.c_str() ); + load( fname1.c_str() ); + get_test_error( testCaseIdx, &test_resps2 ); + fname2 = tempfile(".yml.gz"); + save( fname2.c_str() ); + } + else + ts->printf( cvtest::TS::LOG, "model can not be trained" ); } return code; } @@ -130,15 +130,19 @@ int CV_SLMLTest::validate_test_results( int testCaseIdx ) remove( fname2.c_str() ); } - // 2. compare responses - CV_Assert( test_resps1.size() == test_resps2.size() ); - vector::const_iterator it1 = test_resps1.begin(), it2 = test_resps2.begin(); - for( ; it1 != test_resps1.end(); ++it1, ++it2 ) + if( code >= 0 ) { - if( fabs(*it1 - *it2) > FLT_EPSILON ) + // 2. compare responses + CV_Assert( test_resps1.size() == test_resps2.size() ); + vector::const_iterator it1 = test_resps1.begin(), it2 = test_resps2.begin(); + for( ; it1 != test_resps1.end(); ++it1, ++it2 ) { - ts->printf( cvtest::TS::LOG, "in test case %d responses predicted before saving and after loading is different", testCaseIdx ); - code = cvtest::TS::FAIL_INVALID_OUTPUT; + if( fabs(*it1 - *it2) > FLT_EPSILON ) + { + ts->printf( cvtest::TS::LOG, "in test case %d responses predicted before saving and after loading is different", testCaseIdx ); + code = cvtest::TS::FAIL_INVALID_OUTPUT; + break; + } } } return code; @@ -152,40 +156,41 @@ TEST(ML_ANN, save_load) { CV_SLMLTest test( CV_ANN ); test.safe_run(); } TEST(ML_DTree, save_load) { CV_SLMLTest test( CV_DTREE ); test.safe_run(); } TEST(ML_Boost, save_load) { CV_SLMLTest test( CV_BOOST ); test.safe_run(); } TEST(ML_RTrees, save_load) { CV_SLMLTest test( CV_RTREES ); test.safe_run(); } -TEST(ML_ERTrees, save_load) { CV_SLMLTest test( CV_ERTREES ); test.safe_run(); } +TEST(DISABLED_ML_ERTrees, save_load) { CV_SLMLTest test( CV_ERTREES ); test.safe_run(); } -TEST(ML_SVM, throw_exception_when_save_untrained_model) +/*TEST(ML_SVM, throw_exception_when_save_untrained_model) { - SVM svm; + Ptr svm; string filename = tempfile("svm.xml"); ASSERT_THROW(svm.save(filename.c_str()), Exception); remove(filename.c_str()); -} +}*/ TEST(DISABLED_ML_SVM, linear_save_load) { - CvSVM svm1, svm2, svm3; - svm1.load("SVM45_X_38-1.xml"); - svm2.load("SVM45_X_38-2.xml"); + Ptr svm1, svm2, svm3; + + svm1 = StatModel::load("SVM45_X_38-1.xml"); + svm2 = StatModel::load("SVM45_X_38-2.xml"); string tname = tempfile("a.xml"); - svm2.save(tname.c_str()); - svm3.load(tname.c_str()); + svm2->save(tname); + svm3 = StatModel::load(tname); - ASSERT_EQ(svm1.get_var_count(), svm2.get_var_count()); - ASSERT_EQ(svm1.get_var_count(), svm3.get_var_count()); + ASSERT_EQ(svm1->getVarCount(), svm2->getVarCount()); + ASSERT_EQ(svm1->getVarCount(), svm3->getVarCount()); - int m = 10000, n = svm1.get_var_count(); + int m = 10000, n = svm1->getVarCount(); Mat samples(m, n, CV_32F), r1, r2, r3; randu(samples, 0., 1.); - svm1.predict(samples, r1); - svm2.predict(samples, r2); - svm3.predict(samples, r3); + svm1->predict(samples, r1); + svm2->predict(samples, r2); + svm3->predict(samples, r3); double eps = 1e-4; - EXPECT_LE(cvtest::norm(r1, r2, NORM_INF), eps); - EXPECT_LE(cvtest::norm(r1, r3, NORM_INF), eps); + EXPECT_LE(norm(r1, r2, NORM_INF), eps); + EXPECT_LE(norm(r1, r3, NORM_INF), eps); remove(tname.c_str()); } diff --git a/modules/photo/doc/cloning.rst b/modules/photo/doc/cloning.rst index 384c6b676a504b269ec74b7c9a8a5415acd021cf..0965d3a72ffc2d320ae3fe78bbf604f069abacdc 100644 --- a/modules/photo/doc/cloning.rst +++ b/modules/photo/doc/cloning.rst @@ -7,7 +7,7 @@ seamlessClone ------------- Image editing tasks concern either global changes (color/intensity corrections, filters, deformations) or local changes concerned to a selection. Here we are interested in achieving local changes, ones that are restricted to a region manually selected (ROI), in a seamless and effortless manner. -The extent of the changes ranges from slight distortions to complete replacement by novel content. +The extent of the changes ranges from slight distortions to complete replacement by novel content [PM03]_. .. ocv:function:: void seamlessClone( InputArray src, InputArray dst, InputArray mask, Point p, OutputArray blend, int flags) @@ -25,13 +25,9 @@ The extent of the changes ranges from slight distortions to complete replacement * **NORMAL_CLONE** The power of the method is fully expressed when inserting objects with complex outlines into a new background - * **MIXED_CLONE** The classic method, color-based selection and alpha - masking might be time consuming and often leaves an undesirable halo. Seamless - cloning, even averaged with the original image, is not effective. Mixed seamless - cloning based on a loose selection proves effective. + * **MIXED_CLONE** The classic method, color-based selection and alpha masking might be time consuming and often leaves an undesirable halo. Seamless cloning, even averaged with the original image, is not effective. Mixed seamless cloning based on a loose selection proves effective. - * **FEATURE_EXCHANGE** Feature exchange allows the user to replace easily certain - features of one object by alternative features. + * **FEATURE_EXCHANGE** Feature exchange allows the user to easily replace certain features of one object by alternative features. @@ -97,3 +93,5 @@ region, giving its contents a flat aspect. Here Canny Edge Detector is used. **NOTE:** The algorithm assumes that the color of the source image is close to that of the destination. This assumption means that when the colors don't match, the source image color gets tinted toward the color of the destination image. + +.. [PM03] Patrick Perez, Michel Gangnet, Andrew Blake, "Poisson image editing", ACM Transactions on Graphics (SIGGRAPH), 2003. diff --git a/modules/photo/doc/decolor.rst b/modules/photo/doc/decolor.rst index cf7b9b9c4cf48fdaef6f331846609534c03b3b04..69bf0d590bf7ba8a5ffbfd4869a736e75f7db97f 100644 --- a/modules/photo/doc/decolor.rst +++ b/modules/photo/doc/decolor.rst @@ -6,7 +6,7 @@ Decolorization decolor ------- -Transforms a color image to a grayscale image. It is a basic tool in digital printing, stylized black-and-white photograph rendering, and in many single channel image processing applications. +Transforms a color image to a grayscale image. It is a basic tool in digital printing, stylized black-and-white photograph rendering, and in many single channel image processing applications [CL12]_. .. ocv:function:: void decolor( InputArray src, OutputArray grayscale, OutputArray color_boost ) @@ -17,3 +17,5 @@ Transforms a color image to a grayscale image. It is a basic tool in digital pri :param color_boost: Output 8-bit 3-channel image. This function is to be applied on color images. + +.. [CL12] Cewu Lu, Li Xu, Jiaya Jia, "Contrast Preserving Decolorization", IEEE International Conference on Computational Photography (ICCP), 2012. diff --git a/modules/photo/doc/hdr_imaging.rst b/modules/photo/doc/hdr_imaging.rst index bcd962f86df8b25d530f928d6ba5d39719995494..708ca8790245ef8e5ba3ed9149f5ef540f5af63b 100644 --- a/modules/photo/doc/hdr_imaging.rst +++ b/modules/photo/doc/hdr_imaging.rst @@ -356,7 +356,7 @@ Creates MergeRobertson object .. ocv:function:: Ptr createMergeRobertson() References -========== +--------------------------- .. [DM03] F. Drago, K. Myszkowski, T. Annen, N. Chiba, "Adaptive Logarithmic Mapping For Displaying High Contrast Scenes", Computer Graphics Forum, 2003, 22, 419 - 426. diff --git a/modules/photo/doc/npr.rst b/modules/photo/doc/npr.rst index 123c946c2a95ce7d15332cf8756e546cdf26af6e..c07fd69beb28cc176d2be6454ed5d4f0f7052d05 100644 --- a/modules/photo/doc/npr.rst +++ b/modules/photo/doc/npr.rst @@ -6,7 +6,7 @@ Non-Photorealistic Rendering edgePreservingFilter -------------------- -Filtering is the fundamental operation in image and video processing. Edge-preserving smoothing filters are used in many different applications. +Filtering is the fundamental operation in image and video processing. Edge-preserving smoothing filters are used in many different applications [EM11]_. .. ocv:function:: void edgePreservingFilter(InputArray src, OutputArray dst, int flags = 1, float sigma_s = 60, float sigma_r = 0.4f) @@ -16,9 +16,9 @@ Filtering is the fundamental operation in image and video processing. Edge-prese :param flags: Edge preserving filters: - * **RECURS_FILTER** + * **RECURS_FILTER** = 1 - * **NORMCONV_FILTER** + * **NORMCONV_FILTER** = 2 :param sigma_s: Range between 0 to 200. @@ -72,3 +72,5 @@ Stylization aims to produce digital imagery with a wide variety of effects not f :param sigma_s: Range between 0 to 200. :param sigma_r: Range between 0 to 1. + +.. [EM11] Eduardo S. L. Gastal, Manuel M. Oliveira, "Domain transform for edge-aware image and video processing", ACM Trans. Graph. 30(4): 69, 2011. diff --git a/modules/photo/src/npr.hpp b/modules/photo/src/npr.hpp index 744b2bdfbb480d0dfc8df9a5b72c9809ba96d068..2ff1985aca6ba33d16f8d189153ce25758ed60d7 100644 --- a/modules/photo/src/npr.hpp +++ b/modules/photo/src/npr.hpp @@ -173,6 +173,7 @@ void Domain_Filter::compute_Rfilter(Mat &output, Mat &hz, float sigma_h) { int h = output.rows; int w = output.cols; + int channel = output.channels(); float a = (float) exp((-1.0 * sqrt(2.0)) / sigma_h); @@ -185,11 +186,15 @@ void Domain_Filter::compute_Rfilter(Mat &output, Mat &hz, float sigma_h) for(int j=0;j(i,j) = pow(a,hz.at(i,j)); - for(int i=0; i(i,j) = temp.at(i,j) + (temp.at(i,j-1) - temp.at(i,j)) * V.at(i,j); + for(int c = 0; c(i,j*channel+c) = temp.at(i,j*channel+c) + + (temp.at(i,(j-1)*channel+c) - temp.at(i,j*channel+c)) * V.at(i,j); + } } } @@ -197,7 +202,11 @@ void Domain_Filter::compute_Rfilter(Mat &output, Mat &hz, float sigma_h) { for(int j =w-2; j >= 0; j--) { - temp.at(i,j) = temp.at(i,j) + (temp.at(i,j+1) - temp.at(i,j)) * V.at(i,j+1); + for(int c = 0; c(i,j*channel+c) = temp.at(i,j*channel+c) + + (temp.at(i,(j+1)*channel+c) - temp.at(i,j*channel+c))*V.at(i,j+1); + } } } diff --git a/modules/photo/src/seamless_cloning.cpp b/modules/photo/src/seamless_cloning.cpp index 6ddadb3202e267b398a1be70ebd756c43f7bd575..445c6dae74d97c2fcd300d4b78d6676929478160 100644 --- a/modules/photo/src/seamless_cloning.cpp +++ b/modules/photo/src/seamless_cloning.cpp @@ -108,6 +108,7 @@ void cv::seamlessClone(InputArray _src, InputArray _dst, InputArray _mask, Point Cloning obj; obj.normal_clone(dest,cd_mask,dst_mask,blend,flags); + } void cv::colorChange(InputArray _src, InputArray _mask, OutputArray _dst, float r, float g, float b) @@ -136,7 +137,6 @@ void cv::colorChange(InputArray _src, InputArray _mask, OutputArray _dst, float obj.local_color_change(src,cs_mask,gray,blend,red,green,blue); } - void cv::illuminationChange(InputArray _src, InputArray _mask, OutputArray _dst, float a, float b) { diff --git a/modules/photo/src/seamless_cloning.hpp b/modules/photo/src/seamless_cloning.hpp index 143d55089486c9973490ece0005110c0046ff935..669be9f08911b5cea304dd2be0bdb124a7424740 100644 --- a/modules/photo/src/seamless_cloning.hpp +++ b/modules/photo/src/seamless_cloning.hpp @@ -455,6 +455,8 @@ void Cloning::normal_clone(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, int num) { int w = I.size().width; int h = I.size().height; + int channel = I.channels(); + initialization(I,mask,wmask); @@ -466,20 +468,33 @@ void Cloning::normal_clone(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, int num) } else if(num == 2) { + for(int i=0;i < h; i++) - for(int j=0; j < w; j++) + { + for(int j=0; j < w; j++) { - if(abs(sgx.at(i,j) - sgy.at(i,j)) > abs(grx.at(i,j) - gry.at(i,j))) + for(int c=0;c(i,j) = sgx.at(i,j) * smask.at(i,j); - sry32.at(i,j) = sgy.at(i,j) * smask.at(i,j); - } - else - { - srx32.at(i,j) = grx.at(i,j) * smask.at(i,j); - sry32.at(i,j) = gry.at(i,j) * smask.at(i,j); + if(abs(sgx.at(i,j*channel+c) - sgy.at(i,j*channel+c)) > + abs(grx.at(i,j*channel+c) - gry.at(i,j*channel+c))) + { + + srx32.at(i,j*channel+c) = sgx.at(i,j*channel+c) + * smask.at(i,j); + sry32.at(i,j*channel+c) = sgy.at(i,j*channel+c) + * smask.at(i,j); + } + else + { + srx32.at(i,j*channel+c) = grx.at(i,j*channel+c) + * smask.at(i,j); + sry32.at(i,j*channel+c) = gry.at(i,j*channel+c) + * smask.at(i,j); + } } } + } + } else if(num == 3) { diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 129ae8bc050bfeb37157bfccf7b0766aad82b53b..7d964067afb9d64bfd59af5648573c9df7b31ae1 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -33,7 +33,8 @@ endforeach(m) # header blacklist ocv_list_filterout(opencv_hdrs ".h$") -ocv_list_filterout(opencv_hdrs "opencv2/core/cuda") +ocv_list_filterout(opencv_hdrs "cuda") +ocv_list_filterout(opencv_hdrs "cudev") ocv_list_filterout(opencv_hdrs "opencv2/objdetect/detection_based_tracker.hpp") ocv_list_filterout(opencv_hdrs "opencv2/optim.hpp") diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 622d24e64bdb821110a38c515e56569085e3bd6c..0bd914a248a4c123e3c8e1f5e488fa466c1b9c66 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -10,6 +10,7 @@ #include #include "pyopencv_generated_include.h" +#include "opencv2/core/types_c.h" #include "opencv2/opencv_modules.hpp" @@ -375,6 +376,12 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo info) return true; } +template<> +bool pyopencv_to(PyObject* o, Mat& m, const char* name) +{ + return pyopencv_to(o, m, ArgInfo(name, 0)); +} + template<> PyObject* pyopencv_from(const Mat& m) { @@ -1089,14 +1096,6 @@ bool pyopencv_to(PyObject* obj, CvSlice& r, const char* name) return PyArg_ParseTuple(obj, "ii", &r.start_index, &r.end_index) > 0; } -template<> -PyObject* pyopencv_from(CvDTreeNode* const & node) -{ - double value = node->value; - int ivalue = cvRound(value); - return value == ivalue ? PyInt_FromLong(ivalue) : PyFloat_FromDouble(value); -} - //////////////////////////////////////////////////////////////////////////////////////////////////// static void OnMouse(int event, int x, int y, int flags, void* param) diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 684b80f4e8c4994ce72c83fbd263f0f4de4fb62c..3dc2329c5abd51096cdad9c5017e2f6f3c16ac65 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -267,7 +267,7 @@ class ClassInfo(object): #return sys.exit(-1) if self.bases and self.bases[0].startswith("cv::"): self.bases[0] = self.bases[0][4:] - if self.bases and self.bases[0] == "Algorithm": + if self.bases and self.bases[0] == "cv::Algorithm": self.isalgorithm = True for m in decl[2]: if m.startswith("="): @@ -286,7 +286,7 @@ class ClassInfo(object): code = "static bool pyopencv_to(PyObject* src, %s& dst, const char* name)\n{\n PyObject* tmp;\n bool ok;\n" % (self.cname) code += "".join([gen_template_set_prop_from_map.substitute(propname=p.name,proptype=p.tp) for p in self.props]) if self.bases: - code += "\n return pyopencv_to(src, (%s&)dst, name);\n}\n" % all_classes[self.bases[0]].cname + code += "\n return pyopencv_to(src, (%s&)dst, name);\n}\n" % all_classes[self.bases[0].replace("::", "_")].cname else: code += "\n return true;\n}\n" return code @@ -761,7 +761,7 @@ class PythonWrapperGenerator(object): sys.exit(-1) self.classes[classinfo.name] = classinfo if classinfo.bases and not classinfo.isalgorithm: - classinfo.isalgorithm = self.classes[classinfo.bases[0]].isalgorithm + classinfo.isalgorithm = self.classes[classinfo.bases[0].replace("::", "_")].isalgorithm def add_const(self, name, decl): constinfo = ConstInfo(name, decl[1]) diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index eb9100928f9e3f4cc64492daec062cfe16a8f890..de33aeb911e5b5dade2ece2ea033d6d41212af9b 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -582,6 +582,7 @@ class CppHeaderParser(object): return name if name.startswith("cv."): return name + qualified_name = (("." in name) or ("::" in name)) n = "" for b in self.block_stack: block_type, block_name = b[self.BLOCK_TYPE], b[self.BLOCK_NAME] @@ -590,9 +591,12 @@ class CppHeaderParser(object): if block_type not in ["struct", "class", "namespace"]: print("Error at %d: there are non-valid entries in the current block stack " % (self.lineno, self.block_stack)) sys.exit(-1) - if block_name: + if block_name and (block_type == "namespace" or not qualified_name): n += block_name + "." - return n + name.replace("::", ".") + n += name.replace("::", ".") + if n.endswith(".Algorithm"): + n = "cv.Algorithm" + return n def parse_stmt(self, stmt, end_token): """ @@ -643,7 +647,7 @@ class CppHeaderParser(object): classname = classname[1:] decl = [stmt_type + " " + self.get_dotted_name(classname), "", modlist, []] if bases: - decl[1] = ": " + ", ".join([b if "::" in b else self.get_dotted_name(b).replace(".","::") for b in bases]) + decl[1] = ": " + ", ".join([self.get_dotted_name(b).replace(".","::") for b in bases]) return stmt_type, classname, True, decl if stmt.startswith("class") or stmt.startswith("struct"): @@ -658,7 +662,7 @@ class CppHeaderParser(object): if ("CV_EXPORTS_W" in stmt) or ("CV_EXPORTS_AS" in stmt) or (not self.wrap_mode):# and ("CV_EXPORTS" in stmt)): decl = [stmt_type + " " + self.get_dotted_name(classname), "", modlist, []] if bases: - decl[1] = ": " + ", ".join([b if "::" in b else self.get_dotted_name(b).replace(".","::") for b in bases]) + decl[1] = ": " + ", ".join([self.get_dotted_name(b).replace(".","::") for b in bases]) return stmt_type, classname, True, decl if stmt.startswith("enum"): diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index bba3d333966a3a620f8785741f87d58b37a3454f..96ac5045f5b833749f69de92f5a931f11bcecb20 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -148,7 +148,7 @@ endif(HAVE_INTELPERC) if(IOS) add_definitions(-DHAVE_IOS=1) - list(APPEND videoio_srcs src/ios_conversions.mm src/cap_ios_abstract_camera.mm src/cap_ios_photo_camera.mm src/cap_ios_video_camera.mm) + list(APPEND videoio_srcs src/cap_ios_abstract_camera.mm src/cap_ios_photo_camera.mm src/cap_ios_video_camera.mm) list(APPEND VIDEOIO_LIBRARIES "-framework Accelerate" "-framework AVFoundation" "-framework CoreGraphics" "-framework CoreImage" "-framework CoreMedia" "-framework CoreVideo" "-framework QuartzCore" "-framework AssetsLibrary") endif() diff --git a/samples/cpp/agaricus-lepiota.data b/samples/cpp/agaricus-lepiota.data deleted file mode 100644 index 14fe8bbe77cea19cae82214a52629fa4fed01eeb..0000000000000000000000000000000000000000 --- a/samples/cpp/agaricus-lepiota.data +++ /dev/null @@ -1,8124 +0,0 @@ -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,s,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,y,u -e,f,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,s,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,f,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,s,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,f,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,y,u -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,s,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,f,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,s,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,s,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,f,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,s,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,s,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,y,u -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,s,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,s,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,s,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,s,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,y,u -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,s,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,y,u -e,f,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,s,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,s,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,s,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,y,u -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,s,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,f,f,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,y,u -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,f,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,s,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -e,s,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,f,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,s,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,f,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,y,u -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,s,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,f,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,y,u -e,f,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,s,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,f,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,s,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,s,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,y,u -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,f,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,s,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,y,u -e,f,s,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,y,u -e,f,s,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,s,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,f,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,s,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,y,u -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,f,n,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,y,u -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,s,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,s,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,f,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,s,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,s,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,f,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,s,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,y,u -e,s,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -e,f,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,s,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,y,u -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,s,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,y,u -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,s,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,f,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,f,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,y,u -e,f,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,f,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,s,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,s,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,s,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,s,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,s,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,y,u -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -e,b,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -p,x,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,g -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,s,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,s,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -p,x,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,s,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,w,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,s,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,s,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,s,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,y,u -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,y,n,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,f,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,b,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,g -e,b,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,y,u -e,b,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,y,u -e,f,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,b,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -p,x,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,b,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,m -e,f,s,w,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,f,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,b,s,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,y,t,l,f,w,n,w,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,f,n,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,y,u -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,p -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,y,y,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,f,y,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,y,u -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,f,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,b,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,b,s,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,b,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,g -e,s,f,n,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,y,u -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -e,f,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,y,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -p,x,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,f,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,g -e,f,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,s,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -p,x,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,b,y,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,p -p,x,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,f,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -p,x,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,b,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,b,y,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,m -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,g,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,g,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,y,u -e,b,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,f,f,n,f,n,f,c,n,g,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,y,n,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,x,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,p -e,x,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,k,y,g -e,f,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,y,n,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -p,x,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,b,y,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,s,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,g -e,b,s,w,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,s,p -e,f,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,b,y,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,b,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,b,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,p -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,s,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,s,y,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,y,g -p,x,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,b,y,y,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,f,w,t,a,f,w,n,p,t,b,s,s,w,w,p,w,o,p,u,v,d -e,f,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,x,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,f,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,y,y,t,a,f,c,b,w,e,r,s,y,w,w,p,w,o,p,k,y,g -e,x,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,b,s,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,b,y,y,t,a,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,f,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,b,y,w,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,s,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,s,g -e,f,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -p,x,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,p -e,f,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,s,g -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,y,u -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -p,f,s,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,f,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,y,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -p,x,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,y,y,t,l,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,y,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,y,n,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,b,s,w,t,l,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,n,g -e,x,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,y,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,s,m -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,w,t,l,f,c,b,w,e,c,s,s,w,w,p,w,o,p,k,n,g -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,m -e,f,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,b,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,n,m -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,y,n,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,k,s,g -e,x,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,y,n,t,l,f,c,b,w,e,r,s,y,w,w,p,w,o,p,n,y,g -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,y,u -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,s,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,y,y,t,a,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,y,p -e,x,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -e,f,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -e,x,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,g,f,n,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,s,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,a,g -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -e,b,s,y,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -p,f,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,y,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,g -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -e,f,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,n,g -e,f,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,s,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,f,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,y,t,a,f,w,n,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,w,t,l,f,w,n,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,s,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,f,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,s,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,y,y,t,l,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,y,g -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,u -e,b,s,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,g,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,s,w,t,l,f,w,n,n,t,b,s,s,w,w,p,w,o,p,u,v,d -e,x,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,s,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,s,n,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -p,f,s,w,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,v,g -e,x,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,a,g -e,f,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,s,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,f,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -p,x,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,u -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -p,f,y,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,u -p,f,y,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,s,n,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,s,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,y,y,t,l,f,c,b,n,e,r,s,y,w,w,p,w,o,p,n,s,g -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -e,f,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,a,g -p,f,s,n,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,s,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,f,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,s,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,a,g -e,f,f,w,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,s,y,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,w,t,a,f,w,n,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,n,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,a,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,s,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,a,g -e,b,y,w,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,s,n,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,s,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,f,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,s,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,s,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,y,t,a,f,c,b,p,e,r,s,y,w,w,p,w,o,p,n,s,p -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,n,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,w,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,y,w,t,a,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,s,m -e,x,s,n,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,s,w,t,a,f,c,b,n,e,c,s,s,w,w,p,w,o,p,k,n,m -e,f,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -p,f,y,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,f,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,w,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,u -e,f,s,g,f,n,f,w,b,p,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,s,y,t,l,f,c,b,k,e,c,s,s,w,w,p,w,o,p,k,n,m -e,x,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,g,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,s,g -e,f,s,w,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,g,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,g,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,f,f,n,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -p,f,y,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,b,s,w,t,a,f,c,b,g,e,c,s,s,w,w,p,w,o,p,n,s,m -e,x,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,f,n,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,f,n,f,w,b,p,t,e,f,s,w,w,p,w,o,e,k,s,g -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,s,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,s,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -p,f,s,n,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,s,g -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,s,g,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,s,w,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,s,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,w,f,n,f,w,b,k,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,g,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,f,g,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,s,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,a,g -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,n,f,n,f,w,b,n,t,e,s,f,w,w,p,w,o,e,k,s,g -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,s,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,s,n,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,s,w,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,n,s,g -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,n,s,g -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -p,f,y,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,s,u -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,s,w,t,p,f,c,n,w,e,e,s,s,w,w,p,w,o,p,n,v,u -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,s,g -e,x,s,g,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,n,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,k,a,g -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,n,v,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,s,w,t,p,f,c,n,n,e,e,s,s,w,w,p,w,o,p,k,v,g -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,s,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,g,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,n,a,g -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -p,f,y,n,t,p,f,c,n,k,e,e,s,s,w,w,p,w,o,p,k,s,u -e,x,s,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,s,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,s,w,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,n,s,g -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -p,f,y,w,t,p,f,c,n,p,e,e,s,s,w,w,p,w,o,p,n,v,u -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,s,w,f,n,f,w,b,h,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,n,f,n,f,w,b,k,t,e,f,s,w,w,p,w,o,e,n,a,g -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,s,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,s,w,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,f,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,s,n,f,n,f,w,b,p,t,e,s,f,w,w,p,w,o,e,k,a,g -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,f,w,f,n,f,w,b,h,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,f,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,s,n,f,n,f,w,b,n,t,e,s,s,w,w,p,w,o,e,n,a,g -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,s,g,f,n,f,w,b,h,t,e,f,s,w,w,p,w,o,e,k,s,g -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,f,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,s,g,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,f,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,w,f,n,f,w,b,h,t,e,s,s,w,w,p,w,o,e,n,s,g -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,n,f,n,f,w,b,n,t,e,f,f,w,w,p,w,o,e,k,s,g -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,f,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,s,n,f,n,f,w,b,p,t,e,f,f,w,w,p,w,o,e,n,s,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,s,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,s,w,f,n,f,w,b,k,t,e,s,f,w,w,p,w,o,e,k,s,g -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,s,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,n,f,n,f,w,b,k,t,e,s,s,w,w,p,w,o,e,k,a,g -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,f,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -e,x,f,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,g,f,n,f,w,b,n,t,e,f,s,w,w,p,w,o,e,n,s,g -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,f,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,f,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,f,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -p,x,s,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,s,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -p,x,s,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,x,s,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,s,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,s,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,s,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,s,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,f,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -p,x,s,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,s,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -p,x,s,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,s,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,f,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,f,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,s,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,f,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,s,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,f,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,s,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,s,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,s,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -e,x,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -p,x,s,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,s,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -p,x,f,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,s,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,s,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -p,x,f,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,s,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,f,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,s,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,f,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,f,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,s,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,s,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,f,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,f,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,s,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -p,x,f,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,f,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,s,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,s,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -e,x,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,s,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,s,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,f,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,f,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,s,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,s,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,s,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -p,x,s,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,s,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,s,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,f,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,s,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,f,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -p,x,f,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,s,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,s,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,f,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,f,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,f,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,x,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,f,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,s,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -p,x,f,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,s,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,f,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,s,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,s,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,f,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,s,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,s,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,s,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,s,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,s,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,f,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,f,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,s,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,f,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -e,f,f,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,f,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,f,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -e,x,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,s,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,f,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -e,x,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,s,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,s,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,s,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,s,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,f,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,s,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -p,x,s,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -p,x,f,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,y,d -p,x,f,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,x,f,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,x,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,s,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,g,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,f,w,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,s,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,f,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,s,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,x,f,g,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,f,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,y,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -e,x,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,s,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,s,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,s,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,f,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -p,x,s,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,v,d -p,x,s,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,g,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -e,x,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,k,y,d -p,x,f,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,y,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,s,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,k,v,d -p,x,s,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,f,y,g,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,f,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,y,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,y,d -p,x,s,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,s,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,x,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,f,w,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,f,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,s,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,x,y,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,f,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,s,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -e,x,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,f,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,s,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,g,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,f,g,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,s,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,f,g,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,k,v,d -p,x,s,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -e,f,y,n,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,f,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,f,w,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,k,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -e,x,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,s,p,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,f,p,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -e,x,y,g,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,s,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,v,d -p,x,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,x,s,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -p,x,s,p,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,s,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,n,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,f,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -p,f,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -e,x,y,b,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,f,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,f,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -e,x,f,e,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -e,x,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,s,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,s,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,s,w,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,f,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,f,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,n,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,f,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,s,d -p,x,s,p,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -e,x,f,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -e,f,f,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,k,y,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,n,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,n,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -e,f,y,u,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,y,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -e,x,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -e,x,f,e,t,n,f,c,b,w,t,b,s,s,p,p,p,w,o,p,k,v,d -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,f,w,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,n,y,d -p,x,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,f,y,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -p,b,s,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,g,p,w,o,p,k,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,k,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -e,f,y,e,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,k,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -e,f,f,n,t,n,f,c,b,u,t,b,s,s,w,p,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,k,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,p,t,b,s,s,g,g,p,w,o,p,k,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,s,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,w,t,b,s,s,p,g,p,w,o,p,k,v,d -p,f,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,u -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -e,f,y,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,v,d -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,u,t,b,s,s,g,p,p,w,o,p,k,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -e,f,f,c,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,u -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,k,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,k,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -p,x,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,p,t,b,s,s,p,w,p,w,o,p,n,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,n,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,u -e,f,f,e,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,y,d -p,x,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,x,y,g,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,v,d -p,x,f,g,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,p,f,c,f,w,n,n,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,u -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,p,t,b,s,s,w,w,p,w,o,p,n,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,u -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -p,x,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -e,f,y,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,f,p,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -e,k,y,n,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,x,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -e,f,s,p,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -e,k,s,p,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -e,f,y,g,t,n,f,c,b,w,t,b,s,s,p,w,p,w,o,p,n,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -e,x,y,r,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -p,k,y,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,w,t,b,s,s,g,p,p,w,o,p,k,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -e,x,s,e,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,b,y,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,f,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,u -e,f,y,g,t,n,f,c,b,n,t,b,s,s,g,p,p,w,o,p,n,v,d -p,x,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -e,f,y,e,t,n,f,c,b,n,t,b,s,s,p,g,p,w,o,p,k,v,d -e,x,y,b,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -e,x,y,u,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,u,t,b,s,s,g,w,p,w,o,p,n,y,d -p,f,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -e,f,y,n,t,n,f,c,b,u,t,b,s,s,w,g,p,w,o,p,k,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -e,x,s,b,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,k,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,s,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,u -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,y,g,t,n,f,c,b,u,t,b,s,s,p,g,p,w,o,p,k,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -e,f,y,n,t,n,f,c,b,n,t,b,s,s,g,g,p,w,o,p,n,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -e,k,y,b,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -p,x,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,u -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,p,t,b,s,s,w,p,p,w,o,p,k,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -e,f,f,c,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,g -e,f,s,n,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,g,f,c,f,w,n,u,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -e,x,s,b,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -e,x,y,u,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,v,d -e,f,y,b,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,g -p,f,s,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,g,g,p,w,o,p,n,v,d -e,x,y,u,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,g,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -p,x,f,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,n,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,f,y,b,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,f,g,f,c,f,c,n,u,e,b,s,s,w,w,p,w,o,p,k,s,d -p,f,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,g -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,k,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -e,f,y,u,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,p,p,w,o,p,n,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -p,x,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,u -e,f,y,e,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,y,u,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,s,p,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,n,t,b,s,s,w,g,p,w,o,p,n,y,d -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,f,w,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,s,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,u -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,u -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -e,f,y,u,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -e,f,f,e,t,n,f,c,b,w,t,b,s,s,g,w,p,w,o,p,n,v,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -e,f,f,e,t,n,f,c,b,w,t,b,s,s,w,w,p,w,o,p,k,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,f,p,f,c,f,w,n,p,e,b,s,s,w,w,p,w,o,p,k,s,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -e,x,s,p,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -p,x,s,p,f,c,f,w,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -e,f,f,e,t,n,f,c,b,p,t,b,s,s,p,p,p,w,o,p,n,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -e,f,f,g,t,n,f,c,b,u,t,b,s,s,w,w,p,w,o,p,n,y,d -p,x,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -e,f,y,n,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -p,k,f,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -e,x,y,e,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,d -p,b,f,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,u -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -e,k,y,e,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -e,k,y,n,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,s,w,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -p,b,s,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,g -e,f,y,e,t,n,f,c,b,n,t,b,s,s,w,p,p,w,o,p,n,y,d -p,b,s,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -e,f,y,p,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,g -e,k,y,n,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,u -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -e,k,s,e,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,y,n,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,s,b,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -e,f,f,g,t,n,f,c,b,u,t,b,s,s,p,p,p,w,o,p,n,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -p,x,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -e,f,y,n,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,f,y,e,t,n,f,c,b,p,t,b,s,s,g,p,p,w,o,p,k,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,p,w,p,w,o,p,n,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -p,b,y,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -e,f,y,p,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,u -p,x,f,g,f,c,f,c,n,g,e,b,s,s,w,w,p,w,o,p,k,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -e,x,y,r,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,y,d -e,f,y,w,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,p -e,f,s,e,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -p,x,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,u -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,s,w,f,c,f,c,n,n,e,b,s,s,w,w,p,w,o,p,k,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -e,f,y,e,t,n,f,c,b,u,t,b,s,s,p,w,p,w,o,p,n,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -e,k,s,b,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -e,x,f,n,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -e,x,y,e,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,f,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,k,s,b,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -e,k,s,p,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -p,f,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,u -e,f,s,b,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,u -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,u -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,u -e,f,s,b,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -e,x,s,p,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -e,k,s,n,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,y,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,p -e,f,y,u,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,x,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -e,f,s,b,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -e,x,s,n,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,y,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,b,s,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -e,f,y,u,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -e,x,y,e,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -e,k,s,b,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,u -p,x,s,g,f,c,f,c,n,p,e,b,s,s,w,w,p,w,o,p,n,v,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,d -e,f,y,u,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,u -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -e,k,y,b,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,y,b,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,y,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,u -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,u -p,x,f,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,u -p,f,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,u -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,p -e,x,y,w,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,y,d -e,x,s,e,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,d -e,f,f,e,t,n,f,c,b,n,t,b,s,s,g,w,p,w,o,p,n,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,k,f,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,g -e,k,f,c,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -p,b,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -e,k,y,p,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -e,k,y,b,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,g,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,x,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,u -e,f,y,c,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -e,f,y,p,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,g -p,x,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -e,k,y,c,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,x,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,g -p,f,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,g -e,k,y,c,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,f,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,u -p,c,g,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,x,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,u -p,b,g,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,x,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,u -p,b,f,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,x,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -e,k,s,n,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,k,y,b,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,s,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,s,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -e,x,s,e,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,f,f,n,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,x,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,x,s,b,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -e,f,y,n,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,p -p,f,y,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,x,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,f,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -e,x,y,e,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,g -p,f,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,g -e,k,y,e,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -p,x,f,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -e,x,y,n,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,d -e,x,s,p,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -e,f,s,p,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,b,y,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -e,x,y,c,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -e,f,s,n,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,d -p,x,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,g -e,f,y,c,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,d -e,x,y,w,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,p -p,f,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,k,s,p,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -p,f,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,u -e,k,y,p,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -p,x,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,f,y,f,f,f,c,b,h,e,b,k,k,n,n,p,w,o,l,h,v,d -p,f,f,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -e,k,s,b,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,u -e,f,s,n,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -e,k,y,b,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,k,y,n,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,d -e,x,y,p,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,u -e,x,s,p,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,b,y,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,g -p,x,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,k,y,p,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -e,k,f,c,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,g -e,x,s,n,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,u -p,k,y,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -e,k,f,n,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,k,f,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,g -p,f,y,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -e,k,s,b,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,p -p,b,y,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,g -e,k,y,b,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -p,x,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,g -p,b,y,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,g -e,k,s,n,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,u -p,x,f,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -p,k,y,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,u -e,f,s,e,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,u -e,k,s,n,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -e,x,s,b,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,u -e,k,f,n,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,g -p,x,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,u -p,k,y,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,s,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -e,k,y,p,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,f,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,u -e,f,y,w,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,y,d -e,x,y,r,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,v,d -e,x,s,b,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,g -p,x,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,p -p,f,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,g -p,f,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,g -e,f,s,e,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,d -e,x,s,n,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,g -e,x,f,c,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,f,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -e,x,y,r,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,f,y,e,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,u -e,k,y,n,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -e,f,y,e,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,b,y,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,x,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,f,y,b,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,g -e,f,y,n,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,b,y,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -e,x,y,n,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,x,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,f,y,w,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,f,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,x,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,d -p,x,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,u -p,k,y,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,u -e,x,y,c,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,y,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,p -e,x,s,p,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,f,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -e,f,f,n,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -e,x,f,n,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,u -p,f,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,g -p,x,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,g -e,x,y,r,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,v,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,d -p,f,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -p,f,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,u -p,x,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,u -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,p -p,x,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,y,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -e,f,s,n,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,g -e,k,s,p,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,g -p,x,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,b,y,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,s,b,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,u -e,x,y,p,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,v,p -p,x,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,u -e,f,y,b,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,u -e,x,f,c,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -e,f,s,p,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -e,x,y,n,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -e,f,s,p,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,u -e,k,s,e,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -e,k,y,e,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,g -e,k,s,b,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,g -p,b,y,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,x,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,b,f,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,y,c,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,b,y,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,p -p,f,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -e,x,s,n,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,v,g -e,f,s,b,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -e,f,s,e,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,x,s,n,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,u -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,d -e,f,s,p,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,x,s,e,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -p,b,y,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,u -p,f,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,g -p,x,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,u -e,f,y,r,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,g -e,x,f,c,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,u -e,x,y,u,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,v,p -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,y,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -e,x,y,p,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,u -p,f,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,u -p,f,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,u -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,g -p,f,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,g -e,k,s,e,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,k,y,n,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,u -p,f,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,x,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,b,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,x,y,g,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,v,g -p,f,y,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,g -e,k,y,p,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,g -e,k,s,p,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,d -p,f,y,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,y,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -e,x,y,e,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,p -p,x,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,d -e,k,y,e,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -e,f,y,r,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,g -p,x,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,k,y,e,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -e,k,s,p,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,y,b,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,c,y,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,f,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,x,y,p,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,y,w,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,x,f,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,d -p,x,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,u -p,x,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -e,k,f,n,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,x,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,u -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,p -e,x,y,p,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,b,y,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,p -e,f,y,r,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,d -p,x,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,f,y,n,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,f,y,e,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -e,x,y,b,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,s,n,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,g -e,x,y,b,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,y,g -e,k,y,e,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,u -e,k,y,c,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -e,k,y,n,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,b,s,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,u -e,f,y,r,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,g -e,f,y,u,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,u -e,x,s,e,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,y,p,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,p -e,x,y,n,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,y,e,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,u -p,x,y,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,g -p,f,s,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,g -e,f,s,p,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,x,y,n,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,b,y,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,d -p,x,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,g -p,x,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,b,y,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -e,x,y,c,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,f,y,n,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -e,x,y,r,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,y,d -p,b,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -e,x,s,p,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,x,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,y,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -e,f,s,b,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,u -e,x,y,c,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -e,x,y,w,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,f,y,n,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,k,y,p,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,s,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,g -e,k,y,n,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,f,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,s,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,f,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,g -e,f,s,n,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,d -p,x,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,g -p,b,s,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,u -e,f,y,b,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -e,x,f,c,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,f,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,u -e,k,s,p,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,k,y,n,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,x,y,b,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,g -p,f,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,p -e,x,y,e,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,g -e,x,y,n,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,d -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,d -p,b,y,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,p -p,f,f,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,y,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -e,k,s,e,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,n,p,w,o,l,h,v,p -e,k,s,n,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,y,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,f,y,g,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,f,s,b,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,g -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -e,f,y,p,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,p -e,k,s,e,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -e,x,y,n,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,f,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,u -p,x,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,g -p,x,f,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,g -e,f,y,e,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,g -p,x,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,u -e,k,f,n,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,g -e,k,y,c,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,x,f,g,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,y,p -p,x,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -e,f,y,p,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,n,p,w,o,l,h,y,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,y,g -p,x,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,f,y,r,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,p -p,x,s,g,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,g -p,x,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,s,b,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,g -e,x,y,n,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,f,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -e,f,y,u,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,x,s,n,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,u -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,b,p,w,o,l,h,v,d -e,f,s,b,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -e,k,f,c,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,f,y,p,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,k,y,p,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,g -p,x,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -e,f,s,n,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,g -p,x,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -e,f,s,n,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,p,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,y,p -e,f,f,c,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,b,y,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -e,k,y,e,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,u -p,x,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,g -p,x,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,u -p,x,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,w,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,u -e,f,s,p,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -e,f,y,w,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,v,d -e,x,y,p,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,u -p,k,g,w,t,n,f,w,n,w,e,b,s,s,w,w,p,w,o,p,w,c,l -p,f,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,k,s,e,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,s,g -e,f,s,e,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,g -p,b,s,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -e,f,f,n,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,f,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,u -p,x,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,g -e,x,s,b,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,y,n,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,x,y,r,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,v,d -e,k,s,b,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,u -p,x,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -e,x,y,b,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,g -e,k,s,n,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,g -e,k,y,p,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,v,p -p,x,s,b,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,u -e,k,f,c,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,f,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,g -p,b,s,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,b,s,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,g -e,x,y,n,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,y,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -e,x,s,b,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,s,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,u -p,f,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,u -e,f,y,r,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,s,w,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,u -e,x,y,w,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,k,y,b,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,u -e,f,y,r,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,y,d -p,b,s,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,s,b,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,u -e,f,y,c,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -e,x,y,u,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,u -e,x,y,w,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,v,d -p,x,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -e,x,s,p,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,s,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,y,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,p -p,f,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,y,g,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,v,g -e,f,y,e,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,b,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,v,u -p,x,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,y,p -p,b,y,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -e,f,y,n,f,n,f,w,n,w,e,b,s,f,w,n,p,w,o,e,w,v,l -p,f,f,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,b,p,w,o,l,h,v,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,g -p,x,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,y,y,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,x,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -e,f,f,n,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,u -e,f,s,e,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,d -p,x,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -e,f,f,c,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,x,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,k,y,e,t,n,f,c,b,e,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,w,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,x,s,n,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,p -p,x,s,w,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,u -e,f,s,e,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,v,d -e,f,y,e,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,g -e,k,s,n,t,n,f,c,b,e,e,?,s,s,e,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,s,w,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,g -p,f,y,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -e,f,y,w,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,v,d -e,x,y,w,f,n,f,c,n,h,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,u -e,x,y,u,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,v,d -p,f,s,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,y,f,f,f,c,b,g,e,b,k,k,p,b,p,w,o,l,h,v,d -p,f,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,n,p,w,o,l,h,y,p -p,x,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,y,d -p,x,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,s,g -e,f,y,r,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,v,d -p,f,s,w,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,s,g -e,k,s,n,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,s,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,p,p,p,w,o,l,h,y,g -p,b,s,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,b,s,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,m -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,y,g,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,d -e,x,y,e,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,k,s,b,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -e,x,s,e,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,s,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,m -e,f,s,n,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,f,y,b,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,s,g,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,s,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,h,e,b,k,k,b,b,p,w,o,l,h,y,g -p,x,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,s,g,t,f,f,c,b,w,t,b,f,s,w,w,p,w,o,p,h,v,u -e,x,y,p,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,x,y,u,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,y,p -p,x,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,u -p,x,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,b,p,p,w,o,l,h,y,g -p,b,s,w,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -e,k,y,n,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,x,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,v,p -e,x,y,b,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -p,x,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,g -p,f,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,u -p,f,s,b,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,b,p,w,o,l,h,y,d -p,x,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,w,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,u -e,k,y,n,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,s,e,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -e,x,y,w,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,v,d -p,f,s,b,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,u -p,f,s,g,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,s,u -e,f,s,e,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,y,g -p,x,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,s,w,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,g,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -e,x,f,n,f,n,f,w,n,w,e,b,s,s,w,n,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,p,e,b,k,k,b,n,p,w,o,l,h,y,d -p,x,s,w,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,u -e,x,s,b,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -e,x,s,p,t,n,f,c,b,w,e,?,s,s,w,e,p,w,t,e,w,c,w -p,x,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,g -p,f,f,g,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,p -p,x,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,f,y,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,v,p -p,f,s,w,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,u -e,k,y,n,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -e,x,y,p,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,y,g -e,f,y,b,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,s,e,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,s,g,t,f,f,c,b,p,t,b,f,f,w,w,p,w,o,p,h,v,u -p,f,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,u -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,p -p,x,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,s,w,t,f,f,c,b,h,t,b,s,s,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,n,p,p,w,o,l,h,y,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,n,p,p,w,o,l,h,y,p -p,f,f,g,f,f,f,c,b,g,e,b,k,k,b,n,p,w,o,l,h,v,p -p,b,s,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,b,s,w,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -e,x,y,n,t,n,f,c,b,w,e,?,s,s,w,w,p,w,t,e,w,c,w -p,x,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,x,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,g -p,b,y,y,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,x,s,g,t,f,f,c,b,w,t,b,f,f,w,w,p,w,o,p,h,v,u -e,f,y,p,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,k,s,p,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -e,f,y,n,t,n,f,c,b,w,e,?,s,s,e,e,p,w,t,e,w,c,w -p,b,y,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,s,g,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,v,g -p,x,s,g,t,f,f,c,b,p,t,b,s,s,w,w,p,w,o,p,h,s,g -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,g -p,x,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,y,b,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,r,v,g -p,x,s,b,t,f,f,c,b,h,t,b,f,f,w,w,p,w,o,p,h,v,u -p,b,s,b,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -e,f,y,w,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,y,d -e,x,y,n,f,n,f,w,n,w,e,b,f,s,w,n,p,w,o,e,w,v,l -p,x,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,v,p -p,f,s,b,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,v,g -e,k,s,e,t,n,f,c,b,e,e,?,s,s,e,e,p,w,t,e,w,c,w -p,f,f,y,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,p -p,b,f,n,f,n,f,c,n,w,e,?,k,y,w,n,p,w,o,e,w,v,d -p,x,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,y,y,f,f,f,c,b,h,e,b,k,k,n,b,p,w,o,l,h,y,d -p,x,y,y,f,f,f,c,b,g,e,b,k,k,n,b,p,w,o,l,h,y,g -p,x,s,g,t,f,f,c,b,p,t,b,f,s,w,w,p,w,o,p,h,s,g -p,x,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -e,x,y,n,t,n,f,c,b,w,e,?,s,s,e,w,p,w,t,e,w,c,w -e,f,y,n,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,f,y,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,k,f,n,f,n,f,c,n,w,e,?,k,y,w,y,p,w,o,e,w,v,d -p,f,y,g,f,f,f,c,b,h,e,b,k,k,p,n,p,w,o,l,h,v,g -p,x,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,f,g,f,f,f,c,b,h,e,b,k,k,p,p,p,w,o,l,h,y,d -p,f,f,g,f,f,f,c,b,p,e,b,k,k,b,p,p,w,o,l,h,v,d -p,x,s,g,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,s,g -e,x,y,r,f,n,f,c,n,u,e,?,s,f,w,w,p,w,o,f,h,y,d -p,x,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,s,p,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,m -p,x,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,g -p,x,s,b,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,v,u -e,k,s,e,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -p,f,s,b,t,f,f,c,b,w,t,b,s,f,w,w,p,w,o,p,h,s,g -e,x,f,n,f,n,f,w,n,w,e,b,f,f,w,n,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,v,d -e,f,y,w,f,n,f,c,n,w,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,s,w,t,f,f,c,b,w,t,b,s,s,w,w,p,w,o,p,h,s,u -p,f,s,p,t,n,f,c,b,r,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,y,f,f,f,c,b,p,e,b,k,k,p,n,p,w,o,l,h,y,d -p,f,s,g,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,g -p,x,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,b,p,w,o,l,h,v,g -p,f,s,g,t,f,f,c,b,h,t,b,f,s,w,w,p,w,o,p,h,s,g -p,f,s,w,t,f,f,c,b,h,t,b,s,f,w,w,p,w,o,p,h,s,u -p,x,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,f,y,f,f,f,c,b,p,e,b,k,k,n,n,p,w,o,l,h,y,d -p,x,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,b,y,b,t,n,f,c,b,g,e,b,s,s,w,w,p,w,t,p,r,v,g -p,f,y,y,f,f,f,c,b,g,e,b,k,k,b,p,p,w,o,l,h,v,g -p,x,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,k,y,b,t,n,f,c,b,e,e,?,s,s,w,w,p,w,t,e,w,c,w -e,x,y,w,f,n,f,c,n,p,e,?,s,f,w,w,p,w,o,f,h,y,d -p,f,s,b,t,f,f,c,b,p,t,b,s,f,w,w,p,w,o,p,h,v,u -p,f,y,y,f,f,f,c,b,g,e,b,k,k,n,p,p,w,o,l,h,y,g -p,x,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,c,l -p,f,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,x,f,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -e,x,s,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -e,b,f,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,x,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,k,f,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,x,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -p,x,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,s,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,f,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,k,s,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,f,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,k,s,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,x,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -e,b,s,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,f,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,x,f,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,f,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -e,k,f,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,x,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,x,s,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,f,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,f,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,f,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,y,e,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,f,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,c,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,k,s,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,x,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -e,x,f,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,k,f,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,b,f,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,x,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,c,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -e,x,f,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,x,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,f,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,c,l -p,f,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,x,s,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,c,l -p,x,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,y,n,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,x,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,k,s,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,f,y,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,x,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,f,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,x,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,x,y,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,c,l -e,k,s,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,x,f,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,x,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,c,l -p,k,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,b,y,y,f,n,f,w,n,y,e,c,y,y,y,y,p,y,o,e,w,c,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,b,s,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -e,b,f,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,c,l -e,x,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,f,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,b,s,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -p,x,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -e,b,s,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -e,f,y,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,x,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,x,f,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,k,f,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -e,b,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -p,f,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,x,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -p,f,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,x,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,k,f,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,f,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,b,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,c,l -e,k,s,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,y,e,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,c,l -p,f,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -e,k,f,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,x,s,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,k,s,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -e,k,s,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,k,s,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -e,x,f,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,b,s,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,f,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,k,s,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -e,x,s,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,f,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -e,k,f,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,f,s,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,c,l -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -e,k,f,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,c,l -p,k,y,c,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -e,x,f,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,v,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,v,l -p,k,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,f,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,x,f,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,f,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -e,k,s,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,c,l -p,k,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,x,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,x,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,f,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,y,n,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,k,s,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,f,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -e,f,s,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,f,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,c,l -p,f,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,b,f,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,v,l -p,f,y,n,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,f,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,x,f,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,y,c,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,b,s,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -e,b,f,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,f,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -e,x,y,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,f,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,k,f,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,x,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,k,s,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,v,l -p,f,y,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,k,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,v,l -e,k,s,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,v,l -e,x,f,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,b,s,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,f,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,x,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,f,y,c,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,f,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,f,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -e,k,s,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,x,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,x,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -p,x,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,b,s,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,x,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,f,y,n,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,b,f,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,f,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,k,s,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,x,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,f,y,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,c,l -e,b,s,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,c,l -p,x,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,f,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -e,k,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -p,f,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -e,b,f,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,x,s,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,x,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,x,f,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,k,s,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,x,s,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,v,l -e,k,s,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,x,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,v,l -e,k,s,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,y,e,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,b,s,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,f,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,c,l -e,b,s,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,c,l -e,b,s,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,c,l -p,k,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,v,l -e,k,f,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,c,l -p,k,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,k,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,k,s,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,f,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,f,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,n,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -e,b,f,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -e,k,f,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,k,s,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -e,k,s,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,b,s,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,x,s,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,x,f,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,x,y,e,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -e,f,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -e,k,s,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -p,f,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,k,y,y,f,n,f,w,n,y,e,c,y,y,y,y,p,y,o,e,w,c,l -p,k,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,c,l -e,x,s,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,k,s,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,f,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,b,f,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,c,l -p,k,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -e,x,f,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -e,x,s,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -e,b,f,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,x,y,n,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,c,l -p,x,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,v,l -e,b,s,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -e,b,s,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,b,f,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -e,k,s,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -p,f,y,e,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,f,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,y,c,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -e,k,s,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,f,y,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -e,b,f,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,f,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,k,s,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,c,l -e,f,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,b,s,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,x,f,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -e,x,s,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,k,f,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,c,l -p,f,y,y,f,n,f,w,n,w,e,c,y,y,y,y,p,y,o,e,w,c,l -p,x,y,c,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,x,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,x,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,k,s,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -e,b,s,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,v,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,v,l -e,f,s,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,k,s,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,f,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,b,s,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,v,l -p,k,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,b,s,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,c,y,y,f,n,f,w,n,y,e,c,y,y,y,y,p,y,o,e,w,c,l -p,x,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,x,s,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,v,l -p,k,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,c,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,v,l -p,x,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -e,x,s,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -e,x,f,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,f,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,k,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,c,l -p,f,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,s,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,x,f,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,v,l -p,f,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,f,y,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,x,f,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -e,k,s,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,k,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -e,x,f,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,v,l -p,f,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,c,l -e,x,f,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,c,l -e,b,s,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,x,y,e,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,x,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,f,s,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,y,f,n,f,w,n,y,e,c,y,y,y,y,p,y,o,e,w,c,l -p,k,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,f,y,e,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,x,f,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,x,s,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -e,x,f,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,k,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,c,l -p,f,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,x,s,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -e,x,f,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,c,l -p,k,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,c,l -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,c,l -e,b,s,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,c,l -p,k,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -e,b,s,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,x,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,c,l -e,x,s,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -e,k,s,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -e,x,s,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,y,c,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,x,s,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -e,k,f,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -e,b,f,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,v,l -p,k,y,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,v,l -e,f,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -e,b,s,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,k,s,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,c,l -p,k,s,n,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,v,l -e,x,f,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -e,x,f,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,v,l -p,k,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -e,x,s,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -e,x,f,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,k,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,k,f,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -e,k,f,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,b,f,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -e,b,s,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -e,b,s,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,f,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,k,s,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,x,s,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,s,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,b,y,y,f,n,f,w,n,w,e,c,y,y,y,y,p,y,o,e,w,c,l -p,k,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -e,x,s,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,k,s,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,k,f,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,x,f,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -e,b,s,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,v,l -p,f,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,b,f,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -e,k,s,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,x,s,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,c,l -p,k,s,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,v,l -e,b,f,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,x,s,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,y,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,x,y,e,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,c,l -p,k,s,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -e,b,s,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -e,b,s,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,k,f,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -e,k,f,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -e,x,f,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,f,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,f,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,b,s,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,k,s,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,b,f,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,y,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,f,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,x,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,v,l -e,k,f,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,c,l -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,y,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -p,k,y,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,x,s,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,f,y,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,v,l -p,k,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,b,f,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -e,x,s,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,x,s,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,v,l -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,v,l -p,k,s,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,c,l -p,k,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,k,f,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,v,l -e,f,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -p,k,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -e,f,s,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,x,s,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,c,y,y,f,n,f,w,n,w,e,c,y,y,y,y,p,y,o,e,w,c,l -p,k,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,x,s,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,x,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,y,c,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,f,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,c,l -e,b,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -p,f,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,b,s,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,x,y,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,f,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,c,l -e,b,f,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -e,x,f,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,x,y,n,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,s,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,x,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,v,l -p,k,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,y,y,f,n,f,w,n,w,e,c,y,y,y,y,p,y,o,e,w,c,l -e,k,s,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,s,n,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,k,f,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -e,k,f,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,c,l -e,x,s,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -e,b,s,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -e,x,f,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,k,f,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -e,x,s,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,v,l -e,b,s,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,b,s,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,x,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -e,x,s,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,x,s,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -p,k,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,v,l -e,b,f,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,s,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -p,f,y,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,f,y,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,k,s,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,c,l -p,k,y,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,k,s,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,y,e,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -e,x,s,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,s,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,f,y,c,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,x,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -e,k,f,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,x,y,n,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,f,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,b,f,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -e,b,f,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,c,l -e,x,s,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,p -e,x,s,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -e,b,s,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -p,k,y,e,f,m,f,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,y,e,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -e,x,f,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,v,l -p,k,y,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,v,l -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,v,l -e,k,s,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,y,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,v,l -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,y,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -e,b,f,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,v,l -p,k,s,n,f,s,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,l -e,x,s,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,v,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,c,l -p,k,y,e,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,x,f,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,n,f,s,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,x,s,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,s,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,x,f,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,c,l -e,b,s,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,y,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,x,y,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,v,l -e,x,s,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -e,k,s,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,v,l -e,k,s,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,n,g -p,k,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,x,s,n,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,s,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,x,f,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,f,s,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,v,l -e,x,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -p,k,y,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,s,e,f,s,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,l -e,b,s,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,c,l -e,x,s,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -e,x,s,g,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -p,f,y,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,f,y,n,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -e,f,y,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,k,s,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -p,x,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,x,f,g,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,s,g -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,v,l -p,x,y,e,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,s,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,x,f,g,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,s,g -p,f,y,n,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,v,l -e,b,s,g,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,v,l -p,k,s,e,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,v,l -e,x,f,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -p,k,s,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,x,f,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -e,f,y,p,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,c,l -p,k,s,n,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,y,n,f,s,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,d -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,c,l -p,k,y,n,f,s,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -e,k,f,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -e,f,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -p,k,s,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,x,f,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,c,l -e,k,s,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,c,l -e,f,y,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,k,y,e,f,m,a,c,b,w,e,c,k,y,c,c,p,w,n,n,w,c,d -p,x,y,e,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,y,v,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,o,c,l -e,x,y,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -p,k,y,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,f,s,c,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,v,p -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,c,l -e,b,f,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,n,g -e,k,f,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,c,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,c,l -e,k,f,w,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -p,k,y,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,l -e,b,f,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,y,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,v,l -e,x,y,n,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -e,b,s,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,n,g -p,f,y,n,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,c,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,v,l -e,b,s,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -p,x,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -e,b,f,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -e,b,f,w,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,v,l -p,k,y,e,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,v,l -p,k,s,e,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,d -p,k,y,c,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -p,k,s,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -e,x,s,g,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,d -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,c,l -e,b,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -p,x,s,n,f,f,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,c,l -p,f,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -e,k,s,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -e,k,f,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,s,g -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,c,l -p,k,s,e,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,v,l -p,k,s,n,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,c,l -e,x,y,n,f,n,f,c,b,w,e,b,y,y,n,n,p,w,t,p,w,y,p -e,x,s,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,v,l -p,f,s,n,f,f,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,x,y,e,f,s,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,d -p,k,y,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,s,e,f,f,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,s,e,f,f,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,n,v,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,v,l -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,v,l -e,k,s,g,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,s,g -e,k,f,w,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -p,k,s,e,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,p -e,b,f,w,f,n,f,w,b,g,e,?,k,k,w,w,p,w,t,p,w,n,g -e,x,s,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -e,b,f,w,f,n,f,w,b,p,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,n,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,d -p,f,s,n,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -p,f,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,s,e,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,p -p,k,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,c,l -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,v,l -p,k,y,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -e,x,f,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,n,g -e,k,s,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,k,p,p,p,w,o,e,w,v,p -p,x,s,n,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,c,l -e,b,f,w,f,n,f,w,b,p,e,?,k,s,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,v,l -p,k,y,e,f,y,f,c,n,b,t,?,k,k,p,w,p,w,o,e,w,v,d -e,x,y,g,t,n,f,c,b,w,e,b,s,s,w,w,p,w,t,p,w,y,p -p,k,s,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,v,l -p,k,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,c,l -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,b,v,l -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,c,l -p,k,y,e,f,y,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,b,f,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,s,e,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,s,n,f,y,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,d -e,x,f,w,f,n,f,w,b,w,e,?,s,k,w,w,p,w,t,p,w,s,g -e,f,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,v,l -p,k,y,n,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,x,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,v,l -e,b,f,g,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,y,v,l -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,k,w,p,p,w,o,e,w,v,l -p,k,s,n,f,f,f,c,n,b,t,?,s,s,w,p,p,w,o,e,w,v,p -p,k,s,n,f,s,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,s,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -e,k,f,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,v,l -p,k,s,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -e,b,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,c,l -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,b,c,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,n,c,l -p,k,y,e,f,s,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,l -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,y,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,y,v,l -e,b,f,g,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,n,g -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,o,c,l -e,b,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,y,c,l -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,o,v,l -e,b,f,g,f,n,f,w,b,g,e,?,s,s,w,w,p,w,t,p,w,s,g -p,k,y,e,f,f,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -p,k,s,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,s,w,w,p,w,o,e,w,v,p -p,k,s,e,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,p -p,k,y,n,f,y,f,c,n,b,t,?,s,s,w,w,p,w,o,e,w,v,l -e,b,f,g,f,n,f,w,b,p,e,?,k,k,w,w,p,w,t,p,w,s,g -e,k,f,w,f,n,f,w,b,g,e,?,s,k,w,w,p,w,t,p,w,s,g -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,n,o,p,o,v,l -p,x,s,e,f,f,f,c,n,b,t,?,k,s,w,p,p,w,o,e,w,v,p -e,k,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,n,v,l -p,k,y,e,f,f,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -p,k,s,n,f,f,f,c,n,b,t,?,k,s,p,p,p,w,o,e,w,v,d -p,k,y,e,f,f,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,p -p,k,y,e,f,y,f,c,n,b,t,?,s,s,p,p,p,w,o,e,w,v,p -p,x,s,n,f,y,f,c,n,b,t,?,k,k,w,w,p,w,o,e,w,v,d -e,b,s,g,f,n,f,w,b,g,e,?,k,s,w,w,p,w,t,p,w,n,g -p,x,y,c,f,m,f,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,k,f,w,f,n,f,w,b,w,e,?,k,s,w,w,p,w,t,p,w,n,g -p,k,y,n,f,s,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,k,k,w,p,p,w,o,e,w,v,d -e,k,f,w,f,n,f,w,b,w,e,?,k,k,w,w,p,w,t,p,w,s,g -e,f,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,v,l -p,k,s,e,f,s,f,c,n,b,t,?,s,s,p,w,p,w,o,e,w,v,p -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,n,c,l -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,o,c,l -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,v,l -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,y,v,l -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,v,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,n,c,l -p,k,y,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,l -e,b,s,w,f,n,f,w,b,w,e,?,s,s,w,w,p,w,t,p,w,n,g -e,x,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,o,o,p,n,v,l -e,k,s,w,f,n,f,w,b,p,e,?,s,s,w,w,p,w,t,p,w,n,g -e,k,s,n,f,n,a,c,b,o,e,?,s,s,o,o,p,n,o,p,b,v,l -p,k,y,e,f,y,f,c,n,b,t,?,k,k,p,p,p,w,o,e,w,v,d -p,f,y,c,f,m,a,c,b,y,e,c,k,y,c,c,p,w,n,n,w,c,d -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,v,l -p,k,y,n,f,s,f,c,n,b,t,?,s,k,p,w,p,w,o,e,w,v,l -p,k,s,e,f,y,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -p,k,y,n,f,f,f,c,n,b,t,?,k,s,p,w,p,w,o,e,w,v,d -e,k,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,b,c,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,n,o,p,b,v,l -e,f,s,n,f,n,a,c,b,n,e,?,s,s,o,o,p,o,o,p,b,c,l -p,k,y,n,f,y,f,c,n,b,t,?,s,k,w,w,p,w,o,e,w,v,l -e,x,s,n,f,n,a,c,b,y,e,?,s,s,o,o,p,o,o,p,o,c,l diff --git a/samples/cpp/bagofwords_classification.cpp b/samples/cpp/bagofwords_classification.cpp index ef4f3c73c6824e0d110e8de56cd346a4ae735809..1c50a0ec88f252ad96b27e74c8aebc50160555f2 100644 --- a/samples/cpp/bagofwords_classification.cpp +++ b/samples/cpp/bagofwords_classification.cpp @@ -23,6 +23,7 @@ #define DEBUG_DESC_PROGRESS using namespace cv; +using namespace cv::ml; using namespace std; const string paramsFile = "params.xml"; @@ -677,7 +678,7 @@ void VocData::writeClassifierResultsFile( const string& out_dir, const string& o result_file.close(); } else { string err_msg = "could not open classifier results file '" + output_file + "' for writing. Before running for the first time, a 'results' subdirectory should be created within the VOC dataset base directory. e.g. if the VOC data is stored in /VOC/VOC2010 then the path /VOC/results must be created."; - CV_Error(CV_StsError,err_msg.c_str()); + CV_Error(Error::StsError,err_msg.c_str()); } } @@ -701,9 +702,9 @@ void VocData::writeClassifierResultsFile( const string& out_dir, const string& o string VocData::getResultsFilename(const string& obj_class, const VocTask task, const ObdDatasetType dataset, const int competition, const int number) { if ((competition < 1) && (competition != -1)) - CV_Error(CV_StsBadArg,"competition argument should be a positive non-zero number or -1 to accept the default"); + CV_Error(Error::StsBadArg,"competition argument should be a positive non-zero number or -1 to accept the default"); if ((number < 1) && (number != -1)) - CV_Error(CV_StsBadArg,"number argument should be a positive non-zero number or -1 to accept the default"); + CV_Error(Error::StsBadArg,"number argument should be a positive non-zero number or -1 to accept the default"); string dset, task_type; @@ -815,7 +816,7 @@ void VocData::calcClassifierPrecRecall(const string& input_file, vector& scoregt_file.close(); } else { string err_msg = "could not open scoregt file '" + scoregt_file_str + "' for writing."; - CV_Error(CV_StsError,err_msg.c_str()); + CV_Error(Error::StsError,err_msg.c_str()); } } @@ -974,7 +975,7 @@ void VocData::calcClassifierConfMatRow(const string& obj_class, const vector& if (!gtfile.is_open()) { string err_msg = "could not open VOC ground truth textfile '" + filename + "'."; - CV_Error(CV_StsError,err_msg.c_str()); + CV_Error(Error::StsError,err_msg.c_str()); } string line; @@ -1462,7 +1463,7 @@ void VocData::readClassifierGroundTruth(const string& filename, vector& image_codes.push_back(image); object_present.push_back(obj_present == 1); } else { - if (!gtfile.eof()) CV_Error(CV_StsParseError,"error parsing VOC ground truth textfile."); + if (!gtfile.eof()) CV_Error(Error::StsParseError,"error parsing VOC ground truth textfile."); } } gtfile.close(); @@ -1488,13 +1489,13 @@ void VocData::readClassifierResultsFile(const string& input_file, vector image_codes.push_back(image); scores.push_back(score); } else { - if(!result_file.eof()) CV_Error(CV_StsParseError,"error parsing VOC classifier results file."); + if(!result_file.eof()) CV_Error(Error::StsParseError,"error parsing VOC classifier results file."); } } result_file.close(); } else { string err_msg = "could not open classifier results file '" + input_file + "' for reading."; - CV_Error(CV_StsError,err_msg.c_str()); + CV_Error(Error::StsError,err_msg.c_str()); } } @@ -1545,13 +1546,13 @@ void VocData::readDetectorResultsFile(const string& input_file, vector& bounding_boxes[image_idx].push_back(bounding_box); } } else { - if(!result_file.eof()) CV_Error(CV_StsParseError,"error parsing VOC detector results file."); + if(!result_file.eof()) CV_Error(Error::StsParseError,"error parsing VOC detector results file."); } } result_file.close(); } else { string err_msg = "could not open detector results file '" + input_file + "' for reading."; - CV_Error(CV_StsError,err_msg.c_str()); + CV_Error(Error::StsError,err_msg.c_str()); } } @@ -1595,23 +1596,23 @@ void VocData::extractVocObjects(const string filename, vector& object //object class ------------- - if (extractXMLBlock(object_contents, "name", 0, tag_contents) == -1) CV_Error(CV_StsError,"missing tag in object definition of '" + filename + "'"); + if (extractXMLBlock(object_contents, "name", 0, tag_contents) == -1) CV_Error(Error::StsError,"missing tag in object definition of '" + filename + "'"); object.object_class.swap(tag_contents); //object bounding box ------------- int xmax, xmin, ymax, ymin; - if (extractXMLBlock(object_contents, "xmax", 0, tag_contents) == -1) CV_Error(CV_StsError,"missing tag in object definition of '" + filename + "'"); + if (extractXMLBlock(object_contents, "xmax", 0, tag_contents) == -1) CV_Error(Error::StsError,"missing tag in object definition of '" + filename + "'"); xmax = stringToInteger(tag_contents); - if (extractXMLBlock(object_contents, "xmin", 0, tag_contents) == -1) CV_Error(CV_StsError,"missing tag in object definition of '" + filename + "'"); + if (extractXMLBlock(object_contents, "xmin", 0, tag_contents) == -1) CV_Error(Error::StsError,"missing tag in object definition of '" + filename + "'"); xmin = stringToInteger(tag_contents); - if (extractXMLBlock(object_contents, "ymax", 0, tag_contents) == -1) CV_Error(CV_StsError,"missing tag in object definition of '" + filename + "'"); + if (extractXMLBlock(object_contents, "ymax", 0, tag_contents) == -1) CV_Error(Error::StsError,"missing tag in object definition of '" + filename + "'"); ymax = stringToInteger(tag_contents); - if (extractXMLBlock(object_contents, "ymin", 0, tag_contents) == -1) CV_Error(CV_StsError,"missing tag in object definition of '" + filename + "'"); + if (extractXMLBlock(object_contents, "ymin", 0, tag_contents) == -1) CV_Error(Error::StsError,"missing tag in object definition of '" + filename + "'"); ymin = stringToInteger(tag_contents); object.boundingBox.x = xmin-1; //convert to 0-based indexing @@ -1714,11 +1715,11 @@ void VocData::extractDataFromResultsFilename(const string& input_file, string& c size_t fnameend = input_file_std.rfind(".txt"); if ((fnamestart == input_file_std.npos) || (fnameend == input_file_std.npos)) - CV_Error(CV_StsError,"Could not extract filename of results file."); + CV_Error(Error::StsError,"Could not extract filename of results file."); ++fnamestart; if (fnamestart >= fnameend) - CV_Error(CV_StsError,"Could not extract filename of results file."); + CV_Error(Error::StsError,"Could not extract filename of results file."); //extract dataset and class names, triggering exception if the filename format is not correct string filename = input_file_std.substr(fnamestart, fnameend-fnamestart); @@ -1729,11 +1730,11 @@ void VocData::extractDataFromResultsFilename(const string& input_file, string& c size_t classend = filename.find("_",classstart+1); if (classend == filename.npos) classend = filename.size(); if ((datasetstart == filename.npos) || (classstart == filename.npos)) - CV_Error(CV_StsError,"Error parsing results filename. Is it in standard format of 'comp_{cls/det}__.txt'?"); + CV_Error(Error::StsError,"Error parsing results filename. Is it in standard format of 'comp_{cls/det}__.txt'?"); ++datasetstart; ++classstart; if (((datasetstart-classstart) < 1) || ((classend-datasetstart) < 1)) - CV_Error(CV_StsError,"Error parsing results filename. Is it in standard format of 'comp_{cls/det}__.txt'?"); + CV_Error(Error::StsError,"Error parsing results filename. Is it in standard format of 'comp_{cls/det}__.txt'?"); dataset_name = filename.substr(datasetstart,classstart-datasetstart-1); class_name = filename.substr(classstart,classend-classstart); @@ -1781,7 +1782,7 @@ bool VocData::getClassifierGroundTruthImage(const string& obj_class, const strin return m_classifier_gt_all_present[std::distance(m_classifier_gt_all_ids.begin(),it)] != 0; } else { string err_msg = "could not find classifier ground truth for image '" + id + "' and class '" + obj_class + "'"; - CV_Error(CV_StsError,err_msg.c_str()); + CV_Error(Error::StsError,err_msg.c_str()); } return true; @@ -1814,7 +1815,7 @@ void VocData::getSortOrder(const vector& values, vector& order, b void VocData::readFileToString(const string filename, string& file_contents) { std::ifstream ifs(filename.c_str()); - if (!ifs.is_open()) CV_Error(CV_StsError,"could not open text file"); + if (!ifs.is_open()) CV_Error(Error::StsError,"could not open text file"); stringstream oss; oss << ifs.rdbuf(); @@ -1829,7 +1830,7 @@ int VocData::stringToInteger(const string input_str) stringstream ss(input_str); if ((ss >> result).fail()) { - CV_Error(CV_StsBadArg,"could not perform string to integer conversion"); + CV_Error(Error::StsBadArg,"could not perform string to integer conversion"); } return result; } @@ -1841,7 +1842,7 @@ string VocData::integerToString(const int input_int) stringstream ss; if ((ss << input_int).fail()) { - CV_Error(CV_StsBadArg,"could not perform integer to string conversion"); + CV_Error(Error::StsBadArg,"could not perform integer to string conversion"); } result = ss.str(); return result; @@ -2325,14 +2326,14 @@ static void removeBowImageDescriptorsByCount( vector& images, vector(1) = static_cast(pos_ex)/static_cast(pos_ex+neg_ex); } class_wts_cv = class_wts; - svmParams.class_weights = &class_wts_cv; + svmParams.classWeights = class_wts_cv; } } -static void setSVMTrainAutoParams( CvParamGrid& c_grid, CvParamGrid& gamma_grid, - CvParamGrid& p_grid, CvParamGrid& nu_grid, - CvParamGrid& coef_grid, CvParamGrid& degree_grid ) +static void setSVMTrainAutoParams( ParamGrid& c_grid, ParamGrid& gamma_grid, + ParamGrid& p_grid, ParamGrid& nu_grid, + ParamGrid& coef_grid, ParamGrid& degree_grid ) { - c_grid = CvSVM::get_default_grid(CvSVM::C); + c_grid = SVM::getDefaultGrid(SVM::C); - gamma_grid = CvSVM::get_default_grid(CvSVM::GAMMA); + gamma_grid = SVM::getDefaultGrid(SVM::GAMMA); - p_grid = CvSVM::get_default_grid(CvSVM::P); - p_grid.step = 0; + p_grid = SVM::getDefaultGrid(SVM::P); + p_grid.logStep = 0; - nu_grid = CvSVM::get_default_grid(CvSVM::NU); - nu_grid.step = 0; + nu_grid = SVM::getDefaultGrid(SVM::NU); + nu_grid.logStep = 0; - coef_grid = CvSVM::get_default_grid(CvSVM::COEF); - coef_grid.step = 0; + coef_grid = SVM::getDefaultGrid(SVM::COEF); + coef_grid.logStep = 0; - degree_grid = CvSVM::get_default_grid(CvSVM::DEGREE); - degree_grid.step = 0; + degree_grid = SVM::getDefaultGrid(SVM::DEGREE); + degree_grid.logStep = 0; } -static void trainSVMClassifier( CvSVM& svm, const SVMTrainParamsExt& svmParamsExt, const string& objClassName, VocData& vocData, +static Ptr trainSVMClassifier( const SVMTrainParamsExt& svmParamsExt, const string& objClassName, VocData& vocData, Ptr& bowExtractor, const Ptr& fdetector, const string& resPath ) { /* first check if a previously trained svm for the current class has been saved to file */ string svmFilename = resPath + svmsDir + "/" + objClassName + ".xml.gz"; + Ptr svm; FileStorage fs( svmFilename, FileStorage::READ); if( fs.isOpened() ) { cout << "*** LOADING SVM CLASSIFIER FOR CLASS " << objClassName << " ***" << endl; - svm.load( svmFilename.c_str() ); + svm = StatModel::load( svmFilename ); } else { @@ -2437,20 +2439,24 @@ static void trainSVMClassifier( CvSVM& svm, const SVMTrainParamsExt& svmParamsEx } cout << "TRAINING SVM FOR CLASS ..." << objClassName << "..." << endl; - CvSVMParams svmParams; - CvMat class_wts_cv; + SVM::Params svmParams; + Mat class_wts_cv; setSVMParams( svmParams, class_wts_cv, responses, svmParamsExt.balanceClasses ); - CvParamGrid c_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid; + svm = SVM::create(svmParams); + ParamGrid c_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid; setSVMTrainAutoParams( c_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid ); - svm.train_auto( trainData, responses, Mat(), Mat(), svmParams, 10, c_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid ); + + svm->trainAuto(TrainData::create(trainData, ROW_SAMPLE, responses), 10, + c_grid, gamma_grid, p_grid, nu_grid, coef_grid, degree_grid); cout << "SVM TRAINING FOR CLASS " << objClassName << " COMPLETED" << endl; - svm.save( svmFilename.c_str() ); + svm->save( svmFilename ); cout << "SAVED CLASSIFIER TO FILE" << endl; } + return svm; } -static void computeConfidences( CvSVM& svm, const string& objClassName, VocData& vocData, +static void computeConfidences( const Ptr& svm, const string& objClassName, VocData& vocData, Ptr& bowExtractor, const Ptr& fdetector, const string& resPath ) { @@ -2476,12 +2482,12 @@ static void computeConfidences( CvSVM& svm, const string& objClassName, VocData& if( imageIdx == 0 ) { // In the first iteration, determine the sign of the positive class - float classVal = confidences[imageIdx] = svm.predict( bowImageDescriptors[imageIdx], false ); - float scoreVal = confidences[imageIdx] = svm.predict( bowImageDescriptors[imageIdx], true ); + float classVal = confidences[imageIdx] = svm->predict( bowImageDescriptors[imageIdx], noArray(), 0 ); + float scoreVal = confidences[imageIdx] = svm->predict( bowImageDescriptors[imageIdx], noArray(), StatModel::RAW_OUTPUT ); signMul = (classVal < 0) == (scoreVal < 0) ? 1.f : -1.f; } // svm output of decision function - confidences[imageIdx] = signMul * svm.predict( bowImageDescriptors[imageIdx], true ); + confidences[imageIdx] = signMul * svm->predict( bowImageDescriptors[imageIdx], noArray(), StatModel::RAW_OUTPUT ); } cout << "WRITING QUERY RESULTS TO VOC RESULTS FILE FOR CLASS " << objClassName << "..." << endl; @@ -2591,9 +2597,8 @@ int main(int argc, char** argv) for( size_t classIdx = 0; classIdx < objClasses.size(); ++classIdx ) { // Train a classifier on train dataset - CvSVM svm; - trainSVMClassifier( svm, svmTrainParamsExt, objClasses[classIdx], vocData, - bowExtractor, featureDetector, resPath ); + Ptr svm = trainSVMClassifier( svmTrainParamsExt, objClasses[classIdx], vocData, + bowExtractor, featureDetector, resPath ); // Now use the classifier over all images on the test dataset and rank according to score order // also calculating precision-recall etc. diff --git a/samples/cpp/em.cpp b/samples/cpp/em.cpp index a078588c29d29dd6a566b4f7c18b4945fa6fdb67..be792a904df49c849859a0d26dc53fdf677cb80c 100644 --- a/samples/cpp/em.cpp +++ b/samples/cpp/em.cpp @@ -2,6 +2,7 @@ #include "opencv2/ml.hpp" using namespace cv; +using namespace cv::ml; int main( int /*argc*/, char** /*argv*/ ) { @@ -34,8 +35,9 @@ int main( int /*argc*/, char** /*argv*/ ) samples = samples.reshape(1, 0); // cluster the data - EM em_model(N, EM::COV_MAT_SPHERICAL, TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 300, 0.1)); - em_model.train( samples, noArray(), labels, noArray() ); + Ptr em_model = EM::train( samples, noArray(), labels, noArray(), + EM::Params(N, EM::COV_MAT_SPHERICAL, + TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 300, 0.1))); // classify every image pixel for( i = 0; i < img.rows; i++ ) @@ -44,7 +46,7 @@ int main( int /*argc*/, char** /*argv*/ ) { sample.at(0) = (float)j; sample.at(1) = (float)i; - int response = cvRound(em_model.predict( sample )[1]); + int response = cvRound(em_model->predict2( sample, noArray() )[1]); Scalar c = colors[response]; circle( img, Point(j, i), 1, c*0.75, FILLED ); diff --git a/samples/cpp/letter_recog.cpp b/samples/cpp/letter_recog.cpp index ddbe676291da09f2dac5d51ad3e05fd7923a2983..b6a35e338f74fdc552062c5270f4e7c21748132c 100644 --- a/samples/cpp/letter_recog.cpp +++ b/samples/cpp/letter_recog.cpp @@ -1,11 +1,13 @@ -#include "opencv2/core/core_c.h" +#include "opencv2/core/core.hpp" #include "opencv2/ml/ml.hpp" #include #include -/* +#include -*/ +using namespace std; +using namespace cv; +using namespace cv::ml; static void help() { @@ -33,142 +35,101 @@ static void help() } // This function reads data and responses from the file -static int -read_num_class_data( const char* filename, int var_count, - CvMat** data, CvMat** responses ) +static bool +read_num_class_data( const string& filename, int var_count, + Mat* _data, Mat* _responses ) { const int M = 1024; - FILE* f = fopen( filename, "rt" ); - CvMemStorage* storage; - CvSeq* seq; char buf[M+2]; - float* el_ptr; - CvSeqReader reader; - int i, j; - if( !f ) - return 0; + Mat el_ptr(1, var_count, CV_32F); + int i; + vector responses; + + _data->release(); + _responses->release(); - el_ptr = new float[var_count+1]; - storage = cvCreateMemStorage(); - seq = cvCreateSeq( 0, sizeof(*seq), (var_count+1)*sizeof(float), storage ); + FILE* f = fopen( filename.c_str(), "rt" ); + if( !f ) + { + cout << "Could not read the database " << filename << endl; + return false; + } for(;;) { char* ptr; if( !fgets( buf, M, f ) || !strchr( buf, ',' ) ) break; - el_ptr[0] = buf[0]; + responses.push_back((int)buf[0]); ptr = buf+2; - for( i = 1; i <= var_count; i++ ) + for( i = 0; i < var_count; i++ ) { int n = 0; - sscanf( ptr, "%f%n", el_ptr + i, &n ); + sscanf( ptr, "%f%n", &el_ptr.at(i), &n ); ptr += n + 1; } - if( i <= var_count ) + if( i < var_count ) break; - cvSeqPush( seq, el_ptr ); + _data->push_back(el_ptr); } fclose(f); + Mat(responses).copyTo(*_responses); - *data = cvCreateMat( seq->total, var_count, CV_32F ); - *responses = cvCreateMat( seq->total, 1, CV_32F ); - - cvStartReadSeq( seq, &reader ); - - for( i = 0; i < seq->total; i++ ) - { - const float* sdata = (float*)reader.ptr + 1; - float* ddata = data[0]->data.fl + var_count*i; - float* dr = responses[0]->data.fl + i; - - for( j = 0; j < var_count; j++ ) - ddata[j] = sdata[j]; - *dr = sdata[-1]; - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - } + cout << "The database " << filename << " is loaded.\n"; - cvReleaseMemStorage( &storage ); - delete[] el_ptr; - return 1; + return true; } -static -int build_rtrees_classifier( char* data_filename, - char* filename_to_save, char* filename_to_load ) +template +static Ptr load_classifier(const string& filename_to_load) { - CvMat* data = 0; - CvMat* responses = 0; - CvMat* var_type = 0; - CvMat* sample_idx = 0; - - int ok = read_num_class_data( data_filename, 16, &data, &responses ); - int nsamples_all = 0, ntrain_samples = 0; - int i = 0; - double train_hr = 0, test_hr = 0; - CvRTrees forest; - CvMat* var_importance = 0; - - if( !ok ) - { - printf( "Could not read the database %s\n", data_filename ); - return -1; - } + // load classifier from the specified file + Ptr model = StatModel::load( filename_to_load ); + if( model.empty() ) + cout << "Could not read the classifier " << filename_to_load << endl; + else + cout << "The classifier " << filename_to_load << " is loaded.\n"; - printf( "The database %s is loaded.\n", data_filename ); - nsamples_all = data->rows; - ntrain_samples = (int)(nsamples_all*0.8); + return model; +} - // Create or load Random Trees classifier - if( filename_to_load ) - { - // load classifier from the specified file - forest.load( filename_to_load ); - ntrain_samples = 0; - if( forest.get_tree_count() == 0 ) - { - printf( "Could not read the classifier %s\n", filename_to_load ); - return -1; - } - printf( "The classifier %s is loaded.\n", filename_to_load ); - } - else - { - // create classifier by using and - printf( "Training the classifier ...\n"); +static Ptr +prepare_train_data(const Mat& data, const Mat& responses, int ntrain_samples) +{ + Mat sample_idx = Mat::zeros( 1, data.rows, CV_8U ); + Mat train_samples = sample_idx.colRange(0, ntrain_samples); + train_samples.setTo(Scalar::all(1)); - // 1. create type mask - var_type = cvCreateMat( data->cols + 1, 1, CV_8U ); - cvSet( var_type, cvScalarAll(CV_VAR_ORDERED) ); - cvSetReal1D( var_type, data->cols, CV_VAR_CATEGORICAL ); + int nvars = data.cols; + Mat var_type( nvars + 1, 1, CV_8U ); + var_type.setTo(Scalar::all(VAR_ORDERED)); + var_type.at(nvars) = VAR_CATEGORICAL; - // 2. create sample_idx - sample_idx = cvCreateMat( 1, nsamples_all, CV_8UC1 ); - { - CvMat mat; - cvGetCols( sample_idx, &mat, 0, ntrain_samples ); - cvSet( &mat, cvRealScalar(1) ); + return TrainData::create(data, ROW_SAMPLE, responses, + noArray(), sample_idx, noArray(), var_type); +} - cvGetCols( sample_idx, &mat, ntrain_samples, nsamples_all ); - cvSetZero( &mat ); - } +inline TermCriteria TC(int iters, double eps) +{ + return TermCriteria(TermCriteria::MAX_ITER + (eps > 0 ? TermCriteria::EPS : 0), iters, eps); +} - // 3. train classifier - forest.train( data, CV_ROW_SAMPLE, responses, 0, sample_idx, var_type, 0, - CvRTParams(10,10,0,false,15,0,true,4,100,0.01f,CV_TERMCRIT_ITER)); - printf( "\n"); - } +static void test_and_save_classifier(const Ptr& model, + const Mat& data, const Mat& responses, + int ntrain_samples, int rdelta, + const string& filename_to_save) +{ + int i, nsamples_all = data.rows; + double train_hr = 0, test_hr = 0; // compute prediction error on train and test data for( i = 0; i < nsamples_all; i++ ) { - double r; - CvMat sample; - cvGetRow( data, &sample, i ); + Mat sample = data.row(i); - r = forest.predict( &sample ); - r = fabs((double)r - responses->data.fl[i]) <= FLT_EPSILON ? 1 : 0; + float r = model->predict( sample ); + r = std::abs(r + rdelta - responses.at(i)) <= FLT_EPSILON ? 1.f : 0.f; if( i < ntrain_samples ) train_hr += r; @@ -176,93 +137,98 @@ int build_rtrees_classifier( char* data_filename, test_hr += r; } - test_hr /= (double)(nsamples_all-ntrain_samples); - train_hr /= (double)ntrain_samples; + test_hr /= nsamples_all - ntrain_samples; + train_hr = ntrain_samples > 0 ? train_hr/ntrain_samples : 1.; + printf( "Recognition rate: train = %.1f%%, test = %.1f%%\n", train_hr*100., test_hr*100. ); - printf( "Number of trees: %d\n", forest.get_tree_count() ); - - // Print variable importance - var_importance = (CvMat*)forest.get_var_importance(); - if( var_importance ) + if( !filename_to_save.empty() ) { - double rt_imp_sum = cvSum( var_importance ).val[0]; - printf("var#\timportance (in %%):\n"); - for( i = 0; i < var_importance->cols; i++ ) - printf( "%-2d\t%-4.1f\n", i, - 100.f*var_importance->data.fl[i]/rt_imp_sum); + model->save( filename_to_save ); } +} - //Print some proximitites - printf( "Proximities between some samples corresponding to the letter 'T':\n" ); - { - CvMat sample1, sample2; - const int pairs[][2] = {{0,103}, {0,106}, {106,103}, {-1,-1}}; - for( i = 0; pairs[i][0] >= 0; i++ ) - { - cvGetRow( data, &sample1, pairs[i][0] ); - cvGetRow( data, &sample2, pairs[i][1] ); - printf( "proximity(%d,%d) = %.1f%%\n", pairs[i][0], pairs[i][1], - forest.get_proximity( &sample1, &sample2 )*100. ); - } +static bool +build_rtrees_classifier( const string& data_filename, + const string& filename_to_save, + const string& filename_to_load ) +{ + Mat data; + Mat responses; + bool ok = read_num_class_data( data_filename, 16, &data, &responses ); + if( !ok ) + return ok; + + Ptr model; + + int nsamples_all = data.rows; + int ntrain_samples = (int)(nsamples_all*0.8); + + // Create or load Random Trees classifier + if( !filename_to_load.empty() ) + { + model = load_classifier(filename_to_load); + if( model.empty() ) + return false; + ntrain_samples = 0; + } + else + { + // create classifier by using and + cout << "Training the classifier ...\n"; + Ptr tdata = prepare_train_data(data, responses, ntrain_samples); + model = StatModel::train(tdata, RTrees::Params(10,10,0,false,15,Mat(),true,4,TC(100,0.01f))); + cout << endl; } - // Save Random Trees classifier to file if needed - if( filename_to_save ) - forest.save( filename_to_save ); + test_and_save_classifier(model, data, responses, ntrain_samples, 0, filename_to_save); + cout << "Number of trees: " << model->getRoots().size() << endl; - cvReleaseMat( &sample_idx ); - cvReleaseMat( &var_type ); - cvReleaseMat( &data ); - cvReleaseMat( &responses ); + // Print variable importance + Mat var_importance = model->getVarImportance(); + if( !var_importance.empty() ) + { + double rt_imp_sum = sum( var_importance )[0]; + printf("var#\timportance (in %%):\n"); + int i, n = (int)var_importance.total(); + for( i = 0; i < n; i++ ) + printf( "%-2d\t%-4.1f\n", i, 100.f*var_importance.at(i)/rt_imp_sum); + } - return 0; + return true; } -static -int build_boost_classifier( char* data_filename, - char* filename_to_save, char* filename_to_load ) +static bool +build_boost_classifier( const string& data_filename, + const string& filename_to_save, + const string& filename_to_load ) { const int class_count = 26; - CvMat* data = 0; - CvMat* responses = 0; - CvMat* var_type = 0; - CvMat* temp_sample = 0; - CvMat* weak_responses = 0; - - int ok = read_num_class_data( data_filename, 16, &data, &responses ); - int nsamples_all = 0, ntrain_samples = 0; - int var_count; - int i, j, k; - double train_hr = 0, test_hr = 0; - CvBoost boost; + Mat data; + Mat responses; + Mat weak_responses; + bool ok = read_num_class_data( data_filename, 16, &data, &responses ); if( !ok ) - { - printf( "Could not read the database %s\n", data_filename ); - return -1; - } + return ok; + + int i, j, k; + Ptr model; - printf( "The database %s is loaded.\n", data_filename ); - nsamples_all = data->rows; - ntrain_samples = (int)(nsamples_all*0.5); - var_count = data->cols; + int nsamples_all = data.rows; + int ntrain_samples = (int)(nsamples_all*0.5); + int var_count = data.cols; // Create or load Boosted Tree classifier - if( filename_to_load ) + if( !filename_to_load.empty() ) { - // load classifier from the specified file - boost.load( filename_to_load ); + model = load_classifier(filename_to_load); + if( model.empty() ) + return false; ntrain_samples = 0; - if( !boost.get_weak_predictors() ) - { - printf( "Could not read the classifier %s\n", filename_to_load ); - return -1; - } - printf( "The classifier %s is loaded.\n", filename_to_load ); } else { @@ -275,135 +241,109 @@ int build_boost_classifier( char* data_filename, // // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - CvMat* new_data = cvCreateMat( ntrain_samples*class_count, var_count + 1, CV_32F ); - CvMat* new_responses = cvCreateMat( ntrain_samples*class_count, 1, CV_32S ); + Mat new_data( ntrain_samples*class_count, var_count + 1, CV_32F ); + Mat new_responses( ntrain_samples*class_count, 1, CV_32S ); // 1. unroll the database type mask printf( "Unrolling the database...\n"); for( i = 0; i < ntrain_samples; i++ ) { - float* data_row = (float*)(data->data.ptr + data->step*i); + const float* data_row = data.ptr(i); for( j = 0; j < class_count; j++ ) { - float* new_data_row = (float*)(new_data->data.ptr + - new_data->step*(i*class_count+j)); - for( k = 0; k < var_count; k++ ) - new_data_row[k] = data_row[k]; + float* new_data_row = (float*)new_data.ptr(i*class_count+j); + memcpy(new_data_row, data_row, var_count*sizeof(data_row[0])); new_data_row[var_count] = (float)j; - new_responses->data.i[i*class_count + j] = responses->data.fl[i] == j+'A'; + new_responses.at(i*class_count + j) = responses.at(i) == j+'A'; } } - // 2. create type mask - var_type = cvCreateMat( var_count + 2, 1, CV_8U ); - cvSet( var_type, cvScalarAll(CV_VAR_ORDERED) ); - // the last indicator variable, as well - // as the new (binary) response are categorical - cvSetReal1D( var_type, var_count, CV_VAR_CATEGORICAL ); - cvSetReal1D( var_type, var_count+1, CV_VAR_CATEGORICAL ); - - // 3. train classifier - printf( "Training the classifier (may take a few minutes)...\n"); - boost.train( new_data, CV_ROW_SAMPLE, new_responses, 0, 0, var_type, 0, - CvBoostParams(CvBoost::REAL, 100, 0.95, 5, false, 0 )); - cvReleaseMat( &new_data ); - cvReleaseMat( &new_responses ); - printf("\n"); + Mat var_type( 1, var_count + 2, CV_8U ); + var_type.setTo(Scalar::all(VAR_ORDERED)); + var_type.at(var_count) = var_type.at(var_count+1) = VAR_CATEGORICAL; + + Ptr tdata = TrainData::create(new_data, ROW_SAMPLE, new_responses, + noArray(), noArray(), noArray(), var_type); + vector priors(2); + priors[0] = 1; + priors[1] = 26; + + cout << "Training the classifier (may take a few minutes)...\n"; + model = StatModel::train(tdata, Boost::Params(Boost::GENTLE, 100, 0.95, 5, false, Mat(priors) )); + cout << endl; } - temp_sample = cvCreateMat( 1, var_count + 1, CV_32F ); - weak_responses = cvCreateMat( 1, boost.get_weak_predictors()->total, CV_32F ); + Mat temp_sample( 1, var_count + 1, CV_32F ); + float* tptr = temp_sample.ptr(); // compute prediction error on train and test data + double train_hr = 0, test_hr = 0; for( i = 0; i < nsamples_all; i++ ) { int best_class = 0; double max_sum = -DBL_MAX; - double r; - CvMat sample; - cvGetRow( data, &sample, i ); + const float* ptr = data.ptr(i); for( k = 0; k < var_count; k++ ) - temp_sample->data.fl[k] = sample.data.fl[k]; + tptr[k] = ptr[k]; for( j = 0; j < class_count; j++ ) { - temp_sample->data.fl[var_count] = (float)j; - boost.predict( temp_sample, 0, weak_responses ); - double sum = cvSum( weak_responses ).val[0]; - if( max_sum < sum ) + tptr[var_count] = (float)j; + float s = model->predict( temp_sample, noArray(), StatModel::RAW_OUTPUT ); + if( max_sum < s ) { - max_sum = sum; + max_sum = s; best_class = j + 'A'; } } - r = fabs(best_class - responses->data.fl[i]) < FLT_EPSILON ? 1 : 0; - + double r = std::abs(best_class - responses.at(i)) < FLT_EPSILON ? 1 : 0; if( i < ntrain_samples ) train_hr += r; else test_hr += r; } - test_hr /= (double)(nsamples_all-ntrain_samples); - train_hr /= (double)ntrain_samples; + test_hr /= nsamples_all-ntrain_samples; + train_hr = ntrain_samples > 0 ? train_hr/ntrain_samples : 1.; printf( "Recognition rate: train = %.1f%%, test = %.1f%%\n", train_hr*100., test_hr*100. ); - printf( "Number of trees: %d\n", boost.get_weak_predictors()->total ); + cout << "Number of trees: " << model->getRoots().size() << endl; // Save classifier to file if needed - if( filename_to_save ) - boost.save( filename_to_save ); - - cvReleaseMat( &temp_sample ); - cvReleaseMat( &weak_responses ); - cvReleaseMat( &var_type ); - cvReleaseMat( &data ); - cvReleaseMat( &responses ); + if( !filename_to_save.empty() ) + model->save( filename_to_save ); - return 0; + return true; } -static -int build_mlp_classifier( char* data_filename, - char* filename_to_save, char* filename_to_load ) +static bool +build_mlp_classifier( const string& data_filename, + const string& filename_to_save, + const string& filename_to_load ) { const int class_count = 26; - CvMat* data = 0; - CvMat train_data; - CvMat* responses = 0; - CvMat* mlp_response = 0; - - int ok = read_num_class_data( data_filename, 16, &data, &responses ); - int nsamples_all = 0, ntrain_samples = 0; - int i, j; - double train_hr = 0, test_hr = 0; - CvANN_MLP mlp; + Mat data; + Mat responses; + bool ok = read_num_class_data( data_filename, 16, &data, &responses ); if( !ok ) - { - printf( "Could not read the database %s\n", data_filename ); - return -1; - } + return ok; + + Ptr model; - printf( "The database %s is loaded.\n", data_filename ); - nsamples_all = data->rows; - ntrain_samples = (int)(nsamples_all*0.8); + int nsamples_all = data.rows; + int ntrain_samples = (int)(nsamples_all*0.8); // Create or load MLP classifier - if( filename_to_load ) + if( !filename_to_load.empty() ) { - // load classifier from the specified file - mlp.load( filename_to_load ); + model = load_classifier(filename_to_load); + if( model.empty() ) + return false; ntrain_samples = 0; - if( !mlp.get_layer_count() ) - { - printf( "Could not read the classifier %s\n", filename_to_load ); - return -1; - } - printf( "The classifier %s is loaded.\n", filename_to_load ); } else { @@ -417,328 +357,139 @@ int build_mlp_classifier( char* data_filename, // // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - CvMat* new_responses = cvCreateMat( ntrain_samples, class_count, CV_32F ); + Mat train_data = data.rowRange(0, ntrain_samples); + Mat train_responses = Mat::zeros( ntrain_samples, class_count, CV_32F ); // 1. unroll the responses - printf( "Unrolling the responses...\n"); - for( i = 0; i < ntrain_samples; i++ ) + cout << "Unrolling the responses...\n"; + for( int i = 0; i < ntrain_samples; i++ ) { - int cls_label = cvRound(responses->data.fl[i]) - 'A'; - float* bit_vec = (float*)(new_responses->data.ptr + i*new_responses->step); - for( j = 0; j < class_count; j++ ) - bit_vec[j] = 0.f; - bit_vec[cls_label] = 1.f; + int cls_label = responses.at(i) - 'A'; + train_responses.at(i, cls_label) = 1.f; } - cvGetRows( data, &train_data, 0, ntrain_samples ); // 2. train classifier - int layer_sz[] = { data->cols, 100, 100, class_count }; - CvMat layer_sizes = - cvMat( 1, (int)(sizeof(layer_sz)/sizeof(layer_sz[0])), CV_32S, layer_sz ); - mlp.create( &layer_sizes ); - printf( "Training the classifier (may take a few minutes)...\n"); + int layer_sz[] = { data.cols, 100, 100, class_count }; + int nlayers = (int)(sizeof(layer_sz)/sizeof(layer_sz[0])); + Mat layer_sizes( 1, nlayers, CV_32S, layer_sz ); #if 1 - int method = CvANN_MLP_TrainParams::BACKPROP; + int method = ANN_MLP::Params::BACKPROP; double method_param = 0.001; int max_iter = 300; #else - int method = CvANN_MLP_TrainParams::RPROP; + int method = ANN_MLP::Params::RPROP; double method_param = 0.1; int max_iter = 1000; #endif - mlp.train( &train_data, new_responses, 0, 0, - CvANN_MLP_TrainParams(cvTermCriteria(CV_TERMCRIT_ITER,max_iter,0.01), - method, method_param)); - cvReleaseMat( &new_responses ); - printf("\n"); - } - - mlp_response = cvCreateMat( 1, class_count, CV_32F ); - - // compute prediction error on train and test data - for( i = 0; i < nsamples_all; i++ ) - { - int best_class; - CvMat sample; - cvGetRow( data, &sample, i ); - CvPoint max_loc; - mlp.predict( &sample, mlp_response ); - cvMinMaxLoc( mlp_response, 0, 0, 0, &max_loc, 0 ); - best_class = max_loc.x + 'A'; + Ptr tdata = TrainData::create(train_data, ROW_SAMPLE, train_responses); - int r = fabs((double)best_class - responses->data.fl[i]) < FLT_EPSILON ? 1 : 0; - - if( i < ntrain_samples ) - train_hr += r; - else - test_hr += r; + cout << "Training the classifier (may take a few minutes)...\n"; + model = StatModel::train(tdata, ANN_MLP::Params(layer_sizes, ANN_MLP::SIGMOID_SYM, 0, 0, TC(max_iter,0), method, method_param)); + cout << endl; } - test_hr /= (double)(nsamples_all-ntrain_samples); - train_hr /= (double)ntrain_samples; - printf( "Recognition rate: train = %.1f%%, test = %.1f%%\n", - train_hr*100., test_hr*100. ); - - // Save classifier to file if needed - if( filename_to_save ) - mlp.save( filename_to_save ); - - cvReleaseMat( &mlp_response ); - cvReleaseMat( &data ); - cvReleaseMat( &responses ); - - return 0; + test_and_save_classifier(model, data, responses, ntrain_samples, 'A', filename_to_save); + return true; } -static -int build_knearest_classifier( char* data_filename, int K ) +static bool +build_knearest_classifier( const string& data_filename, int K ) { - const int var_count = 16; - CvMat* data = 0; - CvMat train_data; - CvMat* responses; - - int ok = read_num_class_data( data_filename, 16, &data, &responses ); - int nsamples_all = 0, ntrain_samples = 0; - //int i, j; - //double /*train_hr = 0,*/ test_hr = 0; - CvANN_MLP mlp; - + Mat data; + Mat responses; + bool ok = read_num_class_data( data_filename, 16, &data, &responses ); if( !ok ) - { - printf( "Could not read the database %s\n", data_filename ); - return -1; - } - - printf( "The database %s is loaded.\n", data_filename ); - nsamples_all = data->rows; - ntrain_samples = (int)(nsamples_all*0.8); - - // 1. unroll the responses - printf( "Unrolling the responses...\n"); - cvGetRows( data, &train_data, 0, ntrain_samples ); - - // 2. train classifier - CvMat* train_resp = cvCreateMat( ntrain_samples, 1, CV_32FC1); - for (int i = 0; i < ntrain_samples; i++) - train_resp->data.fl[i] = responses->data.fl[i]; - CvKNearest knearest(&train_data, train_resp); - - CvMat* nearests = cvCreateMat( (nsamples_all - ntrain_samples), K, CV_32FC1); - float* _sample = new float[var_count * (nsamples_all - ntrain_samples)]; - CvMat sample = cvMat( nsamples_all - ntrain_samples, 16, CV_32FC1, _sample ); - float* true_results = new float[nsamples_all - ntrain_samples]; - for (int j = ntrain_samples; j < nsamples_all; j++) - { - float *s = data->data.fl + j * var_count; + return ok; - for (int i = 0; i < var_count; i++) - { - sample.data.fl[(j - ntrain_samples) * var_count + i] = s[i]; - } - true_results[j - ntrain_samples] = responses->data.fl[j]; - } - CvMat *result = cvCreateMat(1, nsamples_all - ntrain_samples, CV_32FC1); - knearest.find_nearest(&sample, K, result, 0, nearests, 0); - int true_resp = 0; - int accuracy = 0; - for (int i = 0; i < nsamples_all - ntrain_samples; i++) - { - if (result->data.fl[i] == true_results[i]) - true_resp++; - for(int k = 0; k < K; k++ ) - { - if( nearests->data.fl[i * K + k] == true_results[i]) - accuracy++; - } - } + Ptr model; - printf("true_resp = %f%%\tavg accuracy = %f%%\n", (float)true_resp / (nsamples_all - ntrain_samples) * 100, - (float)accuracy / (nsamples_all - ntrain_samples) / K * 100); + int nsamples_all = data.rows; + int ntrain_samples = (int)(nsamples_all*0.8); - delete[] true_results; - delete[] _sample; - cvReleaseMat( &train_resp ); - cvReleaseMat( &nearests ); - cvReleaseMat( &result ); - cvReleaseMat( &data ); - cvReleaseMat( &responses ); + // create classifier by using and + cout << "Training the classifier ...\n"; + Ptr tdata = prepare_train_data(data, responses, ntrain_samples); + model = StatModel::train(tdata, KNearest::Params(K, true)); + cout << endl; - return 0; + test_and_save_classifier(model, data, responses, ntrain_samples, 0, string()); + return true; } -static -int build_nbayes_classifier( char* data_filename ) +static bool +build_nbayes_classifier( const string& data_filename ) { - const int var_count = 16; - CvMat* data = 0; - CvMat train_data; - CvMat* responses; - - int ok = read_num_class_data( data_filename, 16, &data, &responses ); - int nsamples_all = 0, ntrain_samples = 0; - //int i, j; - //double /*train_hr = 0, */test_hr = 0; - CvANN_MLP mlp; - + Mat data; + Mat responses; + bool ok = read_num_class_data( data_filename, 16, &data, &responses ); if( !ok ) - { - printf( "Could not read the database %s\n", data_filename ); - return -1; - } - - printf( "The database %s is loaded.\n", data_filename ); - nsamples_all = data->rows; - ntrain_samples = (int)(nsamples_all*0.5); - - // 1. unroll the responses - printf( "Unrolling the responses...\n"); - cvGetRows( data, &train_data, 0, ntrain_samples ); + return ok; - // 2. train classifier - CvMat* train_resp = cvCreateMat( ntrain_samples, 1, CV_32FC1); - for (int i = 0; i < ntrain_samples; i++) - train_resp->data.fl[i] = responses->data.fl[i]; - CvNormalBayesClassifier nbayes(&train_data, train_resp); + Ptr model; - float* _sample = new float[var_count * (nsamples_all - ntrain_samples)]; - CvMat sample = cvMat( nsamples_all - ntrain_samples, 16, CV_32FC1, _sample ); - float* true_results = new float[nsamples_all - ntrain_samples]; - for (int j = ntrain_samples; j < nsamples_all; j++) - { - float *s = data->data.fl + j * var_count; - - for (int i = 0; i < var_count; i++) - { - sample.data.fl[(j - ntrain_samples) * var_count + i] = s[i]; - } - true_results[j - ntrain_samples] = responses->data.fl[j]; - } - CvMat *result = cvCreateMat(1, nsamples_all - ntrain_samples, CV_32FC1); - nbayes.predict(&sample, result); - int true_resp = 0; - //int accuracy = 0; - for (int i = 0; i < nsamples_all - ntrain_samples; i++) - { - if (result->data.fl[i] == true_results[i]) - true_resp++; - } - - printf("true_resp = %f%%\n", (float)true_resp / (nsamples_all - ntrain_samples) * 100); + int nsamples_all = data.rows; + int ntrain_samples = (int)(nsamples_all*0.8); - delete[] true_results; - delete[] _sample; - cvReleaseMat( &train_resp ); - cvReleaseMat( &result ); - cvReleaseMat( &data ); - cvReleaseMat( &responses ); + // create classifier by using and + cout << "Training the classifier ...\n"; + Ptr tdata = prepare_train_data(data, responses, ntrain_samples); + model = StatModel::train(tdata, NormalBayesClassifier::Params()); + cout << endl; - return 0; + test_and_save_classifier(model, data, responses, ntrain_samples, 0, string()); + return true; } -static -int build_svm_classifier( char* data_filename, const char* filename_to_save, const char* filename_to_load ) +static bool +build_svm_classifier( const string& data_filename, + const string& filename_to_save, + const string& filename_to_load ) { - CvMat* data = 0; - CvMat* responses = 0; - CvMat* train_resp = 0; - CvMat train_data; - int nsamples_all = 0, ntrain_samples = 0; - int var_count; - CvSVM svm; - - int ok = read_num_class_data( data_filename, 16, &data, &responses ); + Mat data; + Mat responses; + bool ok = read_num_class_data( data_filename, 16, &data, &responses ); if( !ok ) - { - printf( "Could not read the database %s\n", data_filename ); - return -1; - } - ////////// SVM parameters /////////////////////////////// - CvSVMParams param; - param.kernel_type=CvSVM::LINEAR; - param.svm_type=CvSVM::C_SVC; - param.C=1; - /////////////////////////////////////////////////////////// - - printf( "The database %s is loaded.\n", data_filename ); - nsamples_all = data->rows; - ntrain_samples = (int)(nsamples_all*0.1); - var_count = data->cols; + return ok; + + Ptr model; + + int nsamples_all = data.rows; + int ntrain_samples = (int)(nsamples_all*0.8); // Create or load Random Trees classifier - if( filename_to_load ) + if( !filename_to_load.empty() ) { - // load classifier from the specified file - svm.load( filename_to_load ); + model = load_classifier(filename_to_load); + if( model.empty() ) + return false; ntrain_samples = 0; - if( svm.get_var_count() == 0 ) - { - printf( "Could not read the classifier %s\n", filename_to_load ); - return -1; - } - printf( "The classifier %s is loaded.\n", filename_to_load ); } else { - // train classifier - printf( "Training the classifier (may take a few minutes)...\n"); - cvGetRows( data, &train_data, 0, ntrain_samples ); - train_resp = cvCreateMat( ntrain_samples, 1, CV_32FC1); - for (int i = 0; i < ntrain_samples; i++) - train_resp->data.fl[i] = responses->data.fl[i]; - svm.train(&train_data, train_resp, 0, 0, param); - } - - // classification - std::vector _sample(var_count * (nsamples_all - ntrain_samples)); - CvMat sample = cvMat( nsamples_all - ntrain_samples, 16, CV_32FC1, &_sample[0] ); - std::vector true_results(nsamples_all - ntrain_samples); - for (int j = ntrain_samples; j < nsamples_all; j++) - { - float *s = data->data.fl + j * var_count; - - for (int i = 0; i < var_count; i++) - { - sample.data.fl[(j - ntrain_samples) * var_count + i] = s[i]; - } - true_results[j - ntrain_samples] = responses->data.fl[j]; - } - CvMat *result = cvCreateMat(1, nsamples_all - ntrain_samples, CV_32FC1); + // create classifier by using and + cout << "Training the classifier ...\n"; + Ptr tdata = prepare_train_data(data, responses, ntrain_samples); - printf("Classification (may take a few minutes)...\n"); - double t = (double)cvGetTickCount(); - svm.predict(&sample, result); - t = (double)cvGetTickCount() - t; - printf("Prediction type: %gms\n", t/(cvGetTickFrequency()*1000.)); + SVM::Params params; + params.svmType = SVM::C_SVC; + params.kernelType = SVM::LINEAR; + params.C = 1; - int true_resp = 0; - for (int i = 0; i < nsamples_all - ntrain_samples; i++) - { - if (result->data.fl[i] == true_results[i]) - true_resp++; + model = StatModel::train(tdata, params); + cout << endl; } - printf("true_resp = %f%%\n", (float)true_resp / (nsamples_all - ntrain_samples) * 100); - - if( filename_to_save ) - svm.save( filename_to_save ); - - cvReleaseMat( &train_resp ); - cvReleaseMat( &result ); - cvReleaseMat( &data ); - cvReleaseMat( &responses ); - - return 0; + test_and_save_classifier(model, data, responses, ntrain_samples, 0, filename_to_save); + return true; } int main( int argc, char *argv[] ) { - char* filename_to_save = 0; - char* filename_to_load = 0; - char default_data_filename[] = "./letter-recognition.data"; - char* data_filename = default_data_filename; + string filename_to_save = ""; + string filename_to_load = ""; + string data_filename = "./letter-recognition.data"; int method = 0; int i; @@ -767,18 +518,18 @@ int main( int argc, char *argv[] ) { method = 2; } - else if ( strcmp(argv[i], "-knearest") == 0) - { - method = 3; - } - else if ( strcmp(argv[i], "-nbayes") == 0) - { - method = 4; - } - else if ( strcmp(argv[i], "-svm") == 0) - { - method = 5; - } + else if( strcmp(argv[i], "-knearest") == 0 || strcmp(argv[i], "-knn") == 0 ) + { + method = 3; + } + else if( strcmp(argv[i], "-nbayes") == 0) + { + method = 4; + } + else if( strcmp(argv[i], "-svm") == 0) + { + method = 5; + } else break; } diff --git a/samples/cpp/mushroom.cpp b/samples/cpp/mushroom.cpp deleted file mode 100644 index 60eb9f066ccdeac60dacf91655472cd790240159..0000000000000000000000000000000000000000 --- a/samples/cpp/mushroom.cpp +++ /dev/null @@ -1,322 +0,0 @@ -#include "opencv2/core/core_c.h" -#include "opencv2/ml/ml.hpp" -#include - -static void help() -{ - printf("\nThis program demonstrated the use of OpenCV's decision tree function for learning and predicting data\n" - "Usage :\n" - "./mushroom \n" - "\n" - "The sample demonstrates how to build a decision tree for classifying mushrooms.\n" - "It uses the sample base agaricus-lepiota.data from UCI Repository, here is the link:\n" - "\n" - "Newman, D.J. & Hettich, S. & Blake, C.L. & Merz, C.J. (1998).\n" - "UCI Repository of machine learning databases\n" - "[http://www.ics.uci.edu/~mlearn/MLRepository.html].\n" - "Irvine, CA: University of California, Department of Information and Computer Science.\n" - "\n" - "// loads the mushroom database, which is a text file, containing\n" - "// one training sample per row, all the input variables and the output variable are categorical,\n" - "// the values are encoded by characters.\n\n"); -} - -static int mushroom_read_database( const char* filename, CvMat** data, CvMat** missing, CvMat** responses ) -{ - const int M = 1024; - FILE* f = fopen( filename, "rt" ); - CvMemStorage* storage; - CvSeq* seq; - char buf[M+2], *ptr; - float* el_ptr; - CvSeqReader reader; - int i, j, var_count = 0; - - if( !f ) - return 0; - - // read the first line and determine the number of variables - if( !fgets( buf, M, f )) - { - fclose(f); - return 0; - } - - for( ptr = buf; *ptr != '\0'; ptr++ ) - var_count += *ptr == ','; - assert( ptr - buf == (var_count+1)*2 ); - - // create temporary memory storage to store the whole database - el_ptr = new float[var_count+1]; - storage = cvCreateMemStorage(); - seq = cvCreateSeq( 0, sizeof(*seq), (var_count+1)*sizeof(float), storage ); - - for(;;) - { - for( i = 0; i <= var_count; i++ ) - { - int c = buf[i*2]; - el_ptr[i] = c == '?' ? -1.f : (float)c; - } - if( i != var_count+1 ) - break; - cvSeqPush( seq, el_ptr ); - if( !fgets( buf, M, f ) || !strchr( buf, ',' ) ) - break; - } - fclose(f); - - // allocate the output matrices and copy the base there - *data = cvCreateMat( seq->total, var_count, CV_32F ); - *missing = cvCreateMat( seq->total, var_count, CV_8U ); - *responses = cvCreateMat( seq->total, 1, CV_32F ); - - cvStartReadSeq( seq, &reader ); - - for( i = 0; i < seq->total; i++ ) - { - const float* sdata = (float*)reader.ptr + 1; - float* ddata = data[0]->data.fl + var_count*i; - float* dr = responses[0]->data.fl + i; - uchar* dm = missing[0]->data.ptr + var_count*i; - - for( j = 0; j < var_count; j++ ) - { - ddata[j] = sdata[j]; - dm[j] = sdata[j] < 0; - } - *dr = sdata[-1]; - CV_NEXT_SEQ_ELEM( seq->elem_size, reader ); - } - - cvReleaseMemStorage( &storage ); - delete [] el_ptr; - return 1; -} - - -static CvDTree* mushroom_create_dtree( const CvMat* data, const CvMat* missing, - const CvMat* responses, float p_weight ) -{ - CvDTree* dtree; - CvMat* var_type; - int i, hr1 = 0, hr2 = 0, p_total = 0; - float priors[] = { 1, p_weight }; - - var_type = cvCreateMat( data->cols + 1, 1, CV_8U ); - cvSet( var_type, cvScalarAll(CV_VAR_CATEGORICAL) ); // all the variables are categorical - - dtree = new CvDTree; - - dtree->train( data, CV_ROW_SAMPLE, responses, 0, 0, var_type, missing, - CvDTreeParams( 8, // max depth - 10, // min sample count - 0, // regression accuracy: N/A here - true, // compute surrogate split, as we have missing data - 15, // max number of categories (use sub-optimal algorithm for larger numbers) - 10, // the number of cross-validation folds - true, // use 1SE rule => smaller tree - true, // throw away the pruned tree branches - priors // the array of priors, the bigger p_weight, the more attention - // to the poisonous mushrooms - // (a mushroom will be judjed to be poisonous with bigger chance) - )); - - // compute hit-rate on the training database, demonstrates predict usage. - for( i = 0; i < data->rows; i++ ) - { - CvMat sample, mask; - cvGetRow( data, &sample, i ); - cvGetRow( missing, &mask, i ); - double r = dtree->predict( &sample, &mask )->value; - int d = fabs(r - responses->data.fl[i]) >= FLT_EPSILON; - if( d ) - { - if( r != 'p' ) - hr1++; - else - hr2++; - } - p_total += responses->data.fl[i] == 'p'; - } - - printf( "Results on the training database:\n" - "\tPoisonous mushrooms mis-predicted: %d (%g%%)\n" - "\tFalse-alarms: %d (%g%%)\n", hr1, (double)hr1*100/p_total, - hr2, (double)hr2*100/(data->rows - p_total) ); - - cvReleaseMat( &var_type ); - - return dtree; -} - - -static const char* var_desc[] = -{ - "cap shape (bell=b,conical=c,convex=x,flat=f)", - "cap surface (fibrous=f,grooves=g,scaly=y,smooth=s)", - "cap color (brown=n,buff=b,cinnamon=c,gray=g,green=r,\n\tpink=p,purple=u,red=e,white=w,yellow=y)", - "bruises? (bruises=t,no=f)", - "odor (almond=a,anise=l,creosote=c,fishy=y,foul=f,\n\tmusty=m,none=n,pungent=p,spicy=s)", - "gill attachment (attached=a,descending=d,free=f,notched=n)", - "gill spacing (close=c,crowded=w,distant=d)", - "gill size (broad=b,narrow=n)", - "gill color (black=k,brown=n,buff=b,chocolate=h,gray=g,\n\tgreen=r,orange=o,pink=p,purple=u,red=e,white=w,yellow=y)", - "stalk shape (enlarging=e,tapering=t)", - "stalk root (bulbous=b,club=c,cup=u,equal=e,rhizomorphs=z,rooted=r)", - "stalk surface above ring (ibrous=f,scaly=y,silky=k,smooth=s)", - "stalk surface below ring (ibrous=f,scaly=y,silky=k,smooth=s)", - "stalk color above ring (brown=n,buff=b,cinnamon=c,gray=g,orange=o,\n\tpink=p,red=e,white=w,yellow=y)", - "stalk color below ring (brown=n,buff=b,cinnamon=c,gray=g,orange=o,\n\tpink=p,red=e,white=w,yellow=y)", - "veil type (partial=p,universal=u)", - "veil color (brown=n,orange=o,white=w,yellow=y)", - "ring number (none=n,one=o,two=t)", - "ring type (cobwebby=c,evanescent=e,flaring=f,large=l,\n\tnone=n,pendant=p,sheathing=s,zone=z)", - "spore print color (black=k,brown=n,buff=b,chocolate=h,green=r,\n\torange=o,purple=u,white=w,yellow=y)", - "population (abundant=a,clustered=c,numerous=n,\n\tscattered=s,several=v,solitary=y)", - "habitat (grasses=g,leaves=l,meadows=m,paths=p\n\turban=u,waste=w,woods=d)", - 0 -}; - - -static void print_variable_importance( CvDTree* dtree ) -{ - const CvMat* var_importance = dtree->get_var_importance(); - int i; - char input[1000]; - - if( !var_importance ) - { - printf( "Error: Variable importance can not be retrieved\n" ); - return; - } - - printf( "Print variable importance information? (y/n) " ); - int values_read = scanf( "%1s", input ); - CV_Assert(values_read == 1); - - if( input[0] != 'y' && input[0] != 'Y' ) - return; - - for( i = 0; i < var_importance->cols*var_importance->rows; i++ ) - { - double val = var_importance->data.db[i]; - char buf[100]; - int len = (int)(strchr( var_desc[i], '(' ) - var_desc[i] - 1); - strncpy( buf, var_desc[i], len ); - buf[len] = '\0'; - printf( "%s", buf ); - printf( ": %g%%\n", val*100. ); - } -} - -static void interactive_classification( CvDTree* dtree ) -{ - char input[1000]; - const CvDTreeNode* root; - CvDTreeTrainData* data; - - if( !dtree ) - return; - - root = dtree->get_root(); - data = dtree->get_data(); - - for(;;) - { - const CvDTreeNode* node; - - printf( "Start/Proceed with interactive mushroom classification (y/n): " ); - int values_read = scanf( "%1s", input ); - CV_Assert(values_read == 1); - - if( input[0] != 'y' && input[0] != 'Y' ) - break; - printf( "Enter 1-letter answers, '?' for missing/unknown value...\n" ); - - // custom version of predict - node = root; - for(;;) - { - CvDTreeSplit* split = node->split; - int dir = 0; - - if( !node->left || node->Tn <= dtree->get_pruned_tree_idx() || !node->split ) - break; - - for( ; split != 0; ) - { - int vi = split->var_idx, j; - int count = data->cat_count->data.i[vi]; - const int* map = data->cat_map->data.i + data->cat_ofs->data.i[vi]; - - printf( "%s: ", var_desc[vi] ); - values_read = scanf( "%1s", input ); - CV_Assert(values_read == 1); - - if( input[0] == '?' ) - { - split = split->next; - continue; - } - - // convert the input character to the normalized value of the variable - for( j = 0; j < count; j++ ) - if( map[j] == input[0] ) - break; - if( j < count ) - { - dir = (split->subset[j>>5] & (1 << (j&31))) ? -1 : 1; - if( split->inversed ) - dir = -dir; - break; - } - else - printf( "Error: unrecognized value\n" ); - } - - if( !dir ) - { - printf( "Impossible to classify the sample\n"); - node = 0; - break; - } - node = dir < 0 ? node->left : node->right; - } - - if( node ) - printf( "Prediction result: the mushroom is %s\n", - node->class_idx == 0 ? "EDIBLE" : "POISONOUS" ); - printf( "\n-----------------------------\n" ); - } -} - - -int main( int argc, char** argv ) -{ - CvMat *data = 0, *missing = 0, *responses = 0; - CvDTree* dtree; - const char* base_path = argc >= 2 ? argv[1] : "agaricus-lepiota.data"; - - help(); - - if( !mushroom_read_database( base_path, &data, &missing, &responses ) ) - { - printf( "\nUnable to load the training database\n\n"); - help(); - return -1; - } - - dtree = mushroom_create_dtree( data, missing, responses, - 10 // poisonous mushrooms will have 10x higher weight in the decision tree - ); - cvReleaseMat( &data ); - cvReleaseMat( &missing ); - cvReleaseMat( &responses ); - - print_variable_importance( dtree ); - interactive_classification( dtree ); - delete dtree; - - return 0; -} diff --git a/samples/cpp/points_classifier.cpp b/samples/cpp/points_classifier.cpp index 26858da886e1c9c04b6268fb219484803d1c1f8d..eedec4b6a898e17f23adcf2fa0aeb174b497674a 100644 --- a/samples/cpp/points_classifier.cpp +++ b/samples/cpp/points_classifier.cpp @@ -12,6 +12,7 @@ using namespace std; using namespace cv; +using namespace cv::ml; const Scalar WHITE_COLOR = Scalar(255,255,255); const string winName = "points"; @@ -22,18 +23,20 @@ RNG rng; vector trainedPoints; vector trainedPointsMarkers; -vector classColors; - -#define _NBC_ 0 // normal Bayessian classifier -#define _KNN_ 0 // k nearest neighbors classifier -#define _SVM_ 0 // support vectors machine +const int MAX_CLASSES = 2; +vector classColors(MAX_CLASSES); +int currentClass = 0; +vector classCounters(MAX_CLASSES); + +#define _NBC_ 1 // normal Bayessian classifier +#define _KNN_ 1 // k nearest neighbors classifier +#define _SVM_ 1 // support vectors machine #define _DT_ 1 // decision tree -#define _BT_ 0 // ADA Boost +#define _BT_ 1 // ADA Boost #define _GBT_ 0 // gradient boosted trees -#define _RF_ 0 // random forest -#define _ERT_ 0 // extremely randomized trees -#define _ANN_ 0 // artificial neural networks -#define _EM_ 0 // expectation-maximization +#define _RF_ 1 // random forest +#define _ANN_ 1 // artificial neural networks +#define _EM_ 1 // expectation-maximization static void on_mouse( int event, int x, int y, int /*flags*/, void* ) { @@ -44,76 +47,43 @@ static void on_mouse( int event, int x, int y, int /*flags*/, void* ) if( event == EVENT_LBUTTONUP ) { - if( classColors.empty() ) - return; - trainedPoints.push_back( Point(x,y) ); - trainedPointsMarkers.push_back( (int)(classColors.size()-1) ); + trainedPointsMarkers.push_back( currentClass ); + classCounters[currentClass]++; updateFlag = true; } - else if( event == EVENT_RBUTTONUP ) - { -#if _BT_ - if( classColors.size() < 2 ) - { -#endif - classColors.push_back( Scalar((uchar)rng(256), (uchar)rng(256), (uchar)rng(256)) ); - updateFlag = true; -#if _BT_ - } - else - cout << "New class can not be added, because CvBoost can only be used for 2-class classification" << endl; -#endif - - } //draw if( updateFlag ) { img = Scalar::all(0); - // put the text - stringstream text; - text << "current class " << classColors.size()-1; - putText( img, text.str(), Point(10,25), FONT_HERSHEY_SIMPLEX, 0.8f, WHITE_COLOR, 2 ); - - text.str(""); - text << "total classes " << classColors.size(); - putText( img, text.str(), Point(10,50), FONT_HERSHEY_SIMPLEX, 0.8f, WHITE_COLOR, 2 ); - - text.str(""); - text << "total points " << trainedPoints.size(); - putText(img, text.str(), Point(10,75), FONT_HERSHEY_SIMPLEX, 0.8f, WHITE_COLOR, 2 ); - // draw points for( size_t i = 0; i < trainedPoints.size(); i++ ) - circle( img, trainedPoints[i], 5, classColors[trainedPointsMarkers[i]], -1 ); + { + Vec3b c = classColors[trainedPointsMarkers[i]]; + circle( img, trainedPoints[i], 5, Scalar(c), -1 ); + } imshow( winName, img ); } } -static void prepare_train_data( Mat& samples, Mat& classes ) +static Mat prepare_train_samples(const vector& pts) { - Mat( trainedPoints ).copyTo( samples ); - Mat( trainedPointsMarkers ).copyTo( classes ); - - // reshape trainData and change its type - samples = samples.reshape( 1, samples.rows ); - samples.convertTo( samples, CV_32FC1 ); + Mat samples; + Mat(pts).reshape(1, (int)pts.size()).convertTo(samples, CV_32F); + return samples; } -#if _NBC_ -static void find_decision_boundary_NBC() +static Ptr prepare_train_data() { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier - CvNormalBayesClassifier normalBayesClassifier( trainSamples, trainClasses ); + Mat samples = prepare_train_samples(trainedPoints); + return TrainData::create(samples, ROW_SAMPLE, Mat(trainedPointsMarkers)); +} +static void predict_and_paint(const Ptr& model, Mat& dst) +{ Mat testSample( 1, 2, CV_32FC1 ); for( int y = 0; y < img.rows; y += testStep ) { @@ -122,378 +92,171 @@ static void find_decision_boundary_NBC() testSample.at(0) = (float)x; testSample.at(1) = (float)y; - int response = (int)normalBayesClassifier.predict( testSample ); - circle( imgDst, Point(x,y), 1, classColors[response] ); + int response = (int)model->predict( testSample ); + dst.at(y, x) = classColors[response]; } } } -#endif - -#if _KNN_ -static void find_decision_boundary_KNN( int K ) +#if _NBC_ +static void find_decision_boundary_NBC() { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - // learn classifier -#if defined HAVE_OPENCV_OCL && _OCL_KNN_ - cv::ocl::KNearestNeighbour knnClassifier; - Mat temp, result; - knnClassifier.train(trainSamples, trainClasses, temp, false, K); - cv::ocl::oclMat testSample_ocl, reslut_ocl; -#else - CvKNearest knnClassifier( trainSamples, trainClasses, Mat(), false, K ); -#endif + Ptr normalBayesClassifier = StatModel::train(prepare_train_data(), NormalBayesClassifier::Params()); - Mat testSample( 1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; -#if defined HAVE_OPENCV_OCL && _OCL_KNN_ - testSample_ocl.upload(testSample); - - knnClassifier.find_nearest(testSample_ocl, K, reslut_ocl); + predict_and_paint(normalBayesClassifier, imgDst); +} +#endif - reslut_ocl.download(result); - int response = saturate_cast(result.at(0)); - circle(imgDst, Point(x, y), 1, classColors[response]); -#else - int response = (int)knnClassifier.find_nearest( testSample, K ); - circle( imgDst, Point(x,y), 1, classColors[response] ); -#endif - } - } +#if _KNN_ +static void find_decision_boundary_KNN( int K ) +{ + Ptr knn = StatModel::train(prepare_train_data(), KNearest::Params(K, true)); + predict_and_paint(knn, imgDst); } #endif #if _SVM_ -static void find_decision_boundary_SVM( CvSVMParams params ) +static void find_decision_boundary_SVM( SVM::Params params ) { - img.copyTo( imgDst ); + Ptr svm = StatModel::train(prepare_train_data(), params); + predict_and_paint(svm, imgDst); - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier -#if defined HAVE_OPENCV_OCL && _OCL_SVM_ - cv::ocl::CvSVM_OCL svmClassifier(trainSamples, trainClasses, Mat(), Mat(), params); -#else - CvSVM svmClassifier( trainSamples, trainClasses, Mat(), Mat(), params ); -#endif - - Mat testSample( 1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) + Mat sv = svm->getSupportVectors(); + for( int i = 0; i < sv.rows; i++ ) { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; - - int response = (int)svmClassifier.predict( testSample ); - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); - } - } - - - for( int i = 0; i < svmClassifier.get_support_vector_count(); i++ ) - { - const float* supportVector = svmClassifier.get_support_vector(i); - circle( imgDst, Point(saturate_cast(supportVector[0]),saturate_cast(supportVector[1])), 5, CV_RGB(255,255,255), -1 ); + const float* supportVector = sv.ptr(i); + circle( imgDst, Point(saturate_cast(supportVector[0]),saturate_cast(supportVector[1])), 5, Scalar(255,255,255), -1 ); } - } #endif #if _DT_ static void find_decision_boundary_DT() { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier - CvDTree dtree; - - Mat var_types( 1, trainSamples.cols + 1, CV_8UC1, Scalar(CV_VAR_ORDERED) ); - var_types.at( trainSamples.cols ) = CV_VAR_CATEGORICAL; - - CvDTreeParams params; - params.max_depth = 8; - params.min_sample_count = 2; - params.use_surrogates = false; - params.cv_folds = 0; // the number of cross-validation folds - params.use_1se_rule = false; - params.truncate_pruned_tree = false; + DTrees::Params params; + params.maxDepth = 8; + params.minSampleCount = 2; + params.useSurrogates = false; + params.CVFolds = 0; // the number of cross-validation folds + params.use1SERule = false; + params.truncatePrunedTree = false; - dtree.train( trainSamples, CV_ROW_SAMPLE, trainClasses, - Mat(), Mat(), var_types, Mat(), params ); + Ptr dtree = StatModel::train(prepare_train_data(), params); - Mat testSample(1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; - - int response = (int)dtree.predict( testSample )->value; - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); - } - } + predict_and_paint(dtree, imgDst); } #endif #if _BT_ -void find_decision_boundary_BT() +static void find_decision_boundary_BT() { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier - CvBoost boost; - - Mat var_types( 1, trainSamples.cols + 1, CV_8UC1, Scalar(CV_VAR_ORDERED) ); - var_types.at( trainSamples.cols ) = CV_VAR_CATEGORICAL; - - CvBoostParams params( CvBoost::DISCRETE, // boost_type - 100, // weak_count - 0.95, // weight_trim_rate - 2, // max_depth - false, //use_surrogates - 0 // priors - ); - - boost.train( trainSamples, CV_ROW_SAMPLE, trainClasses, Mat(), Mat(), var_types, Mat(), params ); - - Mat testSample(1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; - - int response = (int)boost.predict( testSample ); - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); - } - } + Boost::Params params( Boost::DISCRETE, // boost_type + 100, // weak_count + 0.95, // weight_trim_rate + 2, // max_depth + false, //use_surrogates + Mat() // priors + ); + + Ptr boost = StatModel::train(prepare_train_data(), params); + predict_and_paint(boost, imgDst); } #endif #if _GBT_ -void find_decision_boundary_GBT() +static void find_decision_boundary_GBT() { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier - CvGBTrees gbtrees; - - Mat var_types( 1, trainSamples.cols + 1, CV_8UC1, Scalar(CV_VAR_ORDERED) ); - var_types.at( trainSamples.cols ) = CV_VAR_CATEGORICAL; - - CvGBTreesParams params( CvGBTrees::DEVIANCE_LOSS, // loss_function_type - 100, // weak_count - 0.1f, // shrinkage - 1.0f, // subsample_portion - 2, // max_depth - false // use_surrogates ) - ); - - gbtrees.train( trainSamples, CV_ROW_SAMPLE, trainClasses, Mat(), Mat(), var_types, Mat(), params ); - - Mat testSample(1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; + GBTrees::Params params( GBTrees::DEVIANCE_LOSS, // loss_function_type + 100, // weak_count + 0.1f, // shrinkage + 1.0f, // subsample_portion + 2, // max_depth + false // use_surrogates ) + ); - int response = (int)gbtrees.predict( testSample ); - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); - } - } + Ptr gbtrees = StatModel::train(prepare_train_data(), params); + predict_and_paint(gbtrees, imgDst); } - #endif #if _RF_ -void find_decision_boundary_RF() +static void find_decision_boundary_RF() { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier - CvRTrees rtrees; - CvRTParams params( 4, // max_depth, + RTrees::Params params( 4, // max_depth, 2, // min_sample_count, 0.f, // regression_accuracy, false, // use_surrogates, 16, // max_categories, - 0, // priors, + Mat(), // priors, false, // calc_var_importance, 1, // nactive_vars, - 5, // max_num_of_trees_in_the_forest, - 0, // forest_accuracy, - CV_TERMCRIT_ITER // termcrit_type + TermCriteria(TermCriteria::MAX_ITER, 5, 0) // max_num_of_trees_in_the_forest, ); - rtrees.train( trainSamples, CV_ROW_SAMPLE, trainClasses, Mat(), Mat(), Mat(), Mat(), params ); - - Mat testSample(1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; - - int response = (int)rtrees.predict( testSample ); - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); - } - } + Ptr rtrees = StatModel::train(prepare_train_data(), params); + predict_and_paint(rtrees, imgDst); } #endif -#if _ERT_ -void find_decision_boundary_ERT() -{ - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - // learn classifier - CvERTrees ertrees; - - Mat var_types( 1, trainSamples.cols + 1, CV_8UC1, Scalar(CV_VAR_ORDERED) ); - var_types.at( trainSamples.cols ) = CV_VAR_CATEGORICAL; - - CvRTParams params( 4, // max_depth, - 2, // min_sample_count, - 0.f, // regression_accuracy, - false, // use_surrogates, - 16, // max_categories, - 0, // priors, - false, // calc_var_importance, - 1, // nactive_vars, - 5, // max_num_of_trees_in_the_forest, - 0, // forest_accuracy, - CV_TERMCRIT_ITER // termcrit_type - ); - - ertrees.train( trainSamples, CV_ROW_SAMPLE, trainClasses, Mat(), Mat(), var_types, Mat(), params ); - - Mat testSample(1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; - - int response = (int)ertrees.predict( testSample ); - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); - } - } -} -#endif - #if _ANN_ -void find_decision_boundary_ANN( const Mat& layer_sizes ) +static void find_decision_boundary_ANN( const Mat& layer_sizes ) { - img.copyTo( imgDst ); - - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); + ANN_MLP::Params params(layer_sizes, ANN_MLP::SIGMOID_SYM, 1, 1, TermCriteria(TermCriteria::MAX_ITER+TermCriteria::EPS, 300, FLT_EPSILON), + ANN_MLP::Params::BACKPROP, 0.001); - // prerare trainClasses - trainClasses.create( trainedPoints.size(), classColors.size(), CV_32FC1 ); - for( int i = 0; i < trainClasses.rows; i++ ) + Mat trainClasses = Mat::zeros( (int)trainedPoints.size(), (int)classColors.size(), CV_32FC1 ); + for( int i = 0; i < trainClasses.rows; i++ ) { - for( int k = 0; k < trainClasses.cols; k++ ) - { - if( k == trainedPointsMarkers[i] ) - trainClasses.at(i,k) = 1; - else - trainClasses.at(i,k) = 0; - } + trainClasses.at(i, trainedPointsMarkers[i]) = 1.f; } - Mat weights( 1, trainedPoints.size(), CV_32FC1, Scalar::all(1) ); - - // learn classifier - CvANN_MLP ann( layer_sizes, CvANN_MLP::SIGMOID_SYM, 1, 1 ); - ann.train( trainSamples, trainClasses, weights ); + Mat samples = prepare_train_samples(trainedPoints); + Ptr tdata = TrainData::create(samples, ROW_SAMPLE, trainClasses); - Mat testSample( 1, 2, CV_32FC1 ); - for( int y = 0; y < img.rows; y += testStep ) - { - for( int x = 0; x < img.cols; x += testStep ) - { - testSample.at(0) = (float)x; - testSample.at(1) = (float)y; - - Mat outputs( 1, classColors.size(), CV_32FC1, testSample.data ); - ann.predict( testSample, outputs ); - Point maxLoc; - minMaxLoc( outputs, 0, 0, 0, &maxLoc ); - circle( imgDst, Point(x,y), 2, classColors[maxLoc.x], 1 ); - } - } + Ptr ann = StatModel::train(tdata, params); + predict_and_paint(ann, imgDst); } #endif #if _EM_ -void find_decision_boundary_EM() +static void find_decision_boundary_EM() { img.copyTo( imgDst ); - Mat trainSamples, trainClasses; - prepare_train_data( trainSamples, trainClasses ); - - vector em_models(classColors.size()); + Mat samples = prepare_train_samples(trainedPoints); - CV_Assert((int)trainClasses.total() == trainSamples.rows); - CV_Assert((int)trainClasses.type() == CV_32SC1); + int i, j, nmodels = (int)classColors.size(); + vector > em_models(nmodels); + Mat modelSamples; - for(size_t modelIndex = 0; modelIndex < em_models.size(); modelIndex++) + for( i = 0; i < nmodels; i++ ) { const int componentCount = 3; - em_models[modelIndex] = EM(componentCount, cv::EM::COV_MAT_DIAGONAL); - Mat modelSamples; - for(int sampleIndex = 0; sampleIndex < trainSamples.rows; sampleIndex++) + modelSamples.release(); + for( j = 0; j < samples.rows; j++ ) { - if(trainClasses.at(sampleIndex) == (int)modelIndex) - modelSamples.push_back(trainSamples.row(sampleIndex)); + if( trainedPointsMarkers[j] == i ) + modelSamples.push_back(samples.row(j)); } // learn models - if(!modelSamples.empty()) - em_models[modelIndex].train(modelSamples); + if( !modelSamples.empty() ) + { + em_models[i] = EM::train(modelSamples, noArray(), noArray(), noArray(), + EM::Params(componentCount, EM::COV_MAT_DIAGONAL)); + } } // classify coordinate plane points using the bayes classifier, i.e. // y(x) = arg max_i=1_modelsCount likelihoods_i(x) Mat testSample(1, 2, CV_32FC1 ); + Mat logLikelihoods(1, nmodels, CV_64FC1, Scalar(-DBL_MAX)); + for( int y = 0; y < img.rows; y += testStep ) { for( int x = 0; x < img.cols; x += testStep ) @@ -501,17 +264,14 @@ void find_decision_boundary_EM() testSample.at(0) = (float)x; testSample.at(1) = (float)y; - Mat logLikelihoods(1, em_models.size(), CV_64FC1, Scalar(-DBL_MAX)); - for(size_t modelIndex = 0; modelIndex < em_models.size(); modelIndex++) + for( i = 0; i < nmodels; i++ ) { - if(em_models[modelIndex].isTrained()) - logLikelihoods.at(modelIndex) = em_models[modelIndex].predict(testSample)[0]; + if( !em_models[i].empty() ) + logLikelihoods.at(i) = em_models[i]->predict2(testSample, noArray())[0]; } Point maxLoc; minMaxLoc(logLikelihoods, 0, 0, 0, &maxLoc); - - int response = maxLoc.x; - circle( imgDst, Point(x,y), 2, classColors[response], 1 ); + imgDst.at(y, x) = classColors[maxLoc.x]; } } } @@ -520,7 +280,7 @@ void find_decision_boundary_EM() int main() { cout << "Use:" << endl - << " right mouse button - to add new class;" << endl + << " key '0' .. '1' - switch to class #n" << endl << " left mouse button - to add new point;" << endl << " key 'r' - to run the ML model;" << endl << " key 'i' - to init (clear) the data." << endl << endl; @@ -532,6 +292,9 @@ int main() imshow( "points", img ); setMouseCallback( "points", on_mouse ); + classColors[0] = Vec3b(0, 255, 0); + classColors[1] = Vec3b(0, 0, 255); + for(;;) { uchar key = (uchar)waitKey(); @@ -542,98 +305,94 @@ int main() { img = Scalar::all(0); - classColors.clear(); trainedPoints.clear(); trainedPointsMarkers.clear(); + classCounters.assign(MAX_CLASSES, 0); imshow( winName, img ); } + if( key == '0' || key == '1' ) + { + currentClass = key - '0'; + } + if( key == 'r' ) // run { + double minVal = 0; + minMaxLoc(classCounters, &minVal, 0, 0, 0); + if( minVal == 0 ) + { + printf("each class should have at least 1 point\n"); + continue; + } + img.copyTo( imgDst ); #if _NBC_ find_decision_boundary_NBC(); - namedWindow( "NormalBayesClassifier", WINDOW_AUTOSIZE ); imshow( "NormalBayesClassifier", imgDst ); #endif #if _KNN_ int K = 3; find_decision_boundary_KNN( K ); - namedWindow( "kNN", WINDOW_AUTOSIZE ); imshow( "kNN", imgDst ); K = 15; find_decision_boundary_KNN( K ); - namedWindow( "kNN2", WINDOW_AUTOSIZE ); imshow( "kNN2", imgDst ); #endif #if _SVM_ //(1)-(2)separable and not sets - CvSVMParams params; - params.svm_type = CvSVM::C_SVC; - params.kernel_type = CvSVM::POLY; //CvSVM::LINEAR; + SVM::Params params; + params.svmType = SVM::C_SVC; + params.kernelType = SVM::POLY; //CvSVM::LINEAR; params.degree = 0.5; params.gamma = 1; params.coef0 = 1; params.C = 1; params.nu = 0.5; params.p = 0; - params.term_crit = cvTermCriteria(CV_TERMCRIT_ITER, 1000, 0.01); + params.termCrit = TermCriteria(TermCriteria::MAX_ITER+TermCriteria::EPS, 1000, 0.01); find_decision_boundary_SVM( params ); - namedWindow( "classificationSVM1", WINDOW_AUTOSIZE ); imshow( "classificationSVM1", imgDst ); params.C = 10; find_decision_boundary_SVM( params ); - namedWindow( "classificationSVM2", WINDOW_AUTOSIZE ); imshow( "classificationSVM2", imgDst ); #endif #if _DT_ find_decision_boundary_DT(); - namedWindow( "DT", WINDOW_AUTOSIZE ); imshow( "DT", imgDst ); #endif #if _BT_ find_decision_boundary_BT(); - namedWindow( "BT", WINDOW_AUTOSIZE ); imshow( "BT", imgDst); #endif #if _GBT_ find_decision_boundary_GBT(); - namedWindow( "GBT", WINDOW_AUTOSIZE ); imshow( "GBT", imgDst); #endif #if _RF_ find_decision_boundary_RF(); - namedWindow( "RF", WINDOW_AUTOSIZE ); imshow( "RF", imgDst); #endif -#if _ERT_ - find_decision_boundary_ERT(); - namedWindow( "ERT", WINDOW_AUTOSIZE ); - imshow( "ERT", imgDst); -#endif - #if _ANN_ Mat layer_sizes1( 1, 3, CV_32SC1 ); layer_sizes1.at(0) = 2; layer_sizes1.at(1) = 5; - layer_sizes1.at(2) = classColors.size(); + layer_sizes1.at(2) = (int)classColors.size(); find_decision_boundary_ANN( layer_sizes1 ); - namedWindow( "ANN", WINDOW_AUTOSIZE ); imshow( "ANN", imgDst ); #endif #if _EM_ find_decision_boundary_EM(); - namedWindow( "EM", WINDOW_AUTOSIZE ); imshow( "EM", imgDst ); #endif } diff --git a/samples/cpp/train_HOG.cpp b/samples/cpp/train_HOG.cpp index e3ee190fc39cf62629cf17ff4a31463adb2439b3..fbd217a968ee83af209ebc017945f897746b753d 100644 --- a/samples/cpp/train_HOG.cpp +++ b/samples/cpp/train_HOG.cpp @@ -8,9 +8,10 @@ #include using namespace cv; +using namespace cv::ml; using namespace std; -void get_svm_detector(const SVM& svm, vector< float > & hog_detector ); +void get_svm_detector(const Ptr& svm, vector< float > & hog_detector ); void convert_to_ml(const std::vector< cv::Mat > & train_samples, cv::Mat& trainData ); void load_images( const string & prefix, const string & filename, vector< Mat > & img_lst ); void sample_neg( const vector< Mat > & full_neg_lst, vector< Mat > & neg_lst, const Size & size ); @@ -20,49 +21,24 @@ void train_svm( const vector< Mat > & gradient_lst, const vector< int > & labels void draw_locations( Mat & img, const vector< Rect > & locations, const Scalar & color ); void test_it( const Size & size ); -void get_svm_detector(const SVM& svm, vector< float > & hog_detector ) +void get_svm_detector(const Ptr& svm, vector< float > & hog_detector ) { - // get the number of variables - const int var_all = svm.get_var_count(); - // get the number of support vectors - const int sv_total = svm.get_support_vector_count(); - // get the decision function - const CvSVMDecisionFunc* decision_func = svm.get_decision_function(); // get the support vectors - const float** sv = new const float*[ sv_total ]; - for( int i = 0 ; i < sv_total ; ++i ) - sv[ i ] = svm.get_support_vector(i); - - CV_Assert( var_all > 0 && - sv_total > 0 && - decision_func != 0 && - decision_func->alpha != 0 && - decision_func->sv_count == sv_total ); - - float svi = 0.f; - - hog_detector.clear(); //clear stuff in vector. - hog_detector.reserve( var_all + 1 ); //reserve place for memory efficiency. - - /** - * hog_detector^i = \sum_j support_vector_j^i * \alpha_j - * hog_detector^dim = -\rho - */ - for( int i = 0 ; i < var_all ; ++i ) - { - svi = 0.f; - for( int j = 0 ; j < sv_total ; ++j ) - { - if( decision_func->sv_index != NULL ) // sometime the sv_index isn't store on YML/XML. - svi += (float)( sv[decision_func->sv_index[j]][i] * decision_func->alpha[ j ] ); - else - svi += (float)( sv[j][i] * decision_func->alpha[ j ] ); - } - hog_detector.push_back( svi ); - } - hog_detector.push_back( (float)-decision_func->rho ); - - delete[] sv; + Mat sv = svm->getSupportVectors(); + const int sv_total = sv.rows; + // get the decision function + Mat alpha, svidx; + double rho = svm->getDecisionFunction(0, alpha, svidx); + + CV_Assert( alpha.total() == 1 && svidx.total() == 1 && sv_total == 1 ); + CV_Assert( (alpha.type() == CV_64F && alpha.at(0) == 1.) || + (alpha.type() == CV_32F && alpha.at(0) == 1.f) ); + CV_Assert( sv.type() == CV_32F ); + hog_detector.clear(); + + hog_detector.resize(sv.cols + 1); + memcpy(&hog_detector[0], sv.data, sv.cols*sizeof(hog_detector[0])); + hog_detector[sv.cols] = (float)-rho; } @@ -263,7 +239,7 @@ Mat get_hogdescriptor_visu(const Mat& color_origImg, vector& descriptorVa int mx = drawX + cellSize/2; int my = drawY + cellSize/2; - rectangle(visu, Point((int)(drawX*zoomFac), (int)(drawY*zoomFac)), Point((int)((drawX+cellSize)*zoomFac), (int)((drawY+cellSize)*zoomFac)), CV_RGB(100,100,100), 1); + rectangle(visu, Point((int)(drawX*zoomFac), (int)(drawY*zoomFac)), Point((int)((drawX+cellSize)*zoomFac), (int)((drawY+cellSize)*zoomFac)), Scalar(100,100,100), 1); // draw in each cell all 9 gradient strengths for (int bin=0; bin& descriptorVa float y2 = my + dirVecY * currentGradStrength * maxVecLen * scale; // draw gradient visualization - line(visu, Point((int)(x1*zoomFac),(int)(y1*zoomFac)), Point((int)(x2*zoomFac),(int)(y2*zoomFac)), CV_RGB(0,255,0), 1); + line(visu, Point((int)(x1*zoomFac),(int)(y1*zoomFac)), Point((int)(x2*zoomFac),(int)(y2*zoomFac)), Scalar(0,255,0), 1); } // for (all bins) @@ -337,28 +313,26 @@ void compute_hog( const vector< Mat > & img_lst, vector< Mat > & gradient_lst, c void train_svm( const vector< Mat > & gradient_lst, const vector< int > & labels ) { - SVM svm; - /* Default values to train SVM */ - SVMParams params; + SVM::Params params; params.coef0 = 0.0; params.degree = 3; - params.term_crit.epsilon = 1e-3; + params.termCrit.epsilon = 1e-3; params.gamma = 0; - params.kernel_type = SVM::LINEAR; + params.kernelType = SVM::LINEAR; params.nu = 0.5; params.p = 0.1; // for EPSILON_SVR, epsilon in loss function? params.C = 0.01; // From paper, soft classifier - params.svm_type = SVM::EPS_SVR; // C_SVC; // EPSILON_SVR; // may be also NU_SVR; // do regression task + params.svmType = SVM::EPS_SVR; // C_SVC; // EPSILON_SVR; // may be also NU_SVR; // do regression task Mat train_data; convert_to_ml( gradient_lst, train_data ); clog << "Start training..."; - svm.train( train_data, Mat( labels ), Mat(), Mat(), params ); + Ptr svm = StatModel::train(train_data, ROW_SAMPLE, Mat(labels), params); clog << "...[done]" << endl; - svm.save( "my_people_detector.yml" ); + svm->save( "my_people_detector.yml" ); } void draw_locations( Mat & img, const vector< Rect > & locations, const Scalar & color ) @@ -380,7 +354,7 @@ void test_it( const Size & size ) Scalar reference( 0, 255, 0 ); Scalar trained( 0, 0, 255 ); Mat img, draw; - SVM svm; + Ptr svm; HOGDescriptor hog; HOGDescriptor my_hog; my_hog.winSize = size; @@ -388,7 +362,7 @@ void test_it( const Size & size ) vector< Rect > locations; // Load the trained SVM. - svm.load( "my_people_detector.yml" ); + svm = StatModel::load( "my_people_detector.yml" ); // Set the trained svm to my_hog vector< float > hog_detector; get_svm_detector( svm, hog_detector ); diff --git a/samples/cpp/tree_engine.cpp b/samples/cpp/tree_engine.cpp index 2c3046fd7357c0cc989ea3fffcf188a27d9c2254..6defc31c5041f8a263e3eed6649bd5c06d0b6bbd 100644 --- a/samples/cpp/tree_engine.cpp +++ b/samples/cpp/tree_engine.cpp @@ -1,63 +1,35 @@ #include "opencv2/ml/ml.hpp" -#include "opencv2/core/core_c.h" +#include "opencv2/core/core.hpp" #include "opencv2/core/utility.hpp" #include +#include #include +using namespace cv; +using namespace cv::ml; + static void help() { printf( - "\nThis sample demonstrates how to use different decision trees and forests including boosting and random trees:\n" - "CvDTree dtree;\n" - "CvBoost boost;\n" - "CvRTrees rtrees;\n" - "CvERTrees ertrees;\n" - "CvGBTrees gbtrees;\n" - "Call:\n\t./tree_engine [-r ] [-c] \n" + "\nThis sample demonstrates how to use different decision trees and forests including boosting and random trees.\n" + "Usage:\n\t./tree_engine [-r ] [-ts type_spec] \n" "where -r specified the 0-based index of the response (0 by default)\n" - "-c specifies that the response is categorical (it's ordered by default) and\n" + "-ts specifies the var type spec in the form ord[n1,n2-n3,n4-n5,...]cat[m1-m2,m3,m4-m5,...]\n" " is the name of training data file in comma-separated value format\n\n"); } - -static int count_classes(CvMLData& data) +static void train_and_print_errs(Ptr model, const Ptr& data) { - cv::Mat r = cv::cvarrToMat(data.get_responses()); - std::map rmap; - int i, n = (int)r.total(); - for( i = 0; i < n; i++ ) + bool ok = model->train(data); + if( !ok ) { - float val = r.at(i); - int ival = cvRound(val); - if( ival != val ) - return -1; - rmap[ival] = 1; + printf("Training failed\n"); } - return (int)rmap.size(); -} - -static void print_result(float train_err, float test_err, const CvMat* _var_imp) -{ - printf( "train error %f\n", train_err ); - printf( "test error %f\n\n", test_err ); - - if (_var_imp) + else { - cv::Mat var_imp = cv::cvarrToMat(_var_imp), sorted_idx; - cv::sortIdx(var_imp, sorted_idx, CV_SORT_EVERY_ROW + CV_SORT_DESCENDING); - - printf( "variable importance:\n" ); - int i, n = (int)var_imp.total(); - int type = var_imp.type(); - CV_Assert(type == CV_32F || type == CV_64F); - - for( i = 0; i < n; i++) - { - int k = sorted_idx.at(i); - printf( "%d\t%f\n", k, type == CV_32F ? var_imp.at(k) : var_imp.at(k)); - } + printf( "train error: %f\n", model->calcError(data, false, noArray()) ); + printf( "test error: %f\n\n", model->calcError(data, true, noArray()) ); } - printf("\n"); } int main(int argc, char** argv) @@ -69,14 +41,14 @@ int main(int argc, char** argv) } const char* filename = 0; int response_idx = 0; - bool categorical_response = false; + std::string typespec; for(int i = 1; i < argc; i++) { if(strcmp(argv[i], "-r") == 0) sscanf(argv[++i], "%d", &response_idx); - else if(strcmp(argv[i], "-c") == 0) - categorical_response = true; + else if(strcmp(argv[i], "-ts") == 0) + typespec = argv[++i]; else if(argv[i][0] != '-' ) filename = argv[i]; else @@ -88,52 +60,32 @@ int main(int argc, char** argv) } printf("\nReading in %s...\n\n",filename); - CvDTree dtree; - CvBoost boost; - CvRTrees rtrees; - CvERTrees ertrees; - CvGBTrees gbtrees; - - CvMLData data; + const double train_test_split_ratio = 0.5; + Ptr data = TrainData::loadFromCSV(filename, 0, response_idx, response_idx+1, typespec); - CvTrainTestSplit spl( 0.5f ); - - if ( data.read_csv( filename ) == 0) + if( data.empty() ) { - data.set_response_idx( response_idx ); - if(categorical_response) - data.change_var_type( response_idx, CV_VAR_CATEGORICAL ); - data.set_train_test_split( &spl ); - - printf("======DTREE=====\n"); - dtree.train( &data, CvDTreeParams( 10, 2, 0, false, 16, 0, false, false, 0 )); - print_result( dtree.calc_error( &data, CV_TRAIN_ERROR), dtree.calc_error( &data, CV_TEST_ERROR ), dtree.get_var_importance() ); - - if( categorical_response && count_classes(data) == 2 ) - { - printf("======BOOST=====\n"); - boost.train( &data, CvBoostParams(CvBoost::DISCRETE, 100, 0.95, 2, false, 0)); - print_result( boost.calc_error( &data, CV_TRAIN_ERROR ), boost.calc_error( &data, CV_TEST_ERROR ), 0 ); //doesn't compute importance - } + printf("ERROR: File %s can not be read\n", filename); + return 0; + } - printf("======RTREES=====\n"); - rtrees.train( &data, CvRTParams( 10, 2, 0, false, 16, 0, true, 0, 100, 0, CV_TERMCRIT_ITER )); - print_result( rtrees.calc_error( &data, CV_TRAIN_ERROR), rtrees.calc_error( &data, CV_TEST_ERROR ), rtrees.get_var_importance() ); + data->setTrainTestSplitRatio(train_test_split_ratio); - printf("======ERTREES=====\n"); - ertrees.train( &data, CvRTParams( 18, 2, 0, false, 16, 0, true, 0, 100, 0, CV_TERMCRIT_ITER )); - print_result( ertrees.calc_error( &data, CV_TRAIN_ERROR), ertrees.calc_error( &data, CV_TEST_ERROR ), ertrees.get_var_importance() ); + printf("======DTREE=====\n"); + Ptr dtree = DTrees::create(DTrees::Params( 10, 2, 0, false, 16, 0, false, false, Mat() )); + train_and_print_errs(dtree, data); - printf("======GBTREES=====\n"); - if (categorical_response) - gbtrees.train( &data, CvGBTreesParams(CvGBTrees::DEVIANCE_LOSS, 100, 0.1f, 0.8f, 5, false)); - else - gbtrees.train( &data, CvGBTreesParams(CvGBTrees::SQUARED_LOSS, 100, 0.1f, 0.8f, 5, false)); - print_result( gbtrees.calc_error( &data, CV_TRAIN_ERROR), gbtrees.calc_error( &data, CV_TEST_ERROR ), 0 ); //doesn't compute importance + if( (int)data->getClassLabels().total() <= 2 ) // regression or 2-class classification problem + { + printf("======BOOST=====\n"); + Ptr boost = Boost::create(Boost::Params(Boost::GENTLE, 100, 0.95, 2, false, Mat())); + train_and_print_errs(boost, data); } - else - printf("File can not be read"); + + printf("======RTREES=====\n"); + Ptr rtrees = RTrees::create(RTrees::Params(10, 2, 0, false, 16, Mat(), false, 0, TermCriteria(TermCriteria::MAX_ITER, 100, 0))); + train_and_print_errs(rtrees, data); return 0; } diff --git a/samples/cpp/tutorial_code/ml/introduction_to_svm/introduction_to_svm.cpp b/samples/cpp/tutorial_code/ml/introduction_to_svm/introduction_to_svm.cpp index 2b4a97d54dd8c02e52271ea287074b100b57c204..f261418043d61de6d7045e7ce21d369b3808f6a1 100644 --- a/samples/cpp/tutorial_code/ml/introduction_to_svm/introduction_to_svm.cpp +++ b/samples/cpp/tutorial_code/ml/introduction_to_svm/introduction_to_svm.cpp @@ -4,29 +4,29 @@ #include using namespace cv; +using namespace cv::ml; -int main() +int main(int, char**) { // Data for visual representation int width = 512, height = 512; Mat image = Mat::zeros(height, width, CV_8UC3); // Set up training data - float labels[4] = {1.0, -1.0, -1.0, -1.0}; - Mat labelsMat(4, 1, CV_32FC1, labels); + int labels[4] = {1, -1, -1, -1}; + Mat labelsMat(4, 1, CV_32SC1, labels); float trainingData[4][2] = { {501, 10}, {255, 10}, {501, 255}, {10, 501} }; Mat trainingDataMat(4, 2, CV_32FC1, trainingData); // Set up SVM's parameters - CvSVMParams params; - params.svm_type = CvSVM::C_SVC; - params.kernel_type = CvSVM::LINEAR; - params.term_crit = cvTermCriteria(CV_TERMCRIT_ITER, 100, 1e-6); + SVM::Params params; + params.svmType = SVM::C_SVC; + params.kernelType = SVM::LINEAR; + params.termCrit = TermCriteria(TermCriteria::MAX_ITER, 100, 1e-6); // Train the SVM - CvSVM SVM; - SVM.train(trainingDataMat, labelsMat, Mat(), Mat(), params); + Ptr svm = StatModel::train(trainingDataMat, ROW_SAMPLE, labelsMat, params); Vec3b green(0,255,0), blue (255,0,0); // Show the decision regions given by the SVM @@ -34,30 +34,30 @@ int main() for (int j = 0; j < image.cols; ++j) { Mat sampleMat = (Mat_(1,2) << j,i); - float response = SVM.predict(sampleMat); + float response = svm->predict(sampleMat); if (response == 1) image.at(i,j) = green; else if (response == -1) - image.at(i,j) = blue; + image.at(i,j) = blue; } // Show the training data int thickness = -1; int lineType = 8; - circle( image, Point(501, 10), 5, Scalar( 0, 0, 0), thickness, lineType); - circle( image, Point(255, 10), 5, Scalar(255, 255, 255), thickness, lineType); - circle( image, Point(501, 255), 5, Scalar(255, 255, 255), thickness, lineType); - circle( image, Point( 10, 501), 5, Scalar(255, 255, 255), thickness, lineType); + circle( image, Point(501, 10), 5, Scalar( 0, 0, 0), thickness, lineType ); + circle( image, Point(255, 10), 5, Scalar(255, 255, 255), thickness, lineType ); + circle( image, Point(501, 255), 5, Scalar(255, 255, 255), thickness, lineType ); + circle( image, Point( 10, 501), 5, Scalar(255, 255, 255), thickness, lineType ); // Show support vectors thickness = 2; lineType = 8; - int c = SVM.get_support_vector_count(); + Mat sv = svm->getSupportVectors(); - for (int i = 0; i < c; ++i) + for (int i = 0; i < sv.rows; ++i) { - const float* v = SVM.get_support_vector(i); + const float* v = sv.ptr(i); circle( image, Point( (int) v[0], (int) v[1]), 6, Scalar(128, 128, 128), thickness, lineType); } diff --git a/samples/cpp/tutorial_code/ml/non_linear_svms/non_linear_svms.cpp b/samples/cpp/tutorial_code/ml/non_linear_svms/non_linear_svms.cpp index bfab746cdf938514f12e938473df3783cb070ba6..3e7cdb3a4e02e293833112da9f192bb14bde7e91 100644 --- a/samples/cpp/tutorial_code/ml/non_linear_svms/non_linear_svms.cpp +++ b/samples/cpp/tutorial_code/ml/non_linear_svms/non_linear_svms.cpp @@ -8,6 +8,7 @@ #define FRAC_LINEAR_SEP 0.9f // Fraction of samples which compose the linear separable part using namespace cv; +using namespace cv::ml; using namespace std; static void help() @@ -30,7 +31,7 @@ int main() //--------------------- 1. Set up training data randomly --------------------------------------- Mat trainData(2*NTRAINING_SAMPLES, 2, CV_32FC1); - Mat labels (2*NTRAINING_SAMPLES, 1, CV_32FC1); + Mat labels (2*NTRAINING_SAMPLES, 1, CV_32SC1); RNG rng(100); // Random value generation class @@ -71,16 +72,15 @@ int main() labels.rowRange(NTRAINING_SAMPLES, 2*NTRAINING_SAMPLES).setTo(2); // Class 2 //------------------------ 2. Set up the support vector machines parameters -------------------- - CvSVMParams params; - params.svm_type = SVM::C_SVC; + SVM::Params params; + params.svmType = SVM::C_SVC; params.C = 0.1; - params.kernel_type = SVM::LINEAR; - params.term_crit = TermCriteria(CV_TERMCRIT_ITER, (int)1e7, 1e-6); + params.kernelType = SVM::LINEAR; + params.termCrit = TermCriteria(TermCriteria::MAX_ITER, (int)1e7, 1e-6); //------------------------ 3. Train the svm ---------------------------------------------------- cout << "Starting training process" << endl; - CvSVM svm; - svm.train(trainData, labels, Mat(), Mat(), params); + Ptr svm = StatModel::train(trainData, ROW_SAMPLE, labels, params); cout << "Finished training process" << endl; //------------------------ 4. Show the decision regions ---------------------------------------- @@ -89,7 +89,7 @@ int main() for (int j = 0; j < I.cols; ++j) { Mat sampleMat = (Mat_(1,2) << i, j); - float response = svm.predict(sampleMat); + float response = svm->predict(sampleMat); if (response == 1) I.at(j, i) = green; else if (response == 2) I.at(j, i) = blue; @@ -117,11 +117,11 @@ int main() //------------------------- 6. Show support vectors -------------------------------------------- thick = 2; lineType = 8; - int x = svm.get_support_vector_count(); + Mat sv = svm->getSupportVectors(); - for (int i = 0; i < x; ++i) + for (int i = 0; i < sv.rows; ++i) { - const float* v = svm.get_support_vector(i); + const float* v = sv.ptr(i); circle( I, Point( (int) v[0], (int) v[1]), 6, Scalar(128, 128, 128), thick, lineType); } diff --git a/samples/cpp/tutorial_code/photo/decolorization/decolor.cpp b/samples/cpp/tutorial_code/photo/decolorization/decolor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..067bad1178f2e81fd0818d0a0ebb5181124b9367 --- /dev/null +++ b/samples/cpp/tutorial_code/photo/decolorization/decolor.cpp @@ -0,0 +1,40 @@ +/* +* decolor.cpp +* +* Author: +* Siddharth Kherada +* +* This tutorial demonstrates how to use OpenCV Decolorization Module. +* +* Input: +* Color Image +* +* Output: +* 1) Grayscale image +* 2) Color boost image +* +*/ + +#include "opencv2/photo.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/core.hpp" +#include + +using namespace std; +using namespace cv; + +int main(int argc, char *argv[]) +{ + CV_Assert(argc == 2); + Mat I; + I = imread(argv[1]); + + Mat gray = Mat(I.size(),CV_8UC1); + Mat color_boost = Mat(I.size(),CV_8UC3); + + decolor(I,gray,color_boost); + imshow("grayscale",gray); + imshow("color_boost",color_boost); + waitKey(0); +} diff --git a/samples/cpp/tutorial_code/photo/non_photorealistic_rendering/npr_demo.cpp b/samples/cpp/tutorial_code/photo/non_photorealistic_rendering/npr_demo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5579ca269f2a90a0b94753d696ee96fa11bb963e --- /dev/null +++ b/samples/cpp/tutorial_code/photo/non_photorealistic_rendering/npr_demo.cpp @@ -0,0 +1,96 @@ +/* +* npr_demo.cpp +* +* Author: +* Siddharth Kherada +* +* This tutorial demonstrates how to use OpenCV Non-Photorealistic Rendering Module. +* 1) Edge Preserve Smoothing +* -> Using Normalized convolution Filter +* -> Using Recursive Filter +* 2) Detail Enhancement +* 3) Pencil sketch/Color Pencil Drawing +* 4) Stylization +* +*/ + +#include +#include "opencv2/photo.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/core.hpp" +#include +#include + +using namespace std; +using namespace cv; + +int main(int argc, char* argv[]) +{ + if(argc < 2) + { + cout << "usage: " << argv[0] << " " << endl; + exit(0); + } + + int num,type; + + Mat I = imread(argv[1]); + + if(!I.data) + { + cout << "Image not found" << endl; + exit(0); + } + + cout << endl; + cout << " Edge Preserve Filter" << endl; + cout << "----------------------" << endl; + + cout << "Options: " << endl; + cout << endl; + + cout << "1) Edge Preserve Smoothing" << endl; + cout << " -> Using Normalized convolution Filter" << endl; + cout << " -> Using Recursive Filter" << endl; + cout << "2) Detail Enhancement" << endl; + cout << "3) Pencil sketch/Color Pencil Drawing" << endl; + cout << "4) Stylization" << endl; + cout << endl; + + cout << "Press number 1-4 to choose from above techniques: "; + + cin >> num; + + Mat img; + + if(num == 1) + { + cout << endl; + cout << "Press 1 for Normalized Convolution Filter and 2 for Recursive Filter: "; + + cin >> type; + + edgePreservingFilter(I,img,type); + imshow("Edge Preserve Smoothing",img); + + } + else if(num == 2) + { + detailEnhance(I,img); + imshow("Detail Enhanced",img); + } + else if(num == 3) + { + Mat img1; + pencilSketch(I,img1, img, 10 , 0.1f, 0.03f); + imshow("Pencil Sketch",img1); + imshow("Color Pencil Sketch",img); + } + else if(num == 4) + { + stylization(I,img); + imshow("Stylization",img); + } + waitKey(0); +} diff --git a/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_demo.cpp b/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_demo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..24d9b7facfb762907b68b4ff3d42ef8169689002 --- /dev/null +++ b/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_demo.cpp @@ -0,0 +1,246 @@ +/* +* cloning_demo.cpp +* +* Author: +* Siddharth Kherada +* +* This tutorial demonstrates how to use OpenCV seamless cloning +* module without GUI. +* +* 1- Normal Cloning +* 2- Mixed Cloning +* 3- Monochrome Transfer +* 4- Color Change +* 5- Illumination change +* 6- Texture Flattening + +* The program takes as input a source and a destination image (for 1-3 methods) +* and ouputs the cloned image. +* +* Download test images from opencv_extra folder @github. +* +*/ + +#include "opencv2/photo.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/core.hpp" +#include +#include + +using namespace std; +using namespace cv; + +int main() +{ + cout << endl; + cout << "Cloning Module" << endl; + cout << "---------------" << endl; + cout << "Options: " << endl; + cout << endl; + cout << "1) Normal Cloning " << endl; + cout << "2) Mixed Cloning " << endl; + cout << "3) Monochrome Transfer " << endl; + cout << "4) Local Color Change " << endl; + cout << "5) Local Illumination Change " << endl; + cout << "6) Texture Flattening " << endl; + cout << endl; + cout << "Press number 1-6 to choose from above techniques: "; + int num = 1; + cin >> num; + cout << endl; + + if(num == 1) + { + string folder = "cloning/Normal_Cloning/"; + string original_path1 = folder + "source1.png"; + string original_path2 = folder + "destination1.png"; + string original_path3 = folder + "mask.png"; + + Mat source = imread(original_path1, IMREAD_COLOR); + Mat destination = imread(original_path2, IMREAD_COLOR); + Mat mask = imread(original_path3, IMREAD_COLOR); + + if(source.empty()) + { + cout << "Could not load source image " << original_path1 << endl; + exit(0); + } + if(destination.empty()) + { + cout << "Could not load destination image " << original_path2 << endl; + exit(0); + } + if(mask.empty()) + { + cout << "Could not load mask image " << original_path3 << endl; + exit(0); + } + + Mat result; + Point p; + p.x = 400; + p.y = 100; + + seamlessClone(source, destination, mask, p, result, 1); + + imshow("Output",result); + imwrite(folder + "cloned.png", result); + } + else if(num == 2) + { + string folder = "cloning/Mixed_Cloning/"; + string original_path1 = folder + "source1.png"; + string original_path2 = folder + "destination1.png"; + string original_path3 = folder + "mask.png"; + + Mat source = imread(original_path1, IMREAD_COLOR); + Mat destination = imread(original_path2, IMREAD_COLOR); + Mat mask = imread(original_path3, IMREAD_COLOR); + + if(source.empty()) + { + cout << "Could not load source image " << original_path1 << endl; + exit(0); + } + if(destination.empty()) + { + cout << "Could not load destination image " << original_path2 << endl; + exit(0); + } + if(mask.empty()) + { + cout << "Could not load mask image " << original_path3 << endl; + exit(0); + } + + Mat result; + Point p; + p.x = destination.size().width/2; + p.y = destination.size().height/2; + + seamlessClone(source, destination, mask, p, result, 2); + + imshow("Output",result); + imwrite(folder + "cloned.png", result); + } + else if(num == 3) + { + string folder = "cloning/Monochrome_Transfer/"; + string original_path1 = folder + "source1.png"; + string original_path2 = folder + "destination1.png"; + string original_path3 = folder + "mask.png"; + + Mat source = imread(original_path1, IMREAD_COLOR); + Mat destination = imread(original_path2, IMREAD_COLOR); + Mat mask = imread(original_path3, IMREAD_COLOR); + + if(source.empty()) + { + cout << "Could not load source image " << original_path1 << endl; + exit(0); + } + if(destination.empty()) + { + cout << "Could not load destination image " << original_path2 << endl; + exit(0); + } + if(mask.empty()) + { + cout << "Could not load mask image " << original_path3 << endl; + exit(0); + } + + Mat result; + Point p; + p.x = destination.size().width/2; + p.y = destination.size().height/2; + + seamlessClone(source, destination, mask, p, result, 3); + + imshow("Output",result); + imwrite(folder + "cloned.png", result); + } + else if(num == 4) + { + string folder = "cloning/Color_Change/"; + string original_path1 = folder + "source1.png"; + string original_path2 = folder + "mask.png"; + + Mat source = imread(original_path1, IMREAD_COLOR); + Mat mask = imread(original_path2, IMREAD_COLOR); + + if(source.empty()) + { + cout << "Could not load source image " << original_path1 << endl; + exit(0); + } + if(mask.empty()) + { + cout << "Could not load mask image " << original_path2 << endl; + exit(0); + } + + Mat result; + + colorChange(source, mask, result, 1.5, .5, .5); + + imshow("Output",result); + imwrite(folder + "cloned.png", result); + } + else if(num == 5) + { + string folder = "cloning/Illumination_Change/"; + string original_path1 = folder + "source1.png"; + string original_path2 = folder + "mask.png"; + + Mat source = imread(original_path1, IMREAD_COLOR); + Mat mask = imread(original_path2, IMREAD_COLOR); + + if(source.empty()) + { + cout << "Could not load source image " << original_path1 << endl; + exit(0); + } + if(mask.empty()) + { + cout << "Could not load mask image " << original_path2 << endl; + exit(0); + } + + Mat result; + + illuminationChange(source, mask, result, 0.2f, 0.4f); + + imshow("Output",result); + imwrite(folder + "cloned.png", result); + } + else if(num == 6) + { + string folder = "cloning/Texture_Flattening/"; + string original_path1 = folder + "source1.png"; + string original_path2 = folder + "mask.png"; + + Mat source = imread(original_path1, IMREAD_COLOR); + Mat mask = imread(original_path2, IMREAD_COLOR); + + if(source.empty()) + { + cout << "Could not load source image " << original_path1 << endl; + exit(0); + } + if(mask.empty()) + { + cout << "Could not load mask image " << original_path2 << endl; + exit(0); + } + + Mat result; + + textureFlattening(source, mask, result, 30, 45, 3); + + imshow("Output",result); + imwrite(folder + "cloned.png", result); + } + waitKey(0); +} diff --git a/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp b/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2457b1215478210a93b55930fc2ff93cf307ca33 --- /dev/null +++ b/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp @@ -0,0 +1,546 @@ +/* +* cloning.cpp +* +* Author: +* Siddharth Kherada +* +* This tutorial demonstrates how to use OpenCV seamless cloning +* module. +* +* 1- Normal Cloning +* 2- Mixed Cloning +* 3- Monochrome Transfer +* 4- Color Change +* 5- Illumination change +* 6- Texture Flattening + +* The program takes as input a source and a destination image (for 1-3 methods) +* and ouputs the cloned image. + +* Step 1: +* -> In the source image, select the region of interest by left click mouse button. A Polygon ROI will be created by left clicking mouse button. +* -> To set the Polygon ROI, click the right mouse button or 'd' key. +* -> To reset the region selected, click the middle mouse button or 'r' key. + +* Step 2: +* -> In the destination image, select the point where you want to place the ROI in the image by left clicking mouse button. +* -> To get the cloned result, click the right mouse button or 'c' key. +* -> To quit the program, use 'q' key. +* +* Result: The cloned image will be displayed. +*/ + +#include +#include "opencv2/photo.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/core.hpp" +#include +#include + +using namespace std; +using namespace cv; + +Mat img0, img1, img2, res, res1, final, final1, blend; + +Point point; +int drag = 0; +int destx, desty; + +int numpts = 100; +Point* pts = new Point[100]; +Point* pts2 = new Point[100]; +Point* pts_diff = new Point[100]; + +int var = 0; +int flag = 0, flag1 = 0, flag4 = 0; + +int minx, miny, maxx, maxy, lenx, leny; +int minxd, minyd, maxxd, maxyd, lenxd, lenyd; + +int channel, num, kernel_size; + +float alpha,beta; + +float red, green, blue; + +double low_t, high_t; + +void source(int, int, int, int, void*); +void destination(int, int, int, int, void*); +void checkfile(char*); + +void source(int event, int x, int y, int, void*) +{ + + if (event == EVENT_LBUTTONDOWN && !drag) + { + if(flag1 == 0) + { + if(var==0) + img1 = img0.clone(); + point = Point(x, y); + circle(img1,point,2,Scalar(0, 0, 255),-1, 8, 0); + pts[var] = point; + var++; + drag = 1; + if(var>1) + line(img1,pts[var-2], point, Scalar(0, 0, 255), 2, 8, 0); + + imshow("Source", img1); + } + } + + if (event == EVENT_LBUTTONUP && drag) + { + imshow("Source", img1); + + drag = 0; + } + if (event == EVENT_RBUTTONDOWN) + { + flag1 = 1; + img1 = img0.clone(); + for(int i = var; i < numpts ; i++) + pts[i] = point; + + if(var!=0) + { + const Point* pts3[1] = {&pts[0]}; + polylines( img1, pts3, &numpts,1, 1, Scalar(0,0,0), 2, 8, 0); + } + + for(int i=0;i im1.size().width || maxyd > im1.size().height || minxd < 0 || minyd < 0) + { + cout << "Index out of range" << endl; + exit(0); + } + + final1 = Mat::zeros(img2.size(),CV_8UC3); + res = Mat::zeros(img2.size(),CV_8UC1); + for(int i=miny, k=minyd;i<(miny+leny);i++,k++) + for(int j=minx,l=minxd ;j<(minx+lenx);j++,l++) + { + for(int c=0;c(k,l*channel+c) = final.at(i,j*channel+c); + + } + } + + const Point* pts6[1] = {&pts2[0]}; + fillPoly(res, pts6, &numpts, 1, Scalar(255, 255, 255), 8, 0); + + if(num == 1 || num == 2 || num == 3) + { + seamlessClone(img0,img2,res1,point,blend,num); + imshow("Cloned Image", blend); + imwrite("cloned.png",blend); + waitKey(0); + } + + for(int i = 0; i < flag ; i++) + { + pts2[i].x=0; + pts2[i].y=0; + } + + minxd = INT_MAX; minyd = INT_MAX; maxxd = INT_MIN; maxyd = INT_MIN; + } + + im1.release(); +} + +int main() +{ + cout << endl; + cout << "Cloning Module" << endl; + cout << "---------------" << endl; + cout << "Step 1:" << endl; + cout << " -> In the source image, select the region of interest by left click mouse button. A Polygon ROI will be created by left clicking mouse button." << endl; + cout << " -> To set the Polygon ROI, click the right mouse button or use 'd' key" << endl; + cout << " -> To reset the region selected, click the middle mouse button or use 'r' key." << endl; + + cout << "Step 2:" << endl; + cout << " -> In the destination image, select the point where you want to place the ROI in the image by left clicking mouse button." << endl; + cout << " -> To get the cloned result, click the right mouse button or use 'c' key." << endl; + cout << " -> To quit the program, use 'q' key." << endl; + cout << endl; + cout << "Options: " << endl; + cout << endl; + cout << "1) Normal Cloning " << endl; + cout << "2) Mixed Cloning " << endl; + cout << "3) Monochrome Transfer " << endl; + cout << "4) Local Color Change " << endl; + cout << "5) Local Illumination Change " << endl; + cout << "6) Texture Flattening " << endl; + + cout << endl; + + cout << "Press number 1-6 to choose from above techniques: "; + cin >> num; + cout << endl; + + minx = INT_MAX; miny = INT_MAX; maxx = INT_MIN; maxy = INT_MIN; + + minxd = INT_MAX; minyd = INT_MAX; maxxd = INT_MIN; maxyd = INT_MIN; + + int flag3 = 0; + + if(num == 1 || num == 2 || num == 3) + { + + string src,dest; + cout << "Enter Source Image: "; + cin >> src; + + cout << "Enter Destination Image: "; + cin >> dest; + + img0 = imread(src); + + img2 = imread(dest); + + if(!img0.data) + { + cout << "Source Image does not exist" << endl; + exit(0); + } + if(!img2.data) + { + cout << "Destination Image does not exist" << endl; + exit(0); + } + + channel = img0.channels(); + + res = Mat::zeros(img2.size(),CV_8UC1); + res1 = Mat::zeros(img0.size(),CV_8UC1); + final = Mat::zeros(img0.size(),CV_8UC3); + final1 = Mat::zeros(img2.size(),CV_8UC3); + //////////// source image /////////////////// + + namedWindow("Source", 1); + setMouseCallback("Source", source, NULL); + imshow("Source", img0); + + /////////// destination image /////////////// + + namedWindow("Destination", 1); + setMouseCallback("Destination", destination, NULL); + imshow("Destination",img2); + + } + else if(num == 4) + { + string src; + cout << "Enter Source Image: "; + cin >> src; + + cout << "Enter RGB values: " << endl; + cout << "Red: "; + cin >> red; + + cout << "Green: "; + cin >> green; + + cout << "Blue: "; + cin >> blue; + + img0 = imread(src); + + if(!img0.data) + { + cout << "Source Image does not exist" << endl; + exit(0); + } + + res1 = Mat::zeros(img0.size(),CV_8UC1); + final = Mat::zeros(img0.size(),CV_8UC3); + + //////////// source image /////////////////// + + namedWindow("Source", 1); + setMouseCallback("Source", source, NULL); + imshow("Source", img0); + + } + else if(num == 5) + { + string src; + cout << "Enter Source Image: "; + cin >> src; + + cout << "alpha: "; + cin >> alpha; + + cout << "beta: "; + cin >> beta; + + img0 = imread(src); + + if(!img0.data) + { + cout << "Source Image does not exist" << endl; + exit(0); + } + + res1 = Mat::zeros(img0.size(),CV_8UC1); + final = Mat::zeros(img0.size(),CV_8UC3); + + //////////// source image /////////////////// + + namedWindow("Source", 1); + setMouseCallback("Source", source, NULL); + imshow("Source", img0); + + } + else if(num == 6) + { + string src; + cout << "Enter Source Image: "; + cin >> src; + + cout << "low_threshold: "; + cin >> low_t; + + cout << "high_threshold: "; + cin >> high_t; + + cout << "kernel_size: "; + cin >> kernel_size; + + img0 = imread(src); + + if(!img0.data) + { + cout << "Source Image does not exist" << endl; + exit(0); + } + + res1 = Mat::zeros(img0.size(),CV_8UC1); + final = Mat::zeros(img0.size(),CV_8UC3); + + //////////// source image /////////////////// + + namedWindow("Source", 1); + setMouseCallback("Source", source, NULL); + imshow("Source", img0); + } + else + { + cout << "Wrong Option Choosen" << endl; + exit(0); + } + + for(;;) + { + char key = (char) waitKey(0); + + if(key == 'd' && flag3 == 0) + { + flag1 = 1; + flag3 = 1; + img1 = img0.clone(); + for(int i = var; i < numpts ; i++) + pts[i] = point; + + if(var!=0) + { + const Point* pts3[1] = {&pts[0]}; + polylines( img1, pts3, &numpts,1, 1, Scalar(0,0,0), 2, 8, 0); + } + + for(int i=0;i