//    module SetResult.h                                                      //
//                                                                            //
//    A) TParaSetResults encapsulates loss function results and model         //
//       characteristics for amodel learned and tested several times on       //
//       different data.                                                      //
//                                                                            //
//    B) Class TParaSetResults encapsulates several TParaSet objects, i.e.    //
//       stores the results of several parameter sets used to learn and test  //
//       a model on several different data sets.                              //
//                                                                            //
//    copyright (c) 2001-2003 by Lars Haendel                                 //
//    mail: lore17@newty.de                                                   //
//    home: www.newty.de                                                      //
//                                                                            //
//    This program is free software; you can redistribute it and/or modify    //
//    it under the terms of the GNU General Public License as published by    //
//    the Free Software Foundation as version 2 of the License.               //
//                                                                            //
//    This program is distributed in the hope that it will be useful,         //
//    but WITHOUT ANY WARRANTY; without even the implied warranty of          //
//    GNU General Public License for more details.                            //
//                                                                            //
//    You should have received a copy of the GNU General Public License       //
//    along with this program; if not, write to the Free Software             //
//    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.               //
//                                                                            //


#include <fstream>         // due to:  ofstream
#include <time>            //          clock_t

#include "lossfunc.h"      //          TLossFunction
#include "util.h"          //          TSorter
#include "ParaSetList.h"   //          TParaSetList

// functions to convert loss result/criterion Id's to strings (criterion names) and vice versa
#define nCriterion 21                                          // # known criterions
int         StringToCriterionId(const char* szString);         // convert criterion name to criterion Id
const char* CriterionIdToString(const int& id);                // convert criterion Id to criterion name
bool        LossOnLearnData(const bool*const& criterion);      // check if loss on learn data needs to be calculated
                                                               // for given criterions enabled

//  class encapsulates loss function and some other results for one parameter set, i.e. the mean and single results
//  obtained in 'nTest' repetitions on different learn and test data
class TParaSetResult

   // constructor/destructor
   TParaSetResult(const int& __nTest, const bool*const& f_crit_to_calc);

   // write description for specified results
   static void WriteResultDescriptions(ofstream& file, const bool*const& f_Criterion, const bool& f_ShowDev=false
                                       , const bool& f_Objective=false);

   // write deviation
   void WriteDeviation(const int& criterion, ofstream& file);

   // write specified results
   void Write(ofstream& file, const bool criterion[], const bool& f_show_dev, const int& testId=-1);

   void Calculate();                                                 // calculate mean and deviation
   float Std   (const int& criterion) { return std[criterion]; };    // return deviation

   // return specified result, either the 'testId'-th one or the mean value
   float Result(const int& criterion, const int& testId=-1);

   // start, stop and get time used to learn or to test
   void StartClock(const bool& test, const int& testId);
   void StopClock (const bool& test, const int& testId);
   clock_t& GetClock(const bool& test, const int& testId=-1);

   inline const int& nTest() { return _nTest; };      // return # tests

   // return reference to result of 'testId'-th test
   void SetLossObjectTest (const int& testId, TLossFunction*& loss)
      { CheckId(testId); SetLossObjectErrorIfTrue(loss_T[testId]!=NULL); loss_T[testId] = loss; };
   void SetLossObjectLearn(const int& testId, TLossFunction*& loss)
      { CheckId(testId); SetLossObjectErrorIfTrue(loss_L[testId]!=NULL); loss_L[testId] = loss; };

   TLossFunction*&  GetLossObjectTest (const int& testId) { CheckId(testId); return loss_T[testId]; };
   TLossFunction*&  GetLossObjectLearn(const int& testId) { CheckId(testId); return loss_L[testId]; };

   float& nBounds  (const int& testId) { CheckId(testId); return _nBounds[testId]; };
   float& HitRate  (const int& testId) { CheckId(testId); return rate[testId]; };
   float& SizeFac  (const int& testId) { CheckId(testId); return sizeFac[testId]; };
   int&   nCuboids (const int& testId) { CheckId(testId); return _nCuboids[testId]; };
   int&   nCuboidsK(const int& testId) { CheckId(testId); return _nCuboidsK[testId]; };

   // write loss
   const char* ResultToText(const int& criterion, const int& width, const bool& f_Weight=false, const int& t=-1);

   bool IsInitialized() const { return loss_T[_nTest-1]; }; // last one


   friend class TParaSetResults;
   float& Rank     (const int& testId) { CheckId(testId); return rank[testId]; };
   float& pVal     (const int& testId) { CheckId(testId); return _pVal[testId]; };

   // calculate mean and deviation of rank and pVal
   void Calculate2();

   // error functions
   void CheckId(const int& testId);
   void SetLossObjectErrorIfTrue(const bool f_error);
   void Error();
   void CheckCriterion(const int& criterion);
   void CheckValue(const float& value, const int& criterion, const bool& f_mean=false);

   int _nTest;                                  // # tests, i.e. # cross-validations/repetitions
   bool f_crit_to_calc[nCriterion];             // criterions for which the mean values will be calc. in Calculate()

   TLossFunction**   loss_T;                    // loss function results test data
   TLossFunction**   loss_L;                    // loss function results on learn data
   float             *_pVal, *rank;             // statistics comparing this parameter set to others
   float*            _nBounds;                  // mean # of active bounds averaged over all cuboids
   float*            rate;                      // average hitrate
   float*            sizeFac;                   // weighting factor regarding model's size
   int*              _nCuboids;                 // # cuboids in learned model
   int*              _nCuboidsK;                // # cuboids whose mass exceeds treshold

   clock_t           *times_L, *times_T;        // learn and test times

   // mean values and deviation estimated over all tests
   clock_t time_L, time_T;
   float mean[nCriterion];                      // mean error criterion
   float std[nCriterion];                       // deviation of error criterion

//  class encapsulates several TParaSetResult objects, i.e. it stores the results of 'nSets' different parameter sets
class TParaSetResults

   // constructor/destructor
   TParaSetResults(const int& _nSets, const int& _nTest, const bool& _f_regression, const bool*const& f_crit_to_calc);

   inline const int& nSets()  { return _nSets;  };
   inline const int& nTest()  { return _nTest;  };

   TParaSetResult*& Get(const int& setId) { CheckId(setId); return res[setId];};    // get setId'th results

   void  CalculateRankAndPVal(const int& ranking);
   int   FindBest(const int& objective);
   void  Write(ofstream& file, const bool*const& f_Criterion, const int& sort, const TParaSetList*const& para
               , const bool& f_ShowDev);


   void CheckId(const int& setId);

   // get parameter set id's sorted regarding specified criterion (eventually weighted by model's size)
   TSorter* SortedId(const int& sort, const bool& f_weight, const int& testId=-1);

   TParaSetResult** res;

   bool f_CritToCalc[nCriterion];      // criterions to calculate
   int  _nSets;                        // # different parameter sets
   int  _nTest;                        // # tests done for each set
   bool f_Regression;                  // flag indicates regression task