//------------------------------------------------------------------------------
// module SetResult.h //
// //
// A) TParaSetResults encapsulates loss function results and model //
// characteristics for amodel learned and tested several times on //
// different data. //
// //
// B) Class TParaSetResults encapsulates several TParaSet objects, i.e. //
// stores the results of several parameter sets used to learn and test //
// a model on several different data sets. //
// //
// copyright (c) 2001-2003 by Lars Haendel //
// mail: lore17@newty.de //
// home: www.newty.de //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 2 of the License. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program; if not, write to the Free Software //
// Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. //
// //
//------------------------------------------------------------------------------
#ifndef SETRESULT_H
#define SETRESULT_H
//----------------------------------------------------------------------------------------------------------------------
#include <fstream> // due to: ofstream
#include <time> // clock_t
#include "lossfunc.h" // TLossFunction
#include "util.h" // TSorter
#include "ParaSetList.h" // TParaSetList
//----------------------------------------------------------------------------------------------------------------------
// functions to convert loss result/criterion Id's to strings (criterion names) and vice versa
#define nCriterion 21 // # known criterions
int StringToCriterionId(const char* szString); // convert criterion name to criterion Id
const char* CriterionIdToString(const int& id); // convert criterion Id to criterion name
bool LossOnLearnData(const bool*const& criterion); // check if loss on learn data needs to be calculated
// for given criterions enabled
//----------------------------------------------------------------------------------------------------------------------
// class encapsulates loss function and some other results for one parameter set, i.e. the mean and single results
// obtained in 'nTest' repetitions on different learn and test data
class TParaSetResult
{
public:
// constructor/destructor
TParaSetResult(const int& __nTest, const bool*const& f_crit_to_calc);
~TParaSetResult();
// write description for specified results
static void WriteResultDescriptions(ofstream& file, const bool*const& f_Criterion, const bool& f_ShowDev=false
, const bool& f_Objective=false);
// write deviation
void WriteDeviation(const int& criterion, ofstream& file);
// write specified results
void Write(ofstream& file, const bool criterion[], const bool& f_show_dev, const int& testId=-1);
void Calculate(); // calculate mean and deviation
float Std (const int& criterion) { return std[criterion]; }; // return deviation
// return specified result, either the 'testId'-th one or the mean value
float Result(const int& criterion, const int& testId=-1);
// start, stop and get time used to learn or to test
void StartClock(const bool& test, const int& testId);
void StopClock (const bool& test, const int& testId);
clock_t& GetClock(const bool& test, const int& testId=-1);
inline const int& nTest() { return _nTest; }; // return # tests
// return reference to result of 'testId'-th test
void SetLossObjectTest (const int& testId, TLossFunction*& loss)
{ CheckId(testId); SetLossObjectErrorIfTrue(loss_T[testId]!=NULL); loss_T[testId] = loss; };
void SetLossObjectLearn(const int& testId, TLossFunction*& loss)
{ CheckId(testId); SetLossObjectErrorIfTrue(loss_L[testId]!=NULL); loss_L[testId] = loss; };
TLossFunction*& GetLossObjectTest (const int& testId) { CheckId(testId); return loss_T[testId]; };
TLossFunction*& GetLossObjectLearn(const int& testId) { CheckId(testId); return loss_L[testId]; };
float& nBounds (const int& testId) { CheckId(testId); return _nBounds[testId]; };
float& HitRate (const int& testId) { CheckId(testId); return rate[testId]; };
float& SizeFac (const int& testId) { CheckId(testId); return sizeFac[testId]; };
int& nCuboids (const int& testId) { CheckId(testId); return _nCuboids[testId]; };
int& nCuboidsK(const int& testId) { CheckId(testId); return _nCuboidsK[testId]; };
// write loss
const char* ResultToText(const int& criterion, const int& width, const bool& f_Weight=false, const int& t=-1);
bool IsInitialized() const { return loss_T[_nTest-1]; }; // last one
private:
friend class TParaSetResults;
float& Rank (const int& testId) { CheckId(testId); return rank[testId]; };
float& pVal (const int& testId) { CheckId(testId); return _pVal[testId]; };
// calculate mean and deviation of rank and pVal
void Calculate2();
// error functions
void CheckId(const int& testId);
void SetLossObjectErrorIfTrue(const bool f_error);
void Error();
void CheckCriterion(const int& criterion);
void CheckValue(const float& value, const int& criterion, const bool& f_mean=false);
int _nTest; // # tests, i.e. # cross-validations/repetitions
bool f_crit_to_calc[nCriterion]; // criterions for which the mean values will be calc. in Calculate()
TLossFunction** loss_T; // loss function results test data
TLossFunction** loss_L; // loss function results on learn data
float *_pVal, *rank; // statistics comparing this parameter set to others
float* _nBounds; // mean # of active bounds averaged over all cuboids
float* rate; // average hitrate
float* sizeFac; // weighting factor regarding model's size
int* _nCuboids; // # cuboids in learned model
int* _nCuboidsK; // # cuboids whose mass exceeds treshold
clock_t *times_L, *times_T; // learn and test times
// mean values and deviation estimated over all tests
clock_t time_L, time_T;
float mean[nCriterion]; // mean error criterion
float std[nCriterion]; // deviation of error criterion
};
//----------------------------------------------------------------------------------------------------------------------
// class encapsulates several TParaSetResult objects, i.e. it stores the results of 'nSets' different parameter sets
class TParaSetResults
{
public:
// constructor/destructor
TParaSetResults(const int& _nSets, const int& _nTest, const bool& _f_regression, const bool*const& f_crit_to_calc);
~TParaSetResults();
inline const int& nSets() { return _nSets; };
inline const int& nTest() { return _nTest; };
TParaSetResult*& Get(const int& setId) { CheckId(setId); return res[setId];}; // get setId'th results
void CalculateRankAndPVal(const int& ranking);
int FindBest(const int& objective);
void Write(ofstream& file, const bool*const& f_Criterion, const int& sort, const TParaSetList*const& para
, const bool& f_ShowDev);
private:
void CheckId(const int& setId);
// get parameter set id's sorted regarding specified criterion (eventually weighted by model's size)
TSorter* SortedId(const int& sort, const bool& f_weight, const int& testId=-1);
TParaSetResult** res;
bool f_CritToCalc[nCriterion]; // criterions to calculate
int _nSets; // # different parameter sets
int _nTest; // # tests done for each set
bool f_Regression; // flag indicates regression task
};
#endif