|
PLearn 0.1
|
#include <SDBWithStats.h>


Public Member Functions | |
| int | nfields () |
| string | fieldname (int i) |
| SDBWithStats (string basename, string path=".", AccessType access=readwrite, bool verbose=true) | |
| void | forgetStats () |
| void | computeStats (unsigned int nrows) |
| void | computeStats () |
| bool | hasStats () |
| returns true if the stats files exist | |
| void | saveStats () |
| void | loadStats () |
| FieldStat & | getStat (int i) |
| const FieldStat & | getStat (int i) const |
| FieldStat & | getStat (const string &fieldname) |
| const FieldStat & | getStat (const string &fieldname) const |
Public Attributes | |
| vector< FieldStat > | fieldstat |
Definition at line 106 of file SDBWithStats.h.
| PLearn::SDBWithStats::SDBWithStats | ( | string | basename, |
| string | path = ".", |
||
| AccessType | access = readwrite, |
||
| bool | verbose = true |
||
| ) |
Definition at line 98 of file SDBWithStats.cc.
References fieldstat, PLearn::SimpleDB< KeyType, QueryResult >::getSchema(), hasStats(), loadStats(), and PLearn::SimpleDB< KeyType, QueryResult >::size().
:SDB(basename,path,access,verbose) { fieldstat.resize(getSchema().size()); if(hasStats()) loadStats(); }

| void PLearn::SDBWithStats::computeStats | ( | unsigned int | nrows | ) |
Definition at line 112 of file SDBWithStats.cc.
References PLearn::Row::begin(), PLearn::CharacterType, PLearn::DateType, PLearn::DoubleType, PLearn::endl(), fieldstat, PLearn::FloatType, forgetStats(), PLearn::RowIterator::getFieldType(), PLearn::SimpleDB< KeyType, QueryResult >::getInRow(), PLearn::SimpleDB< KeyType, QueryResult >::getSchema(), i, PLearn::IntType, PLearn::RowIterator::isMissing(), j, nfields(), PLERROR, PLearn::ShortType, PLearn::SignedCharType, PLearn::StringType, PLearn::todouble(), and PLearn::tostring().
Referenced by PLearn::SDBVMatrix::SDBVMatrix().
{
forgetStats();
const Schema& sc = getSchema();
Row row(&sc);
for(SDB::RowNumber i=0; i<nrows;i++)
{
getInRow(i, row);
Row::const_iterator it = row.begin();
if (nrows>100000 && i%100000==0)
cout << "SDBWithStats::computeStats processing row " << i << " of " << nrows << endl;
for (int j=0; j<nfields(); ++j, ++it)
{
if(it.isMissing())
fieldstat[j].updateMissing();
else
{
switch(it.getFieldType())
{
case StringType:
case CharacterType:
fieldstat[j].updateString(tostring(it));
break;
case SignedCharType:
case ShortType:
case IntType:
case FloatType:
case DoubleType:
case DateType:
fieldstat[j].updateNumber(todouble(it));
break;
default:
PLERROR("Unknown field type");
}
}
}
}
cout << "boucle terminee" << endl;
for (int j=0; j<nfields(); ++j)
fieldstat[j].finalize();
cout << "fini computestats" << endl;
}


| void PLearn::SDBWithStats::computeStats | ( | ) | [inline] |
Definition at line 119 of file SDBWithStats.h.
References computeStats().
Referenced by computeStats().
{ computeStats(size()); }


| string PLearn::SDBWithStats::fieldname | ( | int | i | ) | [inline] |
Definition at line 111 of file SDBWithStats.h.
References i.
Referenced by loadStats(), and saveStats().

| void PLearn::SDBWithStats::forgetStats | ( | ) |
Definition at line 106 of file SDBWithStats.cc.
References PLearn::FieldStat::clear(), getStat(), j, and PLearn::SimpleDB< KeyType, QueryResult >::width().
Referenced by computeStats(), and loadStats().


| FieldStat & PLearn::SDBWithStats::getStat | ( | const string & | fieldname | ) |
Definition at line 268 of file SDBWithStats.cc.
References getStat(), PLearn::SimpleDB< KeyType, QueryResult >::indexOfField(), and PLERROR.
{
int pos = indexOfField(fieldname);
if(pos<0)
PLERROR("No field named %s",fieldname.c_str());
return getStat(pos);
}

Definition at line 254 of file SDBWithStats.cc.
References fieldstat, i, PLERROR, and PLearn::SimpleDB< KeyType, QueryResult >::width().
Referenced by forgetStats(), PLearn::SDBVMSource::getFieldStat(), and getStat().


Definition at line 261 of file SDBWithStats.cc.
References fieldstat, i, PLERROR, and PLearn::SimpleDB< KeyType, QueryResult >::width().

| const FieldStat & PLearn::SDBWithStats::getStat | ( | const string & | fieldname | ) | const |
Definition at line 276 of file SDBWithStats.cc.
References getStat(), PLearn::SimpleDB< KeyType, QueryResult >::indexOfField(), and PLERROR.
{
int pos = indexOfField(fieldname);
if(pos<0)
PLERROR("No field named %s",fieldname.c_str());
return getStat(pos);
}

| bool PLearn::SDBWithStats::hasStats | ( | ) |
returns true if the stats files exist
Definition at line 157 of file SDBWithStats.cc.
References PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), and PLearn::isfile().
Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and SDBWithStats().
{
string numstatsfile = getPath()+getName()+".stats";
string symstatsfile = getPath()+getName()+".symbols";
return isfile(numstatsfile) && isfile(symstatsfile);
}


| void PLearn::SDBWithStats::loadStats | ( | ) |
Definition at line 195 of file SDBWithStats.cc.
References fieldname(), fieldstat, forgetStats(), PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), j, PLearn::FieldStat::max_, PLearn::FieldStat::mean_, PLearn::FieldStat::min_, PLearn::FieldStat::missing_, PLearn::SimpleDB< KeyType, QueryResult >::name, PLearn::FieldStat::nonmissing_, PLERROR, PLearn::FieldStat::stddev_, PLearn::FieldStat::symbolcount, PLearn::FieldStat::symbolid, PLearn::tofloat(), and PLearn::toint().
Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and SDBWithStats().
{
forgetStats();
string numstatsfile = getPath()+getName()+".stats";
string symstatsfile = getPath()+getName()+".symbols";
ifstream numstats(numstatsfile.c_str());
if(!numstats)
PLERROR("could not open file %s for reading",numstatsfile.c_str());
ifstream symstats(symstatsfile.c_str());
if(!symstats)
PLERROR("could not open file %s for reading",symstatsfile.c_str());
for(unsigned int j=0; j<fieldstat.size(); j++)
{
string str_nonmissing_,str_missing_,str_mean_,str_stddev_,str_min_,str_max_;
FieldStat& s = fieldstat[j];
string name;
numstats >> name;
//cout<<"field : "<<j<<" name:"<<name<<" fieldname(j):"<<fieldname(j)<<endl;
if(name!=fieldname(j))
PLERROR("Row number %d of file %s does not correpond to field number %d: %s",j,numstatsfile.c_str(),j,fieldname(j).c_str());
// **** we use strings as intermediate type to handle nans (julien)
numstats >> str_nonmissing_ >> str_missing_ >> str_mean_ >> str_stddev_ >> str_min_ >> str_max_;
if(str_mean_=="nan")str_mean_="";
if(str_stddev_=="nan")str_stddev_="";
if(str_min_=="nan")str_min_="";
if(str_max_=="nan")str_max_="";
s.nonmissing_ = toint(str_nonmissing_);
s.missing_ = toint(str_missing_);
s.mean_ = tofloat(str_mean_);
s.stddev_ = tofloat(str_stddev_);
s.min_ = tofloat(str_min_);
s.max_ = tofloat(str_max_);
//cout <<" "<< s.nonmissing_ <<" "<< s.missing_ <<" "<< s.mean_ <<" "<< s.stddev_ <<" "<< s.min_ <<" "<< s.max_<<endl;
symstats >> name;
if(name!=fieldname(j))
PLERROR("Row number %d of file %s does not correpond to field number %d: %s",j,symstatsfile.c_str(),j,fieldname(j).c_str());
int nsymbols;
symstats >> nsymbols;
string sym;
int symcount;
for(int k=0; k<nsymbols; k++)
{
symstats >> sym >> symcount;
s.symbolcount[sym] = symcount;
s.symbolid[sym]=k;
}
}
}


| int PLearn::SDBWithStats::nfields | ( | ) | [inline] |
Definition at line 110 of file SDBWithStats.h.
Referenced by computeStats().
{ return (int)getSchema().size(); }

| void PLearn::SDBWithStats::saveStats | ( | ) |
Definition at line 164 of file SDBWithStats.cc.
References PLearn::endl(), fieldname(), fieldstat, PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), j, PLearn::FieldStat::max(), PLearn::FieldStat::mean(), PLearn::FieldStat::min(), PLearn::FieldStat::missing(), PLearn::FieldStat::nonmissing(), PLearn::FieldStat::nsymbols(), PLERROR, PLearn::FieldStat::stddev(), and PLearn::FieldStat::symbolcount.
Referenced by PLearn::SDBVMatrix::SDBVMatrix().
{
string numstatsfile = getPath()+getName()+".stats";
string symstatsfile = getPath()+getName()+".symbols";
ofstream numstats(numstatsfile.c_str());
if(!numstats)
PLERROR("could not open file %s for writing",numstatsfile.c_str());
ofstream symstats(symstatsfile.c_str());
if(!symstats)
PLERROR("could not open file %s for writing",symstatsfile.c_str());
numstats.precision(8);
symstats.precision(8);
for(unsigned int j=0; j<fieldstat.size(); j++)
{
FieldStat& s = fieldstat[j];
numstats << fieldname(j) << ' ' << s.nonmissing() << ' ' << s.missing() << ' '
<< s.mean() << ' ' << s.stddev() << ' ' << s.min() << ' ' << s.max() << endl;
symstats << fieldname(j) << ' ';
symstats << s.nsymbols() << " ";
map<string,int>::iterator it;
for(it = s.symbolcount.begin(); it!= s.symbolcount.end(); ++it)
symstats << it->first << ' ' << it->second << " ";
symstats << endl;
}
}


Definition at line 109 of file SDBWithStats.h.
Referenced by computeStats(), getStat(), loadStats(), saveStats(), and SDBWithStats().
1.7.4