Hi,
When you use ibis -r fbdir/:c2,c1 it only sorts the top level directory,
it does not descend into partitions and sort the partitions.
I modified ibis to use ibis::util::gatherParts to find all of the
partitions under the specified directory and sort them too.
One thing I noticed is that the reorder() call doesn't seem to return a
negative value on error. I removed a -part.txt file from one of my
partitions, and when I ran the reorder it says it failed in the logs, but a
negative number wasn't returned to reorder. I'll look into that further
when I get a chance.
Regards,
--- orig/fastbit-2.0.1/examples/ibis.cpp 2014-11-25 09:06:21.000000000 -0700
+++ fastbit-2.0.1/examples/ibis.cpp 2015-02-25 14:10:03.489628996 -0700
@@ -2539,41 +2539,48 @@
// reorder the data directories first, a data directory may be followed
// by ':' and column names
for (unsigned i = 0; i < rdirs.size(); ++ i) {
- long ierr = 0;
- LOGGER(ibis::gVerbose >= 0)
+ long ierr = 0;
+ LOGGER(ibis::gVerbose >= 0)
<< *argv << " -reorder " << rdirs[i];
- char* str = const_cast<char*>(strrchr(rdirs[i], ':'));
- if (str != 0 && str > rdirs[i] && str[1] != '/' && str[1] != '\\') {
- std::string dir;
- for (const char* tmp = rdirs[i]; tmp < str; ++ tmp)
- dir += *tmp;
- str = ibis::util::strnewdup(str+1);
- ibis::table::stringList slist;
- ibis::table::parseNames(str, slist);
- uint32_t nr = 0;
- {
- ibis::part tbl(dir.c_str(), static_cast<const char*>(0));
- ierr = tbl.reorder(slist);
- nr = tbl.nRows();
- }
- delete [] str;
- if ((long)nr == ierr && nr > 0U)
- ibis::util::gatherParts(ibis::datasets, dir.c_str());
- }
- else {
- uint32_t nr = 0;
- {
- ibis::part tbl(rdirs[i], static_cast<const char*>(0));
- ierr = tbl.reorder();
- LOGGER(ibis::gVerbose >= 0 && ierr < 0)
- << "Warning -- " << *argv << " failed to reorder data in "
- << rdirs[i] << ", ibis::part::reorder returned "
- << ierr;
- nr = tbl.nRows();
- }
- if (nr > 0U && (long)nr == ierr)
- ibis::util::gatherParts(ibis::datasets, rdirs[i]);
- }
+ bool has_collist=false;
+ ibis::table::stringList slist;
+ ibis::partList plist;
+ std::string dir;
+ char* str = const_cast<char*>(strrchr(rdirs[i], ':'));
+ if (str != 0 && str > rdirs[i] && str[1] != '/' && str[1] != '\\')
{
+ has_collist= true;
+ for (const char* tmp = rdirs[i]; tmp < str; ++ tmp) {
+ dir += *tmp;
+ }
+ str = ibis::util::strnewdup(str+1);
+ ibis::table::parseNames(str, slist);
+ } else {
+ dir.assign(rdirs[i]);
+ }
+ ibis::util::gatherParts(plist, dir.c_str());
+ bool gather = 0;
+ for (ibis::partList::iterator it = plist.begin(); it !=
plist.end(); ++ it) {
+ const char* ddir = (*it)->currentDataDir();
+ uint32_t nr = 0;
+ {
+ ibis::part tbl(ddir, static_cast<const char*>(0));
+ if(has_collist) {
+ ierr = tbl.reorder(slist);
+ } else {
+ ierr = tbl.reorder();
+ }
+ nr = tbl.nRows();
+ }
+ if ((long)nr == ierr && nr > 0U) {
+ gather = 1;
+ }
+ }
+ if(gather) {
+ ibis::util::gatherParts(ibis::datasets, dir.c_str() );
+ }
+ if(has_collist) {
+ delete[] str;
+ }
}
_______________________________________________
FastBit-users mailing list
[email protected]
https://hpcrdm.lbl.gov/cgi-bin/mailman/listinfo/fastbit-users
