Hi,
I fixed the problem:
$ ./ardea -t /tmp/fbdata2.txt -d /tmp/t6 -m 'c1:int,c2:int' -p 500000
...
/home/justin/FastBit_UDF/fastbit-2.0.1/examples/.libs/lt-ardea -- duration:
4.41886 sec(CPU), 4.3632 sec(elapsed)
$ ./ibis -q "select count(*)" -d /tmp/t6
SELECT count(*) FROM T-_01 WHERE 1=1 produced a table with 1 row and 1
column
9999966
$ diff -u orig/fastbit-2.0.1/src/tafel.cpp fastbit-2.0.1/src/tafel.cpp
--- orig/fastbit-2.0.1/src/tafel.cpp 2014-11-26 11:57:53.000000000 -0700
+++ fastbit-2.0.1/src/tafel.cpp 2015-02-25 03:56:13.508839432 -0700
@@ -1898,7 +1898,9 @@
int ibis::tafel::writeData(const char* dir, const char* tname,
const char* tdesc, const char* idx,
const char* nvpairs, uint32_t voffset) const {
- if (cols.empty() || mrows == 0) return 0; // nothing new to write
+ int prows = mrows - voffset >= maxpart ? maxpart : mrows - voffset;
+
+ if (cols.empty() || prows == 0) return 0; // nothing new to write
if (dir == 0 || *dir == 0) {
dir = "tmp";
LOGGER(ibis::gVerbose >= 0)
@@ -2037,8 +2039,8 @@
<< " row" << (tmp.nRows()>1 ? "s" : "")
<< " and " << tmp.nColumns() << " column"
<< (tmp.nColumns()>1?"s":"")
- << ", will append " << mrows << " new row"
- << (mrows>1 ? "s" : "");
+ << ", will append " << prows << " new row"
+ << (prows>1 ? "s" : "");
}
tmp.emptyCache(); // empty cached content from mydir
}
@@ -2055,8 +2057,8 @@
std::ostringstream oss;
oss << "Data initially wrote with ibis::tablex interface on "
<< stamp << " with " << cols.size() << " column"
- << (cols.size() > 1 ? "s" : "") << " and " << nold + mrows
- << " row" << (nold+mrows>1 ? "s" : "");
+ << (cols.size() > 1 ? "s" : "") << " and " << nold + prows
+ << " row" << (nold+prows>1 ? "s" : "");
olddesc = oss.str();
tdesc = olddesc.c_str();
}
@@ -2093,8 +2095,8 @@
}
}
LOGGER(ibis::gVerbose > 1)
- << "tafel::writeData starting to write " << mrows << " row"
- << (mrows>1?"s":"") << " and " << cols.size() << " column"
+ << "tafel::writeData starting to write " << prows << " row"
+ << (prows>1?"s":"") << " and " << cols.size() << " column"
<< (cols.size()>1?"s":"") << " to " << mydir << " as data partition "
<< tname;
@@ -2112,7 +2114,7 @@
md << "# meta data for data partition " << tname
<< " written by ibis::tafel::writeData on " << stamp << "\n\n"
<< "BEGIN HEADER\nName = " << tname << "\nDescription = "
- << tdesc << "\nNumber_of_rows = " << nold+mrows
+ << tdesc << "\nNumber_of_rows = " << nold+prows
<< "\nNumber_of_columns = " << cols.size()
<< "\nTimestamp = " << currtime;
if (idx != 0 && *idx != 0) {
On Wed, Feb 25, 2015 at 1:58 AM, Justin Swanhart <[email protected]>
wrote:
> Hi,
>
> I am having to problems with indexes I load when the setPartitionMax
> member is used to set a maximum number of rows smaller than the number of
> rows in the input file.
>
> Problem 1 - COUNT(*) on the index, immediately after loading with ardea is
> very wrong
> Problem 2 - Attempting to sort the index fails
>
> I've attached the log and the test data I am using, though it is very
> uninteresting and the problem is not, as far as I can tell, dependent on
> the data, just the number of rows.
>
> The log is verbose level five from ardea and ibis -q and ibis -r.
> The commands used are:
> rm -rf /tmp/t3
> ardea -d /tmp/t3 -t fbdata.txt -p 500000 -m 'c1:int,c2:int'
> ibis -d /tmp/t3 -q "SELECT COUNT(*)"
> ibis -r /tmp/t3:c1,c2
>
> The reorder also throws a string exception which crashes the application,
> in my case mysqld :(
>
>
_______________________________________________
FastBit-users mailing list
[email protected]
https://hpcrdm.lbl.gov/cgi-bin/mailman/listinfo/fastbit-users
