Thanks, Justin, for the patch.
It is now in SVN revision 801.
John
On 2/25/15 9:00 AM, Justin Swanhart wrote:
> Hi,
>
> I made a slight mistake in the first patch. Here is an updated
> version. I ran "make check" and all tests passed.
>
> --- orig/fastbit-2.0.1/src/tafel.cpp2014-11-26 11:57:53.000000000 -0700
> +++ fastbit-2.0.1/src/tafel.cpp2015-02-25 09:52:23.270784656 -0700
> @@ -1898,7 +1898,9 @@
> int ibis::tafel::writeData(const char* dir, const char* tname,
> const char* tdesc, const char* idx,
> const char* nvpairs, uint32_t voffset) const {
> - if (cols.empty() || mrows == 0) return 0; // nothing new to write
> + int prows = maxpart != 0 ? (mrows - voffset >= maxpart ? maxpart
> : mrows - voffset) : mrows;
> +
> + if (cols.empty() || prows == 0) return 0; // nothing new to write
> if (dir == 0 || *dir == 0) {
> dir = "tmp";
> LOGGER(ibis::gVerbose >= 0)
> @@ -2037,8 +2039,8 @@
> << " row" << (tmp.nRows()>1 ? "s" : "")
> << " and " << tmp.nColumns() << " column"
> << (tmp.nColumns()>1?"s":"")
> - << ", will append " << mrows << " new row"
> - << (mrows>1 ? "s" : "");
> + << ", will append " << prows << " new row"
> + << (prows>1 ? "s" : "");
> }
> tmp.emptyCache(); // empty cached content from mydir
> }
> @@ -2055,8 +2057,8 @@
> std::ostringstream oss;
> oss << "Data initially wrote with ibis::tablex interface on "
> << stamp << " with " << cols.size() << " column"
> - << (cols.size() > 1 ? "s" : "") << " and " << nold + mrows
> - << " row" << (nold+mrows>1 ? "s" : "");
> + << (cols.size() > 1 ? "s" : "") << " and " << nold + prows
> + << " row" << (nold+prows>1 ? "s" : "");
> olddesc = oss.str();
> tdesc = olddesc.c_str();
> }
> @@ -2093,8 +2095,8 @@
> }
> }
> LOGGER(ibis::gVerbose > 1)
> -<< "tafel::writeData starting to write " << mrows << " row"
> -<< (mrows>1?"s":"") << " and " << cols.size() << " column"
> +<< "tafel::writeData starting to write " << prows << " row"
> +<< (prows>1?"s":"") << " and " << cols.size() << " column"
> << (cols.size()>1?"s":"") << " to " << mydir << " as data partition "
> << tname;
>
> @@ -2112,7 +2114,7 @@
> md << "# meta data for data partition " << tname
> << " written by ibis::tafel::writeData on " << stamp << "\n\n"
> << "BEGIN HEADER\nName = " << tname << "\nDescription = "
> - << tdesc << "\nNumber_of_rows = " << nold+mrows
> + << tdesc << "\nNumber_of_rows = " << nold+prows
> << "\nNumber_of_columns = " << cols.size()
> << "\nTimestamp = " << currtime;
> if (idx != 0 && *idx != 0) {
>
> --Justin
>
> On Wed, Feb 25, 2015 at 3:57 AM, Justin Swanhart <[email protected]
> <mailto:[email protected]>> wrote:
>
> Hi,
>
> I fixed the problem:
> $ ./ardea -t /tmp/fbdata2.txt -d /tmp/t6 -m 'c1:int,c2:int' -p 500000
> ...
> /home/justin/FastBit_UDF/fastbit-2.0.1/examples/.libs/lt-ardea --
> duration: 4.41886 sec(CPU), 4.3632 sec(elapsed)
>
> $ ./ibis -q "select count(*)" -d /tmp/t6
> SELECT count(*) FROM T-_01 WHERE 1=1 produced a table with 1 row
> and 1 column
> 9999966
>
> $ diff -u orig/fastbit-2.0.1/src/tafel.cpp
> fastbit-2.0.1/src/tafel.cpp
> --- orig/fastbit-2.0.1/src/tafel.cpp2014-11-26 11:57:53.000000000
> -0700
> +++ fastbit-2.0.1/src/tafel.cpp2015-02-25 03:56:13.508839432 -0700
> @@ -1898,7 +1898,9 @@
> int ibis::tafel::writeData(const char* dir, const char* tname,
> const char* tdesc, const char* idx,
> const char* nvpairs, uint32_t voffset)
> const {
> - if (cols.empty() || mrows == 0) return 0; // nothing new to write
> + int prows = mrows - voffset >= maxpart ? maxpart : mrows -
> voffset;
> +
> + if (cols.empty() || prows == 0) return 0; // nothing new to write
> if (dir == 0 || *dir == 0) {
> dir = "tmp";
> LOGGER(ibis::gVerbose >= 0)
> @@ -2037,8 +2039,8 @@
> << " row" << (tmp.nRows()>1 ? "s" : "")
> << " and " << tmp.nColumns() << " column"
> << (tmp.nColumns()>1?"s":"")
> - << ", will append " << mrows << " new row"
> - << (mrows>1 ? "s" : "");
> + << ", will append " << prows << " new row"
> + << (prows>1 ? "s" : "");
> }
> tmp.emptyCache(); // empty cached content from mydir
> }
> @@ -2055,8 +2057,8 @@
> std::ostringstream oss;
> oss << "Data initially wrote with ibis::tablex interface on "
> << stamp << " with " << cols.size() << " column"
> - << (cols.size() > 1 ? "s" : "") << " and " << nold + mrows
> - << " row" << (nold+mrows>1 ? "s" : "");
> + << (cols.size() > 1 ? "s" : "") << " and " << nold + prows
> + << " row" << (nold+prows>1 ? "s" : "");
> olddesc = oss.str();
> tdesc = olddesc.c_str();
> }
> @@ -2093,8 +2095,8 @@
> }
> }
> LOGGER(ibis::gVerbose > 1)
> -<< "tafel::writeData starting to write " << mrows << " row"
> -<< (mrows>1?"s":"") << " and " << cols.size() << " column"
> +<< "tafel::writeData starting to write " << prows << " row"
> +<< (prows>1?"s":"") << " and " << cols.size() << " column"
> << (cols.size()>1?"s":"") << " to " << mydir << " as data partition "
> << tname;
>
> @@ -2112,7 +2114,7 @@
> md << "# meta data for data partition " << tname
> << " written by ibis::tafel::writeData on " << stamp << "\n\n"
> << "BEGIN HEADER\nName = " << tname << "\nDescription = "
> - << tdesc << "\nNumber_of_rows = " << nold+mrows
> + << tdesc << "\nNumber_of_rows = " << nold+prows
> << "\nNumber_of_columns = " << cols.size()
> << "\nTimestamp = " << currtime;
> if (idx != 0 && *idx != 0) {
>
>
> On Wed, Feb 25, 2015 at 1:58 AM, Justin Swanhart
> <[email protected] <mailto:[email protected]>> wrote:
>
> Hi,
>
> I am having to problems with indexes I load when the
> setPartitionMax member is used to set a maximum number of rows
> smaller than the number of rows in the input file.
>
> Problem 1 - COUNT(*) on the index, immediately after loading
> with ardea is very wrong
> Problem 2 - Attempting to sort the index fails
>
> I've attached the log and the test data I am using, though it
> is very uninteresting and the problem is not, as far as I can
> tell, dependent on the data, just the number of rows.
>
> The log is verbose level five from ardea and ibis -q and ibis
> -r.
> The commands used are:
> rm -rf /tmp/t3
> ardea -d /tmp/t3 -t fbdata.txt -p 500000 -m 'c1:int,c2:int'
> ibis -d /tmp/t3 -q "SELECT COUNT(*)"
> ibis -r /tmp/t3:c1,c2
>
> The reorder also throws a string exception which crashes the
> application, in my case mysqld :(
>
>
>
>
>
> _______________________________________________
> FastBit-users mailing list
> [email protected]
> https://hpcrdm.lbl.gov/cgi-bin/mailman/listinfo/fastbit-users
>
_______________________________________________
FastBit-users mailing list
[email protected]
https://hpcrdm.lbl.gov/cgi-bin/mailman/listinfo/fastbit-users
