Hi,
I made a slight mistake in the first patch. Here is an updated version. I
ran "make check" and all tests passed.
--- orig/fastbit-2.0.1/src/tafel.cpp 2014-11-26 11:57:53.000000000 -0700
+++ fastbit-2.0.1/src/tafel.cpp 2015-02-25 09:52:23.270784656 -0700
@@ -1898,7 +1898,9 @@
int ibis::tafel::writeData(const char* dir, const char* tname,
const char* tdesc, const char* idx,
const char* nvpairs, uint32_t voffset) const {
- if (cols.empty() || mrows == 0) return 0; // nothing new to write
+ int prows = maxpart != 0 ? (mrows - voffset >= maxpart ? maxpart :
mrows - voffset) : mrows;
+
+ if (cols.empty() || prows == 0) return 0; // nothing new to write
if (dir == 0 || *dir == 0) {
dir = "tmp";
LOGGER(ibis::gVerbose >= 0)
@@ -2037,8 +2039,8 @@
<< " row" << (tmp.nRows()>1 ? "s" : "")
<< " and " << tmp.nColumns() << " column"
<< (tmp.nColumns()>1?"s":"")
- << ", will append " << mrows << " new row"
- << (mrows>1 ? "s" : "");
+ << ", will append " << prows << " new row"
+ << (prows>1 ? "s" : "");
}
tmp.emptyCache(); // empty cached content from mydir
}
@@ -2055,8 +2057,8 @@
std::ostringstream oss;
oss << "Data initially wrote with ibis::tablex interface on "
<< stamp << " with " << cols.size() << " column"
- << (cols.size() > 1 ? "s" : "") << " and " << nold + mrows
- << " row" << (nold+mrows>1 ? "s" : "");
+ << (cols.size() > 1 ? "s" : "") << " and " << nold + prows
+ << " row" << (nold+prows>1 ? "s" : "");
olddesc = oss.str();
tdesc = olddesc.c_str();
}
@@ -2093,8 +2095,8 @@
}
}
LOGGER(ibis::gVerbose > 1)
- << "tafel::writeData starting to write " << mrows << " row"
- << (mrows>1?"s":"") << " and " << cols.size() << " column"
+ << "tafel::writeData starting to write " << prows << " row"
+ << (prows>1?"s":"") << " and " << cols.size() << " column"
<< (cols.size()>1?"s":"") << " to " << mydir << " as data partition "
<< tname;
@@ -2112,7 +2114,7 @@
md << "# meta data for data partition " << tname
<< " written by ibis::tafel::writeData on " << stamp << "\n\n"
<< "BEGIN HEADER\nName = " << tname << "\nDescription = "
- << tdesc << "\nNumber_of_rows = " << nold+mrows
+ << tdesc << "\nNumber_of_rows = " << nold+prows
<< "\nNumber_of_columns = " << cols.size()
<< "\nTimestamp = " << currtime;
if (idx != 0 && *idx != 0) {
--Justin
On Wed, Feb 25, 2015 at 3:57 AM, Justin Swanhart <[email protected]>
wrote:
> Hi,
>
> I fixed the problem:
> $ ./ardea -t /tmp/fbdata2.txt -d /tmp/t6 -m 'c1:int,c2:int' -p 500000
> ...
> /home/justin/FastBit_UDF/fastbit-2.0.1/examples/.libs/lt-ardea --
> duration: 4.41886 sec(CPU), 4.3632 sec(elapsed)
>
> $ ./ibis -q "select count(*)" -d /tmp/t6
> SELECT count(*) FROM T-_01 WHERE 1=1 produced a table with 1 row and 1
> column
> 9999966
>
> $ diff -u orig/fastbit-2.0.1/src/tafel.cpp fastbit-2.0.1/src/tafel.cpp
> --- orig/fastbit-2.0.1/src/tafel.cpp 2014-11-26 11:57:53.000000000 -0700
> +++ fastbit-2.0.1/src/tafel.cpp 2015-02-25 03:56:13.508839432 -0700
> @@ -1898,7 +1898,9 @@
> int ibis::tafel::writeData(const char* dir, const char* tname,
> const char* tdesc, const char* idx,
> const char* nvpairs, uint32_t voffset) const {
> - if (cols.empty() || mrows == 0) return 0; // nothing new to write
> + int prows = mrows - voffset >= maxpart ? maxpart : mrows - voffset;
> +
> + if (cols.empty() || prows == 0) return 0; // nothing new to write
> if (dir == 0 || *dir == 0) {
> dir = "tmp";
> LOGGER(ibis::gVerbose >= 0)
> @@ -2037,8 +2039,8 @@
> << " row" << (tmp.nRows()>1 ? "s" : "")
> << " and " << tmp.nColumns() << " column"
> << (tmp.nColumns()>1?"s":"")
> - << ", will append " << mrows << " new row"
> - << (mrows>1 ? "s" : "");
> + << ", will append " << prows << " new row"
> + << (prows>1 ? "s" : "");
> }
> tmp.emptyCache(); // empty cached content from mydir
> }
> @@ -2055,8 +2057,8 @@
> std::ostringstream oss;
> oss << "Data initially wrote with ibis::tablex interface on "
> << stamp << " with " << cols.size() << " column"
> - << (cols.size() > 1 ? "s" : "") << " and " << nold + mrows
> - << " row" << (nold+mrows>1 ? "s" : "");
> + << (cols.size() > 1 ? "s" : "") << " and " << nold + prows
> + << " row" << (nold+prows>1 ? "s" : "");
> olddesc = oss.str();
> tdesc = olddesc.c_str();
> }
> @@ -2093,8 +2095,8 @@
> }
> }
> LOGGER(ibis::gVerbose > 1)
> - << "tafel::writeData starting to write " << mrows << " row"
> - << (mrows>1?"s":"") << " and " << cols.size() << " column"
> + << "tafel::writeData starting to write " << prows << " row"
> + << (prows>1?"s":"") << " and " << cols.size() << " column"
> << (cols.size()>1?"s":"") << " to " << mydir << " as data partition "
> << tname;
>
> @@ -2112,7 +2114,7 @@
> md << "# meta data for data partition " << tname
> << " written by ibis::tafel::writeData on " << stamp << "\n\n"
> << "BEGIN HEADER\nName = " << tname << "\nDescription = "
> - << tdesc << "\nNumber_of_rows = " << nold+mrows
> + << tdesc << "\nNumber_of_rows = " << nold+prows
> << "\nNumber_of_columns = " << cols.size()
> << "\nTimestamp = " << currtime;
> if (idx != 0 && *idx != 0) {
>
>
> On Wed, Feb 25, 2015 at 1:58 AM, Justin Swanhart <[email protected]>
> wrote:
>
>> Hi,
>>
>> I am having to problems with indexes I load when the setPartitionMax
>> member is used to set a maximum number of rows smaller than the number of
>> rows in the input file.
>>
>> Problem 1 - COUNT(*) on the index, immediately after loading with ardea
>> is very wrong
>> Problem 2 - Attempting to sort the index fails
>>
>> I've attached the log and the test data I am using, though it is very
>> uninteresting and the problem is not, as far as I can tell, dependent on
>> the data, just the number of rows.
>>
>> The log is verbose level five from ardea and ibis -q and ibis -r.
>> The commands used are:
>> rm -rf /tmp/t3
>> ardea -d /tmp/t3 -t fbdata.txt -p 500000 -m 'c1:int,c2:int'
>> ibis -d /tmp/t3 -q "SELECT COUNT(*)"
>> ibis -r /tmp/t3:c1,c2
>>
>> The reorder also throws a string exception which crashes the application,
>> in my case mysqld :(
>>
>>
>
_______________________________________________
FastBit-users mailing list
[email protected]
https://hpcrdm.lbl.gov/cgi-bin/mailman/listinfo/fastbit-users
