This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 5726692 Accept total rows over Integer.MAX_VALUE. (#6080)
5726692 is described below
commit 5726692f8fe3228c88be628defce44a281387568
Author: es1220 <[email protected]>
AuthorDate: Thu Aug 16 06:03:22 2018 +0900
Accept total rows over Integer.MAX_VALUE. (#6080)
---
.../java/io/druid/indexer/DeterminePartitionsJob.java | 16 ++++++++--------
.../io/druid/indexer/hadoop/DatasourceRecordReader.java | 4 ++--
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git
a/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
b/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
index 9d61d98..fc4dde5 100644
--- a/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
+++ b/indexing-hadoop/src/main/java/io/druid/indexer/DeterminePartitionsJob.java
@@ -577,7 +577,7 @@ public class DeterminePartitionsJob implements Jobby
}
// Respect "poisoning" (negative values mean we can't use this
dimension)
- final int newNumRows = (arg1.numRows >= 0 && arg2.numRows >= 0 ?
arg1.numRows + arg2.numRows : -1);
+ final long newNumRows = (arg1.numRows >= 0 && arg2.numRows >= 0
? arg1.numRows + arg2.numRows : -1);
return new DimValueCount(arg1.dim, arg1.value, newNumRows);
}
}
@@ -620,7 +620,7 @@ public class DeterminePartitionsJob implements Jobby
// First DVC should be the total row count indicator
final DimValueCount firstDvc = iterator.next();
- final int totalRows = firstDvc.numRows;
+ final long totalRows = firstDvc.numRows;
if (!"".equals(firstDvc.dim) || !"".equals(firstDvc.value)) {
throw new IllegalStateException("WTF?! Expected total row indicator on
first k/v pair!");
@@ -899,9 +899,9 @@ public class DeterminePartitionsJob implements Jobby
return distance;
}
- public int getRows()
+ public long getRows()
{
- int sum = 0;
+ long sum = 0;
for (final DimPartition dimPartition : partitions) {
sum += dimPartition.rows;
}
@@ -913,16 +913,16 @@ public class DeterminePartitionsJob implements Jobby
{
public ShardSpec shardSpec = null;
public int cardinality = 0;
- public int rows = 0;
+ public long rows = 0;
}
private static class DimValueCount
{
public final String dim;
public final String value;
- public final int numRows;
+ public final long numRows;
- private DimValueCount(String dim, String value, int numRows)
+ private DimValueCount(String dim, String value, long numRows)
{
this.dim = dim;
this.value = value;
@@ -938,7 +938,7 @@ public class DeterminePartitionsJob implements Jobby
{
final Iterator<String> splits =
TAB_SPLITTER.limit(3).split(text.toString()).iterator();
final String dim = splits.next();
- final int numRows = Integer.parseInt(splits.next());
+ final long numRows = Long.parseLong(splits.next());
final String value = splits.next();
return new DimValueCount(dim, value, numRows);
diff --git
a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
index 739bd9e..b3dde78 100644
---
a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
+++
b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceRecordReader.java
@@ -52,12 +52,12 @@ public class DatasourceRecordReader extends
RecordReader<NullWritable, InputRow>
private DatasourceIngestionSpec spec;
private IngestSegmentFirehose firehose;
- private int rowNum;
+ private long rowNum;
private Row currRow;
private List<QueryableIndex> indexes = Lists.newArrayList();
private List<File> tmpSegmentDirs = Lists.newArrayList();
- private int numRows;
+ private long numRows;
@Override
public void initialize(InputSplit split, final TaskAttemptContext context)
throws IOException
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
