KYLIN-608 rename hllccounter methond
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ecfc3ccd Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ecfc3ccd Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ecfc3ccd Branch: refs/heads/inverted-index Commit: ecfc3ccd9a2c2a666363d78dced750060b8c6505 Parents: bde1e31 Author: honma <ho...@ebay.com> Authored: Thu Feb 12 11:03:33 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Thu Feb 12 11:03:33 2015 +0800 ---------------------------------------------------------------------- .../common/hll/HyperLogLogPlusCounter.java | 72 ++------------------ .../common/util/HyperLogLogCounterTest.java | 4 +- .../cardinality/ColumnCardinalityMapper.java | 2 +- .../cardinality/ColumnCardinalityReducer.java | 4 +- .../job/tools/ColumnCardinalityMapperTest.java | 4 +- .../job/tools/ColumnCardinalityReducerTest.java | 2 +- .../kylin/metadata/measure/HLLCSerializer.java | 4 +- .../measure/fixedlen/FixedHLLCodec.java | 4 +- .../query/sqlfunc/HLLDistinctCountAggFunc.java | 4 +- 9 files changed, 18 insertions(+), 82 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java index 6ab0611..381dea6 100644 --- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java +++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java @@ -169,63 +169,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter // ============================================================================ - public static interface Compressor { - - byte[] compress(ByteBuffer buf, int offset, int length) throws IOException; - - byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException; - } - - static final Compressor GZIP_COMPRESSOR = new Compressor() { - @Override - public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException { - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - GZIPOutputStream gzout = new GZIPOutputStream(bout); - gzout.write(buf.array(), offset, length); - gzout.close(); - return bout.toByteArray(); - } - - @Override - public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException { - ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length); - GZIPInputStream gzin = new GZIPInputStream(bin); - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - IOUtils.copy(gzin, bout); - gzin.close(); - bout.close(); - return bout.toByteArray(); - } - }; - - static final Compressor LZF_COMPRESSOR = new Compressor() { - @Override - public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException { - return LZFEncoder.encode(buf.array(), offset, length); - } - - @Override - public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException { - return LZFDecoder.decode(buf.array(), offset, length); - } - }; - - public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes, - // disable - // due to - // slowness - public static final byte COMPRESSION_FLAG = (byte) 0x02; - public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF - // lib - // has - // a - // bug - // at - // the - // moment - - public void writeCompactRegisters(final ByteBuffer out) throws IOException { - int startPos = out.position(); + public void writeRegisters(final ByteBuffer out) throws IOException { final int indexLen = getRegisterIndexSize(); int size = size(); @@ -253,7 +197,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter } } - public void readCompactRegisters(ByteBuffer in) throws IOException { + public void readRegisters(ByteBuffer in) throws IOException { byte scheme = in.get(); if (scheme == 0) { // map scheme @@ -271,19 +215,11 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter } } - /** - * For compressed output use writeCompactRegisters - * @param out - */ - public void writeRegisters(final ByteBuffer out) { + public void writeRegistersArray(final ByteBuffer out) { out.put(this.registers); } - /** - * For compressed input use readCompactRegisters - * @param in - */ - public void readRegisters(ByteBuffer in) { + public void readRegistersArray(ByteBuffer in) { in.get(registers, 0, m); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java index a7d275a..4a438fb 100644 --- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java +++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java @@ -108,9 +108,9 @@ public class HyperLogLogCounterTest { private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException { long estimate = hllc.getCountEstimate(); buf.clear(); - hllc.writeCompactRegisters(buf); + hllc.writeRegisters(buf); buf.flip(); - hllc.readCompactRegisters(buf); + hllc.readRegisters(buf); Assert.assertEquals(estimate, hllc.getCountEstimate()); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java index 329914a..dd7c720 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java @@ -94,7 +94,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, HCatRecord, IntWr HyperLogLogPlusCounter hllc = hllcMap.get(key); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); buf.clear(); - hllc.writeCompactRegisters(buf); + hllc.writeRegisters(buf); buf.flip(); context.write(new IntWritable(key), new BytesWritable(buf.array(), buf.limit())); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java index b4b55d3..a9ac1a3 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java @@ -50,7 +50,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri for (BytesWritable v : values) { ByteBuffer buffer = ByteBuffer.wrap(v.getBytes()); HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(); - hll.readCompactRegisters(buffer); + hll.readRegisters(buffer); getHllc(skey).merge(hll); hll.clear(); } @@ -77,7 +77,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri HyperLogLogPlusCounter hllc = hllcMap.get(key); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); buf.clear(); - hllc.writeCompactRegisters(buf); + hllc.writeRegisters(buf); buf.flip(); context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate())); // context.write(new Text("ErrorRate_" + key), new http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java index 08c1611..e13289a 100644 --- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java +++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java @@ -84,7 +84,7 @@ public class ColumnCardinalityMapperTest { BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); - hllc.readCompactRegisters(ByteBuffer.wrap(bytes)); + hllc.readRegisters(ByteBuffer.wrap(bytes)); assertTrue(key1 > 0); assertEquals(8, hllc.getCountEstimate()); } @@ -117,7 +117,7 @@ public class ColumnCardinalityMapperTest { BytesWritable value1 = result.get(0).getSecond(); byte[] bytes = value1.getBytes(); HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); - hllc.readCompactRegisters(ByteBuffer.wrap(bytes)); + hllc.readRegisters(ByteBuffer.wrap(bytes)); System.out.println("ab\177ab".length()); assertTrue(key1 > 0); assertEquals(1, hllc.getCountEstimate()); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java index 6ad27a8..034c90e 100644 --- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java +++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java @@ -67,7 +67,7 @@ public class ColumnCardinalityReducerTest { } ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); buf.clear(); - hllc.writeCompactRegisters(buf); + hllc.writeRegisters(buf); buf.flip(); return buf.array(); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java index 20c6c21..e3fcbee 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java @@ -38,7 +38,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> { @Override public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) { try { - value.writeCompactRegisters(out); + value.writeRegisters(out); } catch (IOException e) { throw new RuntimeException(e); } @@ -47,7 +47,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> { @Override public HyperLogLogPlusCounter deserialize(ByteBuffer in) { try { - current.readCompactRegisters(in); + current.readRegisters(in); } catch (IOException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java ---------------------------------------------------------------------- diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java index d787cbc..c6d4dc9 100644 --- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java +++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java @@ -47,12 +47,12 @@ public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> @Override public HyperLogLogPlusCounter read(byte[] buf, int offset) { - current.readRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset)); + current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset)); return current; } @Override public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) { - current.writeRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset)); + current.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset)); } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java index 8d9ace0..356c1e3 100644 --- a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java +++ b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java @@ -118,12 +118,12 @@ public class HLLDistinctCountAggFunc { } @Override - public void writeCompactRegisters(ByteBuffer out) throws IOException { + public void writeRegisters(ByteBuffer out) throws IOException { throw new UnsupportedOperationException(); } @Override - public void readCompactRegisters(ByteBuffer in) throws IOException { + public void readRegisters(ByteBuffer in) throws IOException { throw new UnsupportedOperationException(); }
