Repository: kylin Updated Branches: refs/heads/2.x-staging bb130af0a -> 50b3f429d
minor changes on logging temp hi Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c1e83b05 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c1e83b05 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c1e83b05 Branch: refs/heads/2.x-staging Commit: c1e83b0530a5d497e248694f139f8df28ae0dd0f Parents: bb130af Author: honma <ho...@ebay.com> Authored: Wed Jan 6 11:15:42 2016 +0800 Committer: honma <ho...@ebay.com> Committed: Fri Jan 8 10:51:56 2016 +0800 ---------------------------------------------------------------------- build/bin/kylin.sh | 2 +- .../apache/kylin/gridtable/GTScanRequest.java | 1 - .../hbase/cube/v1/CubeSegmentTupleIterator.java | 2 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 5 +- .../coprocessor/endpoint/CubeVisitService.java | 30 ++- .../endpoint/generated/CubeVisitProtos.java | 181 ++++++++++++++++++- .../endpoint/protobuf/CubeVisit.proto | 1 + 7 files changed, 203 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/build/bin/kylin.sh ---------------------------------------------------------------------- diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index 7f296cd..77a17a8 100644 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -84,7 +84,7 @@ then -Dkylin.hive.dependency=${hive_dependency} \ -Dkylin.hbase.dependency=${hbase_dependency} \ -Dspring.profiles.active=${spring_profile} \ - org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid & + org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid & echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\"" echo "Please visit http://<ip>:7070/kylin" echo "You can check the log at ${KYLIN_HOME}/logs/kylin.log" http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index a73a950..abaec85 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -228,7 +228,6 @@ public class GTScanRequest { } } } - System.out.println("meaningless byte is now " + meaninglessByte); } public boolean hasFilterPushDown() { http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java index f760192..f8edee2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java @@ -304,7 +304,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator { byte[] metricsBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA); if (metricsBytes != null) { ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(metricsBytes); - logger.debug("HBase Metrics: " + "count={}, ms={}, bytes={}, remote_bytes={}, regions={}, not_serving_region={}, rpc={}, rpc_retries={}, remote_rpc={}, remote_rpc_retries={}", // + logger.debug("HBase Metrics when scanning " + this.tableName + " count={}, ms={}, bytes={}, remote_bytes={}, regions={}, not_serving_region={}, rpc={}, rpc_retries={}, remote_rpc={}, remote_rpc_retries={}", // new Object[] { scanCount, scanMetrics.sumOfMillisSecBetweenNexts, scanMetrics.countOfBytesInResults, scanMetrics.countOfBytesInRemoteResults, scanMetrics.countOfRegions, scanMetrics.countOfNSRE, scanMetrics.countOfRPCcalls, scanMetrics.countOfRPCRetries, scanMetrics.countOfRemoteRPCcalls, scanMetrics.countOfRemoteRPCRetries }); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index 14bc851..35df3ed 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -242,11 +242,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { private String getStatsString(CubeVisitProtos.CubeVisitResponse result, int shardIndex) { StringBuilder sb = new StringBuilder(); Stats stats = result.getStats(); - sb.append("Shard " + shardIndex + " on host: " + stats.getHostname()); + sb.append("Shard " + shardIndex + " on host: " + stats.getHostname() + "."); sb.append("Total scanned row: " + stats.getScannedRowCount() + ". "); sb.append("Total filtered/aggred row: " + stats.getAggregatedRowCount() + ". "); sb.append("Time elapsed in EP: " + (stats.getServiceEndTime() - stats.getServiceStartTime()) + "(ms). "); - sb.append("Server CPU usage: " + stats.getSystemCpuLoad() + ", server physical mem left: " + stats.getFreePhysicalMemorySize() + ", server swap mem left:" + stats.getFreeSwapSpaceSize()); + sb.append("Server CPU usage: " + stats.getSystemCpuLoad() + ", server physical mem left: " + stats.getFreePhysicalMemorySize() + ", server swap mem left:" + stats.getFreeSwapSpaceSize() + "."); + sb.append("Etc message: " + stats.getEtcMsg() + "."); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java index 6acaa8b..e396a76 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java @@ -128,6 +128,9 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement RegionScanner innerScanner = null; HRegion region = null; + StringBuilder sb = new StringBuilder(); + byte[] allRows; + try { this.serviceStartTime = System.currentTimeMillis(); @@ -140,13 +143,16 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement Scan scan = CubeHBaseRPC.buildScan(hbaseRawScan); + sb.append(System.currentTimeMillis() - this.serviceStartTime); + sb.append(","); + region = env.getRegion(); region.startRegionOperation(); innerScanner = region.getScanner(scan); - InnerScannerAsIterator cellListIterator = new InnerScannerAsIterator(innerScanner); - CoprocessorBehavior behavior = CoprocessorBehavior.valueOf(request.getBehavior()); + + InnerScannerAsIterator cellListIterator = new InnerScannerAsIterator(innerScanner); if (behavior.ordinal() < CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) { scanReq.setAggrCacheGB(0); // disable mem check if so told } @@ -171,16 +177,27 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement finalRowCount++; } + sb.append(System.currentTimeMillis() - this.serviceStartTime); + sb.append(","); + + //outputStream.close() is not necessary + allRows = outputStream.toByteArray(); + byte[] compressedAllRows = CompressionUtils.compress(allRows); + + sb.append(System.currentTimeMillis() - this.serviceStartTime); + sb.append(","); + OperatingSystemMXBean operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); double systemCpuLoad = operatingSystemMXBean.getSystemCpuLoad(); double freePhysicalMemorySize = operatingSystemMXBean.getFreePhysicalMemorySize(); double freeSwapSpaceSize = operatingSystemMXBean.getFreeSwapSpaceSize(); - //outputStream.close() is not necessary - byte[] allRows = outputStream.toByteArray(); + sb.append(System.currentTimeMillis() - this.serviceStartTime); + sb.append(","); + CubeVisitProtos.CubeVisitResponse.Builder responseBuilder = CubeVisitProtos.CubeVisitResponse.newBuilder(); done.run(responseBuilder.// - setCompressedRows(HBaseZeroCopyByteString.wrap(CompressionUtils.compress(allRows))).//too many array copies + setCompressedRows(HBaseZeroCopyByteString.wrap(compressedAllRows)).//too many array copies setStats(CubeVisitProtos.CubeVisitResponse.Stats.newBuilder().// setAggregatedRowCount(finalScanner.getScannedRowCount() - finalRowCount).// setScannedRowCount(finalScanner.getScannedRowCount()).// @@ -189,7 +206,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement setSystemCpuLoad(systemCpuLoad).// setFreePhysicalMemorySize(freePhysicalMemorySize).// setFreeSwapSpaceSize(freeSwapSpaceSize).// - setHostname(InetAddress.getLocalHost().getHostName()).// + setHostname(InetAddress.getLocalHost().getHostName()).// + setEtcMsg(sb.toString()).// build()).// build()); http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java index 7d9b313..1094f80 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java @@ -1919,6 +1919,21 @@ public final class CubeVisitProtos { */ com.google.protobuf.ByteString getHostnameBytes(); + + // optional string etcMsg = 9; + /** + * <code>optional string etcMsg = 9;</code> + */ + boolean hasEtcMsg(); + /** + * <code>optional string etcMsg = 9;</code> + */ + java.lang.String getEtcMsg(); + /** + * <code>optional string etcMsg = 9;</code> + */ + com.google.protobuf.ByteString + getEtcMsgBytes(); } /** * Protobuf type {@code CubeVisitResponse.Stats} @@ -2011,6 +2026,11 @@ public final class CubeVisitProtos { hostname_ = input.readBytes(); break; } + case 74: { + bitField0_ |= 0x00000100; + etcMsg_ = input.readBytes(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -2206,6 +2226,49 @@ public final class CubeVisitProtos { } } + // optional string etcMsg = 9; + public static final int ETCMSG_FIELD_NUMBER = 9; + private java.lang.Object etcMsg_; + /** + * <code>optional string etcMsg = 9;</code> + */ + public boolean hasEtcMsg() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public java.lang.String getEtcMsg() { + java.lang.Object ref = etcMsg_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + etcMsg_ = s; + } + return s; + } + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public com.google.protobuf.ByteString + getEtcMsgBytes() { + java.lang.Object ref = etcMsg_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + etcMsg_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + private void initFields() { serviceStartTime_ = 0L; serviceEndTime_ = 0L; @@ -2215,6 +2278,7 @@ public final class CubeVisitProtos { freePhysicalMemorySize_ = 0D; freeSwapSpaceSize_ = 0D; hostname_ = ""; + etcMsg_ = ""; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2252,6 +2316,9 @@ public final class CubeVisitProtos { if (((bitField0_ & 0x00000080) == 0x00000080)) { output.writeBytes(8, getHostnameBytes()); } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + output.writeBytes(9, getEtcMsgBytes()); + } getUnknownFields().writeTo(output); } @@ -2293,6 +2360,10 @@ public final class CubeVisitProtos { size += com.google.protobuf.CodedOutputStream .computeBytesSize(8, getHostnameBytes()); } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(9, getEtcMsgBytes()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2353,6 +2424,11 @@ public final class CubeVisitProtos { result = result && getHostname() .equals(other.getHostname()); } + result = result && (hasEtcMsg() == other.hasEtcMsg()); + if (hasEtcMsg()) { + result = result && getEtcMsg() + .equals(other.getEtcMsg()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -2401,6 +2477,10 @@ public final class CubeVisitProtos { hash = (37 * hash) + HOSTNAME_FIELD_NUMBER; hash = (53 * hash) + getHostname().hashCode(); } + if (hasEtcMsg()) { + hash = (37 * hash) + ETCMSG_FIELD_NUMBER; + hash = (53 * hash) + getEtcMsg().hashCode(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -2526,6 +2606,8 @@ public final class CubeVisitProtos { bitField0_ = (bitField0_ & ~0x00000040); hostname_ = ""; bitField0_ = (bitField0_ & ~0x00000080); + etcMsg_ = ""; + bitField0_ = (bitField0_ & ~0x00000100); return this; } @@ -2586,6 +2668,10 @@ public final class CubeVisitProtos { to_bitField0_ |= 0x00000080; } result.hostname_ = hostname_; + if (((from_bitField0_ & 0x00000100) == 0x00000100)) { + to_bitField0_ |= 0x00000100; + } + result.etcMsg_ = etcMsg_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -2628,6 +2714,11 @@ public final class CubeVisitProtos { hostname_ = other.hostname_; onChanged(); } + if (other.hasEtcMsg()) { + bitField0_ |= 0x00000100; + etcMsg_ = other.etcMsg_; + onChanged(); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -2960,6 +3051,80 @@ public final class CubeVisitProtos { return this; } + // optional string etcMsg = 9; + private java.lang.Object etcMsg_ = ""; + /** + * <code>optional string etcMsg = 9;</code> + */ + public boolean hasEtcMsg() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public java.lang.String getEtcMsg() { + java.lang.Object ref = etcMsg_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + etcMsg_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public com.google.protobuf.ByteString + getEtcMsgBytes() { + java.lang.Object ref = etcMsg_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + etcMsg_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public Builder setEtcMsg( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000100; + etcMsg_ = value; + onChanged(); + return this; + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public Builder clearEtcMsg() { + bitField0_ = (bitField0_ & ~0x00000100); + etcMsg_ = getDefaultInstance().getEtcMsg(); + onChanged(); + return this; + } + /** + * <code>optional string etcMsg = 9;</code> + */ + public Builder setEtcMsgBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000100; + etcMsg_ = value; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:CubeVisitResponse.Stats) } @@ -3758,19 +3923,19 @@ public final class CubeVisitProtos { "canRequest\030\002 \002(\014\022\024\n\014hbaseRawScan\030\003 \002(\014\022\032" + "\n\022rowkeyPreambleSize\030\004 \002(\005\0223\n\020hbaseColum" + "nsToGT\030\005 \003(\0132\031.CubeVisitRequest.IntList\032" + - "\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\251\002\n\021CubeVisitRe" + + "\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\271\002\n\021CubeVisitRe" + "sponse\022\026\n\016compressedRows\030\001 \002(\014\022\'\n\005stats\030" + - "\002 \002(\0132\030.CubeVisitResponse.Stats\032\322\001\n\005Stat", + "\002 \002(\0132\030.CubeVisitResponse.Stats\032\342\001\n\005Stat", "s\022\030\n\020serviceStartTime\030\001 \001(\003\022\026\n\016serviceEn" + "dTime\030\002 \001(\003\022\027\n\017scannedRowCount\030\003 \001(\005\022\032\n\022" + "aggregatedRowCount\030\004 \001(\005\022\025\n\rsystemCpuLoa" + "d\030\005 \001(\001\022\036\n\026freePhysicalMemorySize\030\006 \001(\001\022" + "\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020\n\010hostname\030\010" + - " \001(\t2F\n\020CubeVisitService\0222\n\tvisitCube\022\021." + - "CubeVisitRequest\032\022.CubeVisitResponseB`\nE" + - "org.apache.kylin.storage.hbase.cube.v2.c" + - "oprocessor.endpoint.generatedB\017CubeVisit" + - "ProtosH\001\210\001\001\240\001\001" + " \001(\t\022\016\n\006etcMsg\030\t \001(\t2F\n\020CubeVisitService" + + "\0222\n\tvisitCube\022\021.CubeVisitRequest\032\022.CubeV" + + "isitResponseB`\nEorg.apache.kylin.storage" + + ".hbase.cube.v2.coprocessor.endpoint.gene" + + "ratedB\017CubeVisitProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -3800,7 +3965,7 @@ public final class CubeVisitProtos { internal_static_CubeVisitResponse_Stats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_CubeVisitResponse_Stats_descriptor, - new java.lang.String[] { "ServiceStartTime", "ServiceEndTime", "ScannedRowCount", "AggregatedRowCount", "SystemCpuLoad", "FreePhysicalMemorySize", "FreeSwapSpaceSize", "Hostname", }); + new java.lang.String[] { "ServiceStartTime", "ServiceEndTime", "ScannedRowCount", "AggregatedRowCount", "SystemCpuLoad", "FreePhysicalMemorySize", "FreeSwapSpaceSize", "Hostname", "EtcMsg", }); return null; } }; http://git-wip-us.apache.org/repos/asf/kylin/blob/c1e83b05/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto index aa05c0b..d60d406 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto @@ -32,6 +32,7 @@ message CubeVisitResponse { optional double freePhysicalMemorySize = 6; optional double freeSwapSpaceSize = 7; optional string hostname = 8; + optional string etcMsg = 9; } required bytes compressedRows = 1; required Stats stats = 2;
