This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 19706b2 KYLIN-4010 Support date format "yyyy-MM-dd'T'HH:mm:ss.SSSZZ"
new 92273d4 KYLIN-4010 Auto adjust offset according to query server's
timezone for time derived column
new 4f3d8e7 KYLIN-4141 Build Global Dictionary in no time
new a6fc3e7 KYLIN-4167 Clarify overall design for realtime OLAP
new e8ad658 KYLIN-4167 Refactor streaming coordinator Phase1
new 80384da KYLIN-4167 Fix minor bug
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build/conf/kylin-server-log4j.properties | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 18 +
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 2 +-
.../kylin/dimension}/TimeDerivedColumnType.java | 2 +-
.../dimension}/TimeDerivedColumnTypeTest.java | 2 +-
.../kylin/storage/gtrecord/CubeTupleConverter.java | 32 +-
.../kylin/engine/mr/StreamingCubingEngine.java | 5 +-
.../apache/kylin/query/relnode/OLAPFilterRel.java | 5 +
.../query/relnode/visitor/TupleFilterVisitor.java | 24 +-
.../relnode/visitor/TupleFilterVisitorTest.java | 9 +-
.../controller/StreamingCoordinatorController.java | 8 +-
.../rest/service/StreamingCoordinatorService.java | 27 +-
.../kylin/rest/service/StreamingV2Service.java | 40 +-
.../stream/rpc/HttpStreamDataSearchClient.java | 5 +-
.../kylin/stream/coordinator/Coordinator.java | 3 +-
.../stream/coordinator/StreamMetadataStore.java | 11 +-
.../coordinator/ZookeeperStreamMetadataStore.java | 66 +-
.../coordinator/client/CoordinatorClient.java | 15 +
.../client/CoordinatorClientFactory.java | 17 +-
.../coordinator/coordinate/BuildJobSubmitter.java | 464 +++++++++++++
.../coordinate/ReceiverClusterManager.java | 651 +++++++++++++++++++
.../coordinate/SegmentJobBuildInfo.java | 73 +++
.../coordinate/StreamingCoordinator.java | 644 ++++++++++++++++++
.../coordinate/annotations/NonSideEffect.java | 40 +-
.../annotations/NotAtomicAndNotIdempotent.java | 45 ++
.../annotations/NotAtomicIdempotent.java | 44 +-
.../stream/coordinator/doctor/ClusterDoctor.java | 16 +-
.../coordinator/doctor/ClusterStateChecker.java | 23 +-
.../exception/ClusterStateException.java | 6 +-
.../coordinator/exception/CoordinateException.java | 3 +
.../exception/NotLeadCoordinatorException.java | 2 +
.../coordinator/exception/StoreException.java | 2 +
.../src/main/resources/log4j.properties | 15 +-
.../coordinate/BuildJobSubmitterTest.java | 194 ++++++
.../coordinator/coordinate/StreamingTestBase.java | 335 ++++++++++
stream-core/pom.xml | 5 +
.../stream/core/client/ReceiverAdminClient.java | 76 +++
.../core/consumer/StreamingConsumerChannel.java | 12 +
.../stream/core/dict/LocalDictionaryStore.java | 144 +++++
.../stream/core/dict/RemoteDictionaryStore.java | 215 ++++++
.../core/dict/StreamingDictionaryClient.java | 216 +++++++
.../core/dict/StreamingDistributedDictionary.java | 91 +++
.../kylin/stream/core/model/SegmentBuildState.java | 2 +-
.../stream/core/model/stats/ClusterState.java | 8 +
.../stream/core/model/stats/ReplicaSetState.java | 3 +
.../core/query/StreamingDataQueryPlanner.java | 2 +-
.../stream/core/query/StreamingTupleConverter.java | 29 +-
.../core/storage/IStreamingSegmentStore.java | 5 +
.../core/storage/StreamingSegmentManager.java | 61 +-
.../storage/columnar/ColumnarSegmentStore.java | 14 +-
.../core/storage/columnar/ColumnarStoreCache.java | 16 +-
.../storage/columnar/ColumnarStoreDimDesc.java | 2 +-
.../storage/columnar/FragmentFileSearcher.java | 718 ++++++++++-----------
.../core/storage/columnar/SegmentMemoryStore.java | 12 +-
.../columnar/TimeDerivedColumnEncoding.java | 2 +-
.../core/util/CompareFilterTimeRangeChecker.java | 1 +
.../kylin/stream/server/StreamingServer.java | 19 +-
.../server/rest/controller/AdminController.java | 10 +-
.../stream/source/kafka/TimedJsonStreamParser.java | 2 +-
60 files changed, 3957 insertions(+), 560 deletions(-)
rename {stream-core/src/main/java/org/apache/kylin/stream/core/util =>
core-metadata/src/main/java/org/apache/kylin/dimension}/TimeDerivedColumnType.java
(99%)
rename {stream-core/src/test/java/org/apache/kylin/stream/core/util =>
core-metadata/src/test/java/org/apache/kylin/dimension}/TimeDerivedColumnTypeTest.java
(99%)
create mode 100644
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java
create mode 100644
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/ReceiverClusterManager.java
create mode 100644
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/SegmentJobBuildInfo.java
create mode 100644
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java
copy
core-common/src/main/java/org/apache/kylin/common/util/ByteBufferOutputStream.java
=>
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NonSideEffect.java
(54%)
create mode 100644
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NotAtomicAndNotIdempotent.java
copy core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java
=>
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NotAtomicIdempotent.java
(50%)
copy core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java =>
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/doctor/ClusterDoctor.java
(70%)
copy
core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java
=>
stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/doctor/ClusterStateChecker.java
(62%)
copy dev-support/test_all_against_hdp_2_4_0_0_169.sh =>
stream-coordinator/src/main/resources/log4j.properties (69%)
mode change 100755 => 100644
create mode 100644
stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitterTest.java
create mode 100644
stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/coordinate/StreamingTestBase.java
create mode 100644
stream-core/src/main/java/org/apache/kylin/stream/core/dict/LocalDictionaryStore.java
create mode 100644
stream-core/src/main/java/org/apache/kylin/stream/core/dict/RemoteDictionaryStore.java
create mode 100644
stream-core/src/main/java/org/apache/kylin/stream/core/dict/StreamingDictionaryClient.java
create mode 100644
stream-core/src/main/java/org/apache/kylin/stream/core/dict/StreamingDistributedDictionary.java
