This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch realtime-streaming
in repository https://gitbox.apache.org/repos/asf/kylin.git.
discard 74655f1 KYLIN-3821 Add consume lag stats
discard 812803c KYLIN-3744 Add configuration and fix bugs
discard 8b374c5 Minor, fix rat check failed
discard cce8ac8 KYLIN-3786 Add integration test for realtime-streaming
discard 99c8883 KYLIN-3800 Measure get incorrect result
discard f5a6b8d KYLIN-3789 Stream receiver admin page issue fix
discard 67101ce KYLIN-3791 Map return by Maps.transformValues is a immutable
view
discard 7590263 KYLIN-3787 NPE throws when dimension value has null when
query real-time data
discard ca0a8ac6 KYLIN-3784 Hadoop Common ReflectionUtils.printThreadInfo
signature change
discard 50514f8 KYLIN-3744 Make coordinator well tested and add some javadoc
discard 55f3bbd KYLIN-3759 Fix classnotfound if lambda is enable
discard 5b85ebe KYLIN-3768 Save streaming metadata a standard kylin path in
zookeeper
discard 6bb8155 KYLIN-3747 Use FQDN to register a streaming receiver instead
of ip
discard a3e76c0 KYLIN-3745 Real-time segment state changed from active to
immutable is not sequently
discard c406029 KYLIN-3742 Fix DataRequest for NPE and add some javadoc
discard d76d516 Merge pull request #412 from hit-lacus/realtime-streaming
discard f84996d KYLIN-3730 TableMetadataManager.reloadSourceTableQuietly is
wrong
discard 884b837 KYLIN-3690 New streaming backend implementation - streaming
job
discard 266f78c KYLIN-3690 New streaming backend implementation - streaming
consumer
discard 4318d0a KYLIN-3690 New streaming backend implementation
discard fd78f8b KYLIN-3691 New streaming ui implementation
new 0d9fbbe KYLIN-3416 Group by with expression can not aggregate exactly
new 195640e KYLIN-3710 JDBC data source support Spark cubing
new cddb493 KYLIN-3680 Spark cubing failed with JDBC data source
new 0b31a91 KYLIN-3597 Improve code smell
new 2e5505c ProjectL2Cache don't be invalidated when adding override
config in cube level
new 23732b2 KYLIN-3597 fix sonar issues
new ba0fbf9 KYLIN-3406 Ignore execute output file lose
new bef9e5c APACHE-KYLIN-2932: Simplify the thread model for in-memory
cubing
new fab6955 KYLIN-3406 Fix for two inconsistent ExecuteManger
new ab51ea9 set JAVA_HOME before server start or stop (#385)
new 1358e80 Revert "KYLIN-3648 Use Path as the input of addResource"
new f031005 KYLIN-3718 Segments in volatile range is more than designated
value
new 6590abf KYLIN-2841 LIMIT is buggy with subquery
new 6283046 KYLIN-3597 improve code smell
new 32a3150 KYLIN-3597 fix sonar issues
new fb54f38 KYLIN-3707 add configuration for setting isolation-level for
sqoop
new 9fb5b46 KYLIN-3721 modify sourceCacheKey for RDBMS
new dd0943a KYLIN-3607 add hbase-hadoop*-compat*.jar
new 13d183a KYLIN-3720 add column family check when save/update cube desc
new 45f6f99 minor, add forked calcite source location
new d4ad0fa KYLIN-3699 Spark cubing failed when build with empty data
new af77cfa KYLIN-3544 refine guava cache for the recommended cuboids of
cube planner
new adb83bb KYLIN-1819 Fix exception swallowed when start
DefaultScheduler fail
new 9b205fe Merge pull request #405 from kyotoYaho/APACHE-KYLIN-1819
new 91efa0c KYLIN-3365 Add unit test for the coprocessor code,
CubeVisitService
new 6f760ef Minor, delete file before create it
new b31385f KYLIN-3734 Fix UT error: invalid path of global job lock
new 9bf0063 KYLIN-3724 Kylin IT test sql is unreasonable
new 95b812f KYLIN-3655 reinitialize CubeInstance when clear segments
new 5982bb7 KYLIN-3737 refactor cache part for RDBMS
new 3b9c5a5 KYLIN-3597 fix sonar issues
new 0c60c6b KYLIN-3021 check MapReduce job failed reason and include the
diagnostics into email notification
new 0c92d1c minor, fix format in pom
new 8cfe32c KYLIN-3540 move queryService of CubeController to CubeService
new 4db6a37 KYLIN-3540 refactor the interface of querying on SYSTEM
project
new 4850dac KYLIN-3540 estimate the row counts of source cuboids which
are not built & remove mandatory cuboids recommendation
new 1af62e4 KYLIN-3738 Edit cube measure may make the decimal type change
unexpectly revert KYLIN-2243 8c0c44b887e2caa21b097c2334f8d21c42462e80
new 338a8f1 KYLIN-3597 fix sonar issues
new 72f5333 KYLIN-3729 CLUSTER BY CAST(field AS STRING) will accelerate
base cuboid build with UHC global dict
new 5976c69 KYLIN-3643 Inner column filter can not pushdown
new 7539ac4 KYLIN-3731 Segment pruning ignore complex data type
new eadb2a8 minor, remove one duplicated line
new e1ba10a KYLIN-3597 fix sonar issues
new e3896af KYLIN-3628 The wrong result when a query with one lookup table
new 49e835a KYLIN-3597 fix sonar problems
new a20f04f KYLIN-3430 Global Dictionary Cleanup
new cc00df6 KYLIN-2898 update LICENSE for couchbase/spymemcached
new bfe1364 KYLIN-3748 No realization found exception thrown when a ready
cube is building
new 33edddf KYLIN-3479 add configuration to override xml for RDBMS
new 250f1a1 KYLIN-3752 Increase broadcaster's concurrency to avoid
exceptions
new b780759 KYLIN-3740 fix NPE during step of Extract Fact Table Distinct
Columns
new d5bfbb1 KYLIN-3722 Disable limit push down for multi olapcontext
new 108765e KYLIN-2972 CacheKey from SQLRequest should ignore the case of
project name
new 0e83b2d KYLIN-3722 minor, update property name to follow naming
convention
new 1f36967 KYLIN-2972 minor fix
new 5c9c3df KYLIN-3725 disable javadoc check for all profiles
new 601deae KYLIN-3580 Avoid using entrySet to add elements
new fe8014e KYLIN-3767 Print the malformed JSON data consumed from Kafka
Topic
new 4c88f18 KYLIN-3772 CubeMetaIngester works abnormally
new 3825a3d KYLIN-3597 fix sonar issues
new e778377 KYLIN-3597 fix sonar issues
new 1cf2322 KYLIN-3770 ZipFileUtils is not compatible CubeMetaIngester
new 4a025fe KYLIN-3776 Float type in MySQL not properly converted to HIVE
Double Type
new e413a81 KYLIN-3776 Float type in MySQL not properly converted to HIVE
Double Type
new b4dace4 KYLIN-3773 modify dependency datasource-sdk
new a96e642 KYLIN-3778 fix project level datasource
new 168fa86 Add built instance in Job info
new d3235e7 KYLIN-3597 fix sonar issues
new c44832c KYLIN-3703 get negative value when query kylin
new b99beb6 KYLIN-3494: Build cube with spark reports
ArrayIndexOutOfBoundsException
new d678fe7 KYLIN-3494 ArrayIndexOutOfBoundsException may occur in
streaming cube building
new 2cea735 KYLIN 3790 return wrong error code when command executor is
interrupted
new 7437af8 KYLIN-2725 fix cube desc signature issue
new 33ef4a9 KYLIN-3793 KafkaSampleProducer exit after generating 1 message
new b1f1b7b KYLIN-3794 fix mergeToInClause not working properly in some
edge cases
new bcf1b52 KYLIN-3799 Fix wrong hive-exec jar may be found by
find-hive-dependency.sh on CDH
new 24f1987 KYLIN-3797 Too many or filters may break Kylin server when
flatting filter
new 4fb7eee minor, fix NPE
new 5e9d533 KYLIN-3798 SQL Server cannot show databases
new 7e9a8e1 minor, add more log when fail to locate kylin.properties
new 2b759f9 #KYLIN-3814, add pause interval before the job retry
new 0d43ae9 KYLIN-3597 fix sonar issues and refactor test cases
new 96d72ca KYLIN-3801 Add --text flag in grep in find-hive-dependency.sh
new 73ae41c KYLIN-3804 Advanced Snapshot Table save not friendly
new 03714e0 minor, update sonarcloud token
new d362acb KYLIN-3815 Unexpected behavior when joinning streaming table
and hive table Minor, replace hql with quoted table identity KYLIN-3815 code
review
new 6eee273 Update version to 3.0.0-SNAPSHOT
new 0b92dd4 KYLIN-3819 kylin.sh run in the foreground
new 0b0ceee KYLIN-3822 fix the param parse error in DeployCoprocessorCLI
(#476)
new 9d125e1 KYLIN-3824
new 500f102 KYLIN-3826 MergeCuboidJob only uploads necessary segment's
dictionary
new fb34122 KYLIN-3828 Fix heading empty string be ignored in
StringUtil.join
new 8e71751 KYLIN-3782 ZookeeperDistributedLock can't acquir lock on
windows
new bab003c KYLIN-3816 Make CI cover streaming table join case
new bfb30d9 Minor, reset querycontext before execute query
new c152807 KYLIN-3833 Avoid OOM.
new e5c7c0d KYLIN-3714 Register kryo for spark spilling process.
new 6bc2aae KYLIN-3836 fix Kylin StringUtil.join() may cause NPE if
iterator is empty
new 68bffc1 KYLIN-3727 Check if where is no directories, then finish job
successfully.
new 5cd0909 KYLIN-3838 Fix retry mechanism is invalid when build with
spark
new 30ef767 KYLIN-3820 Add a curator-based scheduler
new 2688b82 KYLIN-3834, add monitor for curator-based scheduler.
new a5e0842 Minor, change travis ci log level to INFO
new 662a7f9 KYLIN-3795 Submit Spark jobs via Apache Livy
new f968e31 KYLIN-2620 Make the condition stricter to answer query with
topN
new 636f93a KYLIN-3716 FastThreadLocal replaces ThreadLocal
new 7519629 KYLIN-3835 [Defective TableSchemaUpdateChecker] Don't check
used models when reload table
new d00aced Added small Unit Tests
new 69c2c17 Removed .* import statements
new 0234301 KYLIN-3842 kylinProperties.js Unable to get the public
configuration of the first line in the front end
new c0b9e51 KYLIN-3808 fix RestAPI /api/jobs always returns 0 for
exec_start_time and exec_end_time and exec_interrupt_time fields
new d240e0d KYLIN-3571 Not build Spark in Kylin's binary package
new fb52a1d KYLIN-3818 After Cube disabled, auto-merge cube job still
running
new a1ade72 KYLIN-3830 return wrong result when 'SELECT SUM(dim1)'
without set a relative metric of dim1.
new 92e9299 KYLIN-3795 minor code refactor
new a6aa9c8 KYLIN-3865 Centralize the zookeeper related info
new 6a80d36 KYLIN-3867 leverage system properties to enable JDBC to use
key store & trust store for https connection
new 699d650 KYLIN-3867 leverage jdbc specified properties to enable JDBC
to use key store & trust store for https connection
new fe43a3e KYLIN-3862 add LICENSE and NOTICE to binary package
new d23d2bd KYLIN-3866 Whether to set mapreduce.application.classpath is
determined by the user
new 6b9a3d2 KYLIN-3571 give a message to user when spark not found
new 7b88a99 add a copy phase for copying jacoco related jars to
dev-support folder
new 756d582 minor, move Move m2e lifecycle-mapping into m2e-only profile
to avoid spurious warnings
new 0121af2 KYLIN-3878 disable sonar check before the problem be
identified
new 4706fe8 KYLIN-3878 remove the comment
new 90879fb KYLIN-3817: Duration in Cube building is a negative number
new aa61c1a KYLIN-3864 Provide a function to judge whether the os type is
Mac os x or not
new e16d2cf KYLIN-3882: kylin master build failed for pom issues
new 9f7ec94 KYLIN-3874 "Convert Cuboid Data to HFile" failed when HBase
and MR use different HDFS clusters
new 0431554 KYLIN-3880: DataType is incompatible in Kylin HBase
coprocessor
new 9c8d30b KYLIN-3888 TableNotDisabledException when running "Convert
Lookup Table to HFile"
new 62d5b03 KYLIN-3839 Strorage clean up after refreshing and deleting
segment
new 980b967 KYLIN-3884: loading hfile to HBase failed for temporary dir
in output path
new 4fa40bd Add log for deleting temporary path
new 24d08d2 KYLIN-3886: Missing argument for options for yarn command
new 4abe712 Remove a blank
new 14d0a37 KYLIN-3474 Tableau 10.5 get malformed token
new d9b028b KYLIN-3756 Support check-port-availability script for mac os x
new 6ee0212 KYLIN-3895 Failed to register new MBean when
"kylin.server.query-metrics-enabled" set true
new c05ae0c KYLIN-3691 New streaming ui implementation
new 0943599 KYLIN-3690 New streaming backend implementation
new dd4ea5b KYLIN-3690 New streaming backend implementation - streaming
consumer
new 363749e KYLIN-3690 New streaming backend implementation - streaming
job
new 6da9795 KYLIN-3730 TableMetadataManager.reloadSourceTableQuietly is
wrong
new 4a1a4de KYLIN-3742 Fix DataRequest for NPE and add some javadoc
new 4e370b2 KYLIN-3745 Real-time segment state changed from active to
immutable is not sequently
new b253b2c KYLIN-3747 Use FQDN to register a streaming receiver instead
of ip
new 1516d8c KYLIN-3768 Save streaming metadata a standard kylin path in
zookeeper
new 63268a0 KYLIN-3759 Fix classnotfound if lambda is enable
new 13b7612 KYLIN-3744 Make coordinator well tested and add some javadoc
new b7a52af KYLIN-3784 Hadoop Common ReflectionUtils.printThreadInfo
signature change
new 2cbb8cd KYLIN-3787 NPE throws when dimension value has null when
query real-time data
new bf446aa KYLIN-3791 Map return by Maps.transformValues is a immutable
view
new 0669cee KYLIN-3789 Stream receiver admin page issue fix
new 6cd7061 KYLIN-3800 Measure get incorrect result
new 14be024 KYLIN-3786 Add integration test for realtime-streaming
new ad9b49b Minor, fix rat check failed
new 9a363c1 KYLIN-3744 Add configuration and fix bugs
new e70a6f7 KYLIN-3821 Add consume lag stats
new 1b586bc KYLIN-3654 rebase master
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (74655f1)
\
N -- N -- N refs/heads/realtime-streaming (1b586bc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 7700 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
.travis.yml | 7 +-
LICENSE | 7 +
assembly/pom.xml | 2 +-
.../test/java/org/apache/kylin/job/DeployUtil.java | 56 +-
.../job/streaming/StreamingTableDataGenerator.java | 21 +-
build/bin/check-port-availability.sh | 8 +-
build/bin/download-spark.sh | 73 +++
build/bin/find-hive-dependency.sh | 10 +-
build/bin/find-spark-dependency.sh | 5 +
build/bin/kylin.sh | 26 +-
...check-port-availability.sh => set-java-home.sh} | 25 +-
build/bin/{check-port-availability.sh => util.sh} | 18 +-
build/script/compress.sh | 7 +-
build/script/package.sh | 1 -
cache/pom.xml | 2 +-
core-common/pom.xml | 2 +-
.../java/org/apache/kylin/common/KylinConfig.java | 24 +-
.../org/apache/kylin/common/KylinConfigBase.java | 142 ++++--
.../java/org/apache/kylin/common/KylinVersion.java | 8 +-
.../java/org/apache/kylin/common/QueryContext.java | 25 +-
.../apache/kylin/common/QueryContextFacade.java | 3 +-
.../java/org/apache/kylin/common/ServerMode.java | 71 +++
.../apache/kylin/common/debug/BackdoorToggles.java | 3 +-
.../apache/kylin/common/livy/LivyRestBuilder.java | 155 ++++++
.../apache/kylin/common/livy/LivyRestClient.java | 136 +++++
.../apache/kylin/common/livy/LivyRestExecutor.java | 122 +++++
.../apache/kylin/common/livy/LivyStateEnum.java | 17 +-
.../org/apache/kylin/common/livy/LivyTypeEnum.java | 17 +-
.../kylin/common/lock/DistributedLockFactory.java | 2 +-
.../common/metrics/metrics2/CodahaleMetrics.java | 3 +-
.../metrics/metrics2/JsonFileMetricsReporter.java | 13 +-
.../common/metrics/perflog/PerfLoggerFactory.java | 3 +-
.../kylin/common/persistence/ContentWriter.java | 10 +-
.../persistence/ExponentialBackoffRetry.java | 3 +-
.../common/persistence/FileResourceStore.java | 11 +-
.../common/persistence/HDFSResourceStore.java | 14 +-
.../common/persistence/JDBCConnectionManager.java | 3 +-
.../common/persistence/JDBCResourceStore.java | 21 +-
.../kylin/common/persistence/ResourceStore.java | 3 +-
.../kylin/common/persistence/ResourceTool.java | 12 +-
.../apache/kylin/common/restclient/RestClient.java | 65 ++-
.../kylin/common/threadlocal/InternalThread.java | 74 +++
.../common/threadlocal/InternalThreadLocal.java | 200 ++++++++
.../common/threadlocal/InternalThreadLocalMap.java | 169 ++++++
.../kylin/common/util/CliCommandExecutor.java | 2 +-
.../org/apache/kylin/common/util/HadoopUtil.java | 27 +-
.../org/apache/kylin/common/util/SSHClient.java | 7 +-
.../kylin/common/util/SourceConfigurationUtil.java | 3 +-
.../org/apache/kylin/common/util/StringUtil.java | 34 +-
.../java/org/apache/kylin/common/util/ZKUtil.java | 231 +++++++++
.../apache/kylin/common/util/ZooKeeperUtil.java | 63 ---
.../src/main/resources/kylin-defaults.properties | 6 +-
.../apache/kylin/common/util/StringUtilTest.java | 23 +
core-cube/pom.xml | 2 +-
.../apache/kylin/cube/CubeCapabilityChecker.java | 21 +-
.../org/apache/kylin/cube/CubeDescManager.java | 26 +-
.../java/org/apache/kylin/cube/CubeInstance.java | 30 +-
.../java/org/apache/kylin/cube/CubeManager.java | 55 +-
.../java/org/apache/kylin/cube/CubeUpdate.java | 20 +-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 15 +-
.../apache/kylin/cube/common/SegmentPruner.java | 4 +
.../cube/cuboid/algorithm/BPUSCalculator.java | 2 +-
.../cube/cuboid/algorithm/CuboidRecommender.java | 72 +--
.../kylin/cube/cuboid/algorithm/CuboidStats.java | 131 ++---
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 226 ++++++--
.../cuboid/algorithm/greedy/GreedyAlgorithm.java | 62 +--
.../cube/inmemcubing/CompoundCuboidWriter.java | 8 +
.../kylin/cube/inmemcubing/DoggedCubeBuilder.java | 25 +-
.../cube/inmemcubing/ICuboidGTTableWriter.java | 47 ++
.../kylin/cube/inmemcubing/ICuboidWriter.java | 3 +
.../kylin/cube/inmemcubing/InMemCubeBuilder.java | 56 +-
.../apache/kylin/cube/inmemcubing2/CuboidTask.java | 53 ++
.../DefaultCuboidCollectorWithCallBack.java | 53 ++
.../cube/inmemcubing2/DoggedCubeBuilder2.java | 442 ++++++++++++++++
.../ICuboidCollectorWithCallBack.java} | 62 +--
.../ICuboidResultListener.java} | 59 +--
.../kylin/cube/inmemcubing2/InMemCubeBuilder2.java | 409 +++++++++++++++
.../java/org/apache/kylin/gridtable/GTUtil.java | 4 +
.../org/apache/kylin/cube/CubeInstanceTest.java | 12 +
.../org/apache/kylin/cube/CubeManagerTest.java | 174 ++++---
.../cube/cuboid/algorithm/CuboidStatsUtilTest.java | 162 +++++-
core-dictionary/pom.xml | 2 +-
.../apache/kylin/dict/Number2BytesConverter.java | 3 +-
core-job/pom.xml | 17 +-
.../org/apache/kylin/engine/EngineFactory.java | 3 +-
.../java/org/apache/kylin/job/JobInstance.java | 11 +
.../java/org/apache/kylin/job/JoinedFlatTable.java | 6 +-
.../org/apache/kylin/job/SchedulerFactory.java | 3 +-
.../org/apache/kylin/job/dao/ExecutableDao.java | 34 +-
.../kylin/job/execution/AbstractExecutable.java | 25 +-
.../job/execution/DefaultChainedExecutable.java | 2 +
.../job/impl/curator/CuratorLeaderSelector.java | 116 +++++
.../kylin/job/impl/curator/CuratorScheduler.java | 268 ++++++++++
.../job/impl/threadpool/DefaultFetcherRunner.java | 14 +-
.../job/impl/threadpool/DefaultScheduler.java | 23 +-
.../job/impl/threadpool/DistributedScheduler.java | 20 +-
.../kylin/job/impl/threadpool/FetcherRunner.java | 9 +-
.../job/impl/threadpool/PriorityFetcherRunner.java | 14 +-
.../lock/zookeeper}/ZookeeperDistributedLock.java | 99 +---
.../job/lock/zookeeper}/ZookeeperJobLock.java | 2 +-
.../kylin/job/util/FlatTableSqlQuoteUtils.java | 24 +
.../org/apache/kylin/job/ErrorTestExecutable.java | 2 +-
.../impl/curator/CuratorLeaderSelectorTest.java | 107 ++++
.../job/impl/curator/CuratorSchedulerTest.java | 136 +++++
.../kylin/job/impl/curator/ExampleServer.java | 68 +++
.../job/impl/threadpool/DefaultSchedulerTest.java | 24 +-
core-metadata/pom.xml | 2 +-
.../measure/percentile/PercentileSerializer.java | 6 +-
.../kylin/measure/topn/DoubleDeltaSerializer.java | 6 +-
.../apache/kylin/measure/topn/TopNMeasureType.java | 48 +-
.../kylin/metadata/cachesync/Broadcaster.java | 9 +-
.../apache/kylin/metadata/datatype/DataType.java | 10 +-
.../metadata/datatype/DataTypeSerializer.java | 5 +-
.../apache/kylin/metadata/filter/TupleFilter.java | 30 +-
.../metadata/filter/function/BuiltInMethod.java | 3 +-
.../kylin/metadata/model/DataModelManager.java | 2 +-
.../apache/kylin/metadata/model/FunctionDesc.java | 3 +-
.../org/apache/kylin/metadata/model/Segments.java | 2 +-
.../kylin/metadata/realization/SQLDigest.java | 8 +-
.../org/apache/kylin/source/SourceManager.java | 3 +-
.../main/java/org/apache/kylin/util/KryoUtils.java | 3 +-
.../kylin/metadata/filter/TupleFilterTest.java | 64 +++
core-metrics/pom.xml | 2 +-
.../apache/kylin/metrics/lib/impl/RecordEvent.java | 3 +-
.../metrics/lib/impl/RecordEventTimeDetail.java | 5 +-
core-storage/pom.xml | 2 +-
.../org/apache/kylin/storage/StorageFactory.java | 3 +-
.../storage/gtrecord/CubeScanRangePlanner.java | 9 +-
.../storage/gtrecord/GTCubeStorageQueryBase.java | 10 +-
datasource-sdk/pom.xml | 2 +-
.../datasource/adaptor/AbstractJdbcAdaptor.java | 10 +-
.../sdk/datasource/adaptor/DefaultAdaptor.java | 122 +++--
.../sdk/datasource/framework/JdbcConnector.java | 4 +-
.../framework/conv/DefaultConfiguer.java | 9 +-
.../datasource/framework/conv/SqlConverter.java | 6 +
.../framework/def/DataSourceDefProvider.java | 10 +-
.../src/main/resources/datasource/mysql.xml | 9 +-
.../framework/conv/SqlConverterTest.java | 62 ++-
.../framework/def/DataSourceDefTest.java | 7 +
.../test/resources/datasource/default.xml.override | 202 ++++++++
dev-support/jacocoagent.jar | Bin 257327 -> 0 bytes
engine-mr/pom.xml | 2 +-
.../kylin/engine/mr/BatchMergeJobBuilder2.java | 1 -
.../java/org/apache/kylin/engine/mr/IInput.java | 31 +-
.../java/org/apache/kylin/engine/mr/IMRInput.java | 24 +-
.../java/org/apache/kylin/engine/mr/MRUtil.java | 7 +-
.../kylin/engine/mr/common/AbstractHadoopJob.java | 17 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../engine/mr/common/CuboidRecommenderUtil.java | 49 +-
.../engine/mr/common/CuboidStatsReaderUtil.java | 23 +
.../kylin/engine/mr/common/HadoopCmdOutput.java | 74 ++-
.../kylin/engine/mr/common/JobInfoConverter.java | 2 +
.../engine/mr/steps/FactDistinctColumnsJob.java | 22 +-
.../mr/steps/FactDistinctColumnsReducer.java | 2 +-
.../mr/steps/InMemCuboidFromBaseCuboidMapper.java | 25 +-
.../kylin/engine/mr/steps/InMemCuboidMapper.java | 23 +-
.../engine/mr/steps/InMemCuboidMapperBase.java | 30 +-
.../kylin/engine/mr/steps/KVGTRecordWriter.java | 4 +-
.../kylin/engine/mr/steps/MergeCuboidJob.java | 6 +-
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 6 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 3 +
engine-spark/pom.xml | 2 +-
.../org/apache/kylin/engine/spark/ISparkInput.java | 24 +-
.../kylin/engine/spark/KylinKryoRegistrator.java | 2 +
.../engine/spark/SparkBatchCubingJobBuilder2.java | 4 +-
.../engine/spark/SparkBatchMergeJobBuilder2.java | 4 +-
.../apache/kylin/engine/spark/SparkExecutable.java | 29 +-
.../kylin/engine/spark/SparkExecutableFactory.java | 21 +-
.../kylin/engine/spark/SparkExecutableLivy.java | 276 ++++++++++
.../kylin/engine/spark/SparkFactDistinct.java | 10 +-
.../kylin/engine/spark/SparkSqlOnLivyBatch.scala | 52 ++
.../org/apache/kylin/engine/spark/SparkUtil.java | 73 +--
.../engine/spark/exception/SparkException.java | 25 +-
.../cube/test_streaming_join_table_cube.json | 17 +
.../cube_desc/test_streaming_join_table_cube.json | 151 ++++++
.../localmeta/data/DEFAULT.STREAMING_CATEGORY.csv | 10 +
examples/test_case_data/localmeta/kylin.properties | 2 +
.../test_streaming_join_table_model.json | 39 ++
.../test_case_data/localmeta/project/default.json | 6 +
.../table/DEFAULT.STREAMING_CATEGORY.json | 22 +
.../localmeta/table/DEFAULT.STREAMING_TABLE.json | 5 +
examples/test_case_data/sandbox/kylin.properties | 2 +-
jdbc/pom.xml | 2 +-
.../java/org/apache/kylin/jdbc/KylinClient.java | 58 ++-
kylin-it/pom.xml | 57 ++-
.../inmemcubing/ITDoggedCubeBuilderStressTest.java | 6 +
.../cube/inmemcubing/ITDoggedCubeBuilderTest.java | 73 ++-
.../cube/inmemcubing/ITInMemCubeBuilderTest.java | 7 +
.../kylin/job/BaseTestDistributedScheduler.java | 24 +-
.../kylin/provision/BuildCubeWithEngine.java | 2 +-
.../kylin/provision/BuildCubeWithStream.java | 122 +++--
.../java/org/apache/kylin/query/H2Database.java | 1 +
.../org/apache/kylin/query/ITKylinQueryTest.java | 22 +-
.../java/org/apache/kylin/query/KylinTestBase.java | 2 +
.../kylin/realtime/BuildCubeWithStreamV2.java | 12 +-
.../apache/kylin/storage/hbase/ITStorageTest.java | 3 +-
.../hbase/ITZookeeperDistributedLockTest.java | 2 +-
.../query/sql_distinct_precisely/query02.sql | 2 +-
.../query/sql_distinct_precisely/query04.sql | 2 +-
.../query02.sql => sql_limit/query06.sql} | 16 +-
.../query02.sql => sql_streaming/query11.sql} | 9 +-
.../query02.sql => sql_streaming/query12.sql} | 9 +-
.../query02.sql => sql_streaming/query13.sql} | 9 +-
.../query02.sql => sql_streaming/query14.sql} | 9 +-
.../test/resources/query/sql_subquery/query35.sql | 2 +
.../query02.sql => sql_subquery/query37.sql} | 20 +-
metrics-reporter-hive/pom.xml | 2 +-
metrics-reporter-kafka/pom.xml | 2 +-
pom.xml | 125 +++--
query/pom.xml | 2 +-
.../query/enumerator/LookupTableEnumerator.java | 12 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 8 +
.../apache/kylin/query/relnode/OLAPContext.java | 14 +-
.../apache/kylin/query/relnode/OLAPLimitRel.java | 5 +-
.../org/apache/kylin/query/relnode/OLAPRel.java | 9 +-
.../apache/kylin/query/relnode/OLAPSortRel.java | 19 +-
.../query/relnode/visitor/TupleFilterVisitor.java | 83 +--
.../org/apache/kylin/query/util/QueryUtil.java | 11 +-
.../relnode/visitor/TupleFilterVisitorTest.java | 86 ++++
server-base/pom.xml | 8 +-
.../kylin/rest/controller/CubeController.java | 101 ++--
.../kylin/rest/controller/DashboardController.java | 66 +--
.../kylin/rest/controller/ProjectController.java | 6 +-
.../apache/kylin/rest/job/MetadataCleanupJob.java | 49 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 8 +-
.../java/org/apache/kylin/rest/msg/MsgPicker.java | 4 +-
.../org/apache/kylin/rest/request/SQLRequest.java | 11 +-
.../apache/kylin/rest/security/ManagedUser.java | 2 +-
.../org/apache/kylin/rest/service/CubeService.java | 130 ++++-
.../kylin/rest/service/DashboardService.java | 34 +-
.../org/apache/kylin/rest/service/JobService.java | 38 +-
.../apache/kylin/rest/service/QueryService.java | 14 +-
.../rest/service/TableSchemaUpdateChecker.java | 160 +++++-
.../apache/kylin/rest/service/TableService.java | 2 +-
server/pom.xml | 7 +-
.../apache/kylin/rest/service/ServiceTestBase.java | 10 +-
source-hive/pom.xml | 2 +-
.../source/hive/CreateFlatHiveTableByLivyStep.java | 124 +++++
.../apache/kylin/source/hive/HiveInputBase.java | 130 ++++-
.../org/apache/kylin/source/hive/HiveMRInput.java | 103 +---
.../apache/kylin/source/hive/HiveSparkInput.java | 84 +--
.../hive/RedistributeFlatHiveTableByLivyStep.java | 149 ++++++
source-jdbc/pom.xml | 2 +-
...JdbcHiveMRInput.java => JdbcHiveInputBase.java} | 49 +-
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java | 225 +-------
.../kylin/source/jdbc/JdbcHiveSparkInput.java | 48 ++
.../org/apache/kylin/source/jdbc/JdbcSource.java | 3 +
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 2 +-
...JdbcHiveMRInput.java => JdbcHiveInputBase.java} | 30 +-
.../source/jdbc/extensible/JdbcHiveMRInput.java | 116 +----
.../source/jdbc/extensible/JdbcHiveSparkInput.java | 55 ++
.../kylin/source/jdbc/extensible/JdbcSource.java | 3 +
.../jdbc/metadata/SQLServerJdbcMetadata.java | 13 +-
.../org/apache/kylin/source/jdbc/SqlUtilTest.java | 33 +-
.../jdbc/extensible/JdbcHiveMRInputTest.java | 12 +-
.../jdbc/metadata/SQLServerJdbcMetadataTest.java | 4 +-
source-kafka/pom.xml | 2 +-
.../apache/kylin/source/kafka/KafkaInputBase.java | 177 +++----
.../apache/kylin/source/kafka/KafkaMRInput.java | 90 +---
.../org/apache/kylin/source/kafka/KafkaSource.java | 4 +-
.../apache/kylin/source/kafka/KafkaSparkInput.java | 86 +---
.../kylin/source/kafka/TimedJsonStreamParser.java | 2 +
.../source/kafka/hadoop/KafkaFlatTableMapper.java | 5 +-
.../kafka/model/StreamCubeFactTableDesc.java | 133 +++++
.../source/kafka/util/KafkaSampleProducer.java | 12 +-
.../kafka/hadoop/KafkaInputRecordReaderTest.java | 60 +++
storage-hbase/pom.xml | 15 +-
.../kylin/storage/hbase/HBaseConnection.java | 5 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 12 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +-
.../hbase/lookup/LookupTableToHFileJob.java | 4 +-
.../kylin/storage/hbase/steps/BulkLoadJob.java | 36 +-
.../kylin/storage/hbase/steps/CubeHFileJob.java | 13 +
.../kylin/storage/hbase/steps/HBaseJobSteps.java | 98 ++--
.../hbase/steps/HBaseSparkOutputTransition.java | 2 +-
.../kylin/storage/hbase/steps/HBaseSparkSteps.java | 8 +-
.../storage/hbase/util/DeployCoprocessorCLI.java | 9 +-
.../kylin/storage/hbase/util/RowCounterCLI.java | 7 +-
.../kylin/storage/hbase/util/StorageCleanUtil.java | 93 ++++
.../kylin/storage/hbase/util/ZookeeperUtil.java | 55 --
.../coprocessor/endpoint/CubeVisitServiceTest.java | 568 +++++++++++++++++++++
storage-stream/pom.xml | 2 +-
stream-coordinator/pom.xml | 2 +-
.../apache/kylin/stream/coordinator/ZKUtils.java | 3 +-
stream-core/pom.xml | 2 +-
stream-receiver/pom.xml | 2 +-
stream-source-kafka/pom.xml | 2 +-
tomcat-ext/pom.xml | 2 +-
tool-assembly/pom.xml | 2 +-
tool/pom.xml | 2 +-
.../apache/kylin/tool/AbstractInfoExtractor.java | 12 +
.../org/apache/kylin/tool/CubeMetaExtractor.java | 2 +
.../apache/kylin/tool/JobInstanceExtractor.java | 1 +
.../java/org/apache/kylin/tool/KylinConfigCLI.java | 2 +-
.../tool/metrics/systemcube/CubeDescCreator.java | 1 -
.../apache/kylin/tool/CubeMetaExtractorTest.java | 1 +
webapp/app/js/controllers/cubeAdvanceSetting.js | 24 +-
webapp/app/js/controllers/cubeMeasures.js | 1 -
webapp/app/js/controllers/sourceMeta.js | 4 +-
webapp/app/js/services/kylinProperties.js | 6 +-
.../partials/cubeDesigner/advanced_settings.html | 143 +++---
webapp/app/partials/jobs/job_steps.html | 4 +
303 files changed, 8764 insertions(+), 2715 deletions(-)
create mode 100755 build/bin/download-spark.sh
copy build/bin/{check-port-availability.sh => set-java-home.sh} (61%)
copy build/bin/{check-port-availability.sh => util.sh} (61%)
create mode 100644
core-common/src/main/java/org/apache/kylin/common/ServerMode.java
create mode 100644
core-common/src/main/java/org/apache/kylin/common/livy/LivyRestBuilder.java
create mode 100644
core-common/src/main/java/org/apache/kylin/common/livy/LivyRestClient.java
create mode 100644
core-common/src/main/java/org/apache/kylin/common/livy/LivyRestExecutor.java
copy
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java =>
core-common/src/main/java/org/apache/kylin/common/livy/LivyStateEnum.java (73%)
copy
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java =>
core-common/src/main/java/org/apache/kylin/common/livy/LivyTypeEnum.java (73%)
create mode 100644
core-common/src/main/java/org/apache/kylin/common/threadlocal/InternalThread.java
create mode 100644
core-common/src/main/java/org/apache/kylin/common/threadlocal/InternalThreadLocal.java
create mode 100644
core-common/src/main/java/org/apache/kylin/common/threadlocal/InternalThreadLocalMap.java
create mode 100644
core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java
delete mode 100644
core-common/src/main/java/org/apache/kylin/common/util/ZooKeeperUtil.java
create mode 100755
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidGTTableWriter.java
create mode 100755
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing2/CuboidTask.java
create mode 100755
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing2/DefaultCuboidCollectorWithCallBack.java
create mode 100755
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing2/DoggedCubeBuilder2.java
copy
core-cube/src/main/java/org/apache/kylin/cube/{inmemcubing/ICuboidWriter.java
=> inmemcubing2/ICuboidCollectorWithCallBack.java} (72%)
mode change 100644 => 100755
copy
core-cube/src/main/java/org/apache/kylin/cube/{inmemcubing/ICuboidWriter.java
=> inmemcubing2/ICuboidResultListener.java} (72%)
mode change 100644 => 100755
create mode 100755
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing2/InMemCubeBuilder2.java
create mode 100644
core-job/src/main/java/org/apache/kylin/job/impl/curator/CuratorLeaderSelector.java
create mode 100644
core-job/src/main/java/org/apache/kylin/job/impl/curator/CuratorScheduler.java
rename {storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util =>
core-job/src/main/java/org/apache/kylin/job/lock/zookeeper}/ZookeeperDistributedLock.java
(69%)
rename {storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util =>
core-job/src/main/java/org/apache/kylin/job/lock/zookeeper}/ZookeeperJobLock.java
(98%)
create mode 100644
core-job/src/test/java/org/apache/kylin/job/impl/curator/CuratorLeaderSelectorTest.java
create mode 100644
core-job/src/test/java/org/apache/kylin/job/impl/curator/CuratorSchedulerTest.java
create mode 100644
core-job/src/test/java/org/apache/kylin/job/impl/curator/ExampleServer.java
create mode 100644
datasource-sdk/src/test/resources/datasource/default.xml.override
delete mode 100644 dev-support/jacocoagent.jar
copy engine-spark/src/main/java/org/apache/kylin/engine/spark/ISparkInput.java
=> engine-mr/src/main/java/org/apache/kylin/engine/mr/IInput.java (66%)
copy
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java =>
engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableFactory.java
(64%)
create mode 100644
engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableLivy.java
create mode 100644
engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkSqlOnLivyBatch.scala
copy
core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ICuboidWriter.java =>
engine-spark/src/main/java/org/apache/kylin/engine/spark/exception/SparkException.java
(60%)
create mode 100644
examples/test_case_data/localmeta/cube/test_streaming_join_table_cube.json
create mode 100644
examples/test_case_data/localmeta/cube_desc/test_streaming_join_table_cube.json
create mode 100644
examples/test_case_data/localmeta/data/DEFAULT.STREAMING_CATEGORY.csv
create mode 100644
examples/test_case_data/localmeta/model_desc/test_streaming_join_table_model.json
create mode 100644
examples/test_case_data/localmeta/table/DEFAULT.STREAMING_CATEGORY.json
copy kylin-it/src/test/resources/query/{sql_distinct_precisely/query02.sql =>
sql_limit/query06.sql} (76%)
copy kylin-it/src/test/resources/query/{sql_distinct_precisely/query02.sql =>
sql_streaming/query11.sql} (76%)
copy kylin-it/src/test/resources/query/{sql_distinct_precisely/query02.sql =>
sql_streaming/query12.sql} (76%)
copy kylin-it/src/test/resources/query/{sql_distinct_precisely/query02.sql =>
sql_streaming/query13.sql} (73%)
copy kylin-it/src/test/resources/query/{sql_distinct_precisely/query02.sql =>
sql_streaming/query14.sql} (75%)
copy kylin-it/src/test/resources/query/{sql_distinct_precisely/query02.sql =>
sql_subquery/query37.sql} (61%)
mode change 100644 => 100755
query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
mode change 100644 => 100755
query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
mode change 100644 => 100755
query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
create mode 100644
query/src/test/java/org/apache/kylin/query/relnode/visitor/TupleFilterVisitorTest.java
create mode 100644
source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableByLivyStep.java
create mode 100644
source-hive/src/main/java/org/apache/kylin/source/hive/RedistributeFlatHiveTableByLivyStep.java
copy
source-jdbc/src/main/java/org/apache/kylin/source/jdbc/{JdbcHiveMRInput.java =>
JdbcHiveInputBase.java} (93%)
create mode 100644
source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveSparkInput.java
copy
source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/{JdbcHiveMRInput.java
=> JdbcHiveInputBase.java} (85%)
create mode 100644
source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveSparkInput.java
create mode 100644
source-kafka/src/main/java/org/apache/kylin/source/kafka/model/StreamCubeFactTableDesc.java
create mode 100644
source-kafka/src/test/java/org/apache/kylin/source/kafka/hadoop/KafkaInputRecordReaderTest.java
create mode 100644
storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanUtil.java
delete mode 100644
storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperUtil.java
create mode 100644
storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitServiceTest.java
