This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch kylin-on-parquet
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 011ff08 KYLIN-3726: KylinSession should load spark properties from
spark-defaults.conf
new 8a0d3f6 KYLIN-3777 Remove dim dict from parquet file
new 8f3195e KYLIN-3777 Remove dict from query side
new 05e4bd2 KYLIN-3777 Pass CI
new 768d7ca KYLIN-3777 Fix missing statistics resources
The 7546 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/common/KylinConfigBase.java | 6 +-
.../org/apache/kylin/common/util/TimeUtil.java | 53 +++
.../java/org/apache/kylin/gridtable/GTUtil.java | 136 +-------
.../filter/BuiltInFunctionTupleFilter.java | 13 +
.../kylin/metadata/filter/CaseTupleFilter.java | 8 +
.../kylin/metadata/filter/ColumnTupleFilter.java | 10 +
.../kylin/metadata/filter/CompareTupleFilter.java | 20 ++
.../kylin/metadata/filter/ConstantTupleFilter.java | 10 +
.../kylin/metadata/filter/DynamicTupleFilter.java | 10 +
.../kylin/metadata/filter/ExtractTupleFilter.java | 10 +
.../kylin/metadata/filter/FunctionTupleFilter.java | 5 +
.../kylin/metadata/filter/LogicalTupleFilter.java | 10 +
.../apache/kylin/metadata/filter/TupleFilter.java | 17 +
.../kylin/metadata/filter/TupleFilterVisitor.java | 22 ++
.../kylin/metadata/filter/TupleFilterVisitor2.java | 42 +++
.../filter/TupleFilterVisitor2Adaptor.java | 100 ++++++
.../apache/kylin/metadata/filter/TupleFilters.java | 146 ++++++++
.../metadata/filter/UDF/MassInTupleFilter.java | 5 +
.../metadata/filter/UnsupportedTupleFilter.java | 10 +
.../org/apache/kylin/storage/IStorageQuery.java | 4 +
.../org/apache/kylin/storage/StorageContext.java | 8 +
.../storage/gtrecord/CubeScanRangePlanner.java | 10 -
.../kylin/storage/hybrid/HybridStorageQuery.java | 12 +
.../kylin/engine/mr/common/AbstractHadoopJob.java | 1 +
kylin-it/jacoco-it.exec | Bin 2432563 -> 0 bytes
.../org/apache/kylin/jdbc/ITJDBCDriver2Test.java | 9 +
.../apache/kylin/query/ITFailfastQuery2Test.java | 6 +
.../org/apache/kylin/query/ITKylinQuery2Test.java | 81 ++++-
.../org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../java/org/apache/kylin/query/KylinTestBase.java | 6 +-
.../apache/kylin/query/relnode/OLAPFilterRel.java | 21 +-
.../apache/kylin/query/relnode/OLAPProjectRel.java | 12 +-
.../query/relnode/visitor/TupleFilterVisitor.java | 1 +
.../kylin/rest/service/KylinQueryTimeoutTest.java | 5 +
.../storage/hbase/cube/v2/CubeStorageQuery.java | 5 +
.../apache/kylin/storage/parquet/NameMapping.java | 15 +-
.../kylin/storage/parquet/NameMappingFactory.java | 111 ++++++
.../kylin/storage/parquet/ParquetSchema.java | 151 ++++++++
.../kylin/storage/parquet/ParquetStorage.java | 4 +-
.../kylin/storage/parquet/cube/ColumnFiller.java | 7 +-
.../kylin/storage/parquet/cube/CubeSparkRPC.java | 106 ------
.../storage/parquet/cube/DerivedColumnFiller.java | 56 +++
.../storage/parquet/cube/DerivedIndexMapping.java | 91 +++++
...CubeStorageQuery.java => LookupTableCache.java} | 31 +-
.../storage/parquet/cube/ParquetColumnFiller.java | 94 +++++
.../storage/parquet/cube/ParquetStorageQuery.java | 381 +++++++++++++++++++++
.../storage/parquet/cube/ParquetTupleIterator.java | 162 +++++++++
.../{CubeStorageQuery.java => PkColumnFiller.java} | 20 +-
.../storage/parquet/cube/ToSparkFilterVisitor.java | 301 ++++++++++++++++
.../parquet/cube/TranslateDerivedVisitor.java | 334 ++++++++++++++++++
.../parquet/cube/UnsupportedFilterException.java | 15 +-
.../storage/parquet/spark/ParquetPayload.java | 220 ------------
.../kylin/storage/parquet/spark/ParquetTask.java | 309 -----------------
.../storage/parquet/spark/SparkSubmitter.java | 37 --
.../spark/gtscanner/ParquetRecordGTScanner.java | 139 --------
.../gtscanner/ParquetRecordGTScanner4Cube.java | 32 --
.../parquet/steps/ConvertToParquetReducer.java | 22 +-
.../storage/parquet/steps/MRCubeParquetJob.java | 21 +-
.../storage/parquet/steps/ParquetConvertor.java | 238 ++++---------
.../storage/parquet/steps/SparkCubeParquet.java | 46 ++-
60 files changed, 2505 insertions(+), 1254 deletions(-)
create mode 100644
core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterVisitor.java
create mode 100644
core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterVisitor2.java
create mode 100644
core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterVisitor2Adaptor.java
create mode 100644
core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilters.java
delete mode 100644 kylin-it/jacoco-it.exec
copy
core-metadata/src/main/java/org/apache/kylin/metadata/tuple/IEvaluatableTuple.java
=>
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/NameMapping.java
(78%)
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/NameMappingFactory.java
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/ParquetSchema.java
copy jdbc/src/main/java/org/apache/kylin/jdbc/JdbcFactory.java =>
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ColumnFiller.java
(82%)
delete mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/CubeSparkRPC.java
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/DerivedColumnFiller.java
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/DerivedIndexMapping.java
copy
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/{CubeStorageQuery.java
=> LookupTableCache.java} (50%)
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ParquetColumnFiller.java
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ParquetStorageQuery.java
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ParquetTupleIterator.java
rename
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/{CubeStorageQuery.java
=> PkColumnFiller.java} (66%)
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/ToSparkFilterVisitor.java
create mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/TranslateDerivedVisitor.java
copy core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java =>
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/cube/UnsupportedFilterException.java
(77%)
delete mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/ParquetPayload.java
delete mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/ParquetTask.java
delete mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/SparkSubmitter.java
delete mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/gtscanner/ParquetRecordGTScanner.java
delete mode 100644
storage-parquet/src/main/java/org/apache/kylin/storage/parquet/spark/gtscanner/ParquetRecordGTScanner4Cube.java
