[15/16] kylin git commit: half way
half way Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/52d0bc17 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/52d0bc17 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/52d0bc17 Branch: refs/heads/KYLIN-2033 Commit: 52d0bc1703d89b68548005b071e4da9bb25234ed Parents: 478066d Author: Yang Li Authored: Thu Sep 22 21:03:39 2016 +0800 Committer: Yang Li Committed: Thu Sep 22 21:04:05 2016 +0800 -- .../kylin/metadata/project/ProjectManager.java | 3 -- .../kylin/storage/hybrid/HybridManager.java | 54 .../engine/streaming/StreamingManager.java | 14 ++--- .../kylin/source/kafka/KafkaConfigManager.java | 18 --- 4 files changed, 51 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/52d0bc17/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 972d40f..be69df3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -31,9 +31,6 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.common.restclient.Broadcaster; import org.apache.kylin.common.restclient.Broadcaster.Event; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.cuboid.Cuboid; -import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.common.restclient.CaseInsensitiveStringCache; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; http://git-wip-us.apache.org/repos/asf/kylin/blob/52d0bc17/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java -- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java index f772777..d73a1a9 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java @@ -29,8 +29,9 @@ import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; import org.apache.kylin.common.restclient.Broadcaster; import org.apache.kylin.common.restclient.Broadcaster.Event; -import org.apache.kylin.cube.CubeManager.SyncListener; import org.apache.kylin.common.restclient.CaseInsensitiveStringCache; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.project.RealizationEntry; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.IRealizationProvider; @@ -85,42 +86,49 @@ public class HybridManager implements IRealizationProvider { private HybridManager(KylinConfig config) throws IOException { logger.info("Initializing HybridManager with config " + config); this.config = config; -this.hybridMap = new CaseInsensitiveStringCache(config, "hybrid"); -Broadcaster.getInstance(config).registerListener(new SyncListener(), "hybrid", "cube"); -loadAllHybridInstance(); +this.hybridMap = new CaseInsensitiveStringCache(config, "hybrid", new SyncListener()); +reloadAllHybridInstance(); } -private class SyncListener implements Broadcaster.Listener { +private class SyncListener extends Broadcaster.Listener { + @Override -public void clearAll() { -// TODO Auto-generated method stub - +public void onClearAll(Broadcaster broadcaster) throws IOException { +clearCache(); } @Override -public void notify(String entity, Event event, String cacheKey) { -if (event == Event.CREATE || event == Event.UPDATE) { -switch (entity) { -case "hybrid": -loadAllHybridInstance(); -break; -case "cube": -reloadHybridInstanceByChild(RealizationType.CUBE, cacheKey); -break; +public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException { +for (IRealization real : ProjectManager.getInstance(config).listAllRealizations(p
[14/16] kylin git commit: half way
half way Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/478066d0 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/478066d0 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/478066d0 Branch: refs/heads/KYLIN-2033 Commit: 478066d03509db845c7b2c6485a27722b95b96f8 Parents: 4c9b67f Author: Li Yang Authored: Thu Sep 22 18:42:37 2016 +0800 Committer: Yang Li Committed: Thu Sep 22 21:04:05 2016 +0800 -- .../kylin/common/restclient/AbstractCache.java | 50 ++ .../common/restclient/AbstractRestCache.java| 52 -- .../kylin/common/restclient/Broadcaster.java| 164 +++ .../restclient/CaseInsensitiveStringCache.java | 5 +- .../kylin/common/restclient/RestClient.java | 4 +- .../common/restclient/SingleValueCache.java | 16 +- .../org/apache/kylin/cube/CubeDescManager.java | 43 - .../java/org/apache/kylin/cube/CubeManager.java | 34 +++- .../apache/kylin/metadata/MetadataManager.java | 88 +- .../kylin/metadata/project/ProjectManager.java | 52 +- .../kylin/storage/hybrid/HybridManager.java | 28 +++- .../engine/streaming/StreamingManager.java | 18 +- .../kylin/rest/controller/CacheController.java | 23 ++- .../apache/kylin/rest/service/CacheService.java | 110 +++-- .../apache/kylin/rest/service/CubeService.java | 30 .../kylin/rest/service/CacheServiceTest.java| 15 +- .../kylin/source/kafka/KafkaConfigManager.java | 18 +- .../storage/hbase/util/CubeMigrationCLI.java| 2 +- 18 files changed, 536 insertions(+), 216 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/478066d0/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java b/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java new file mode 100644 index 000..42692ea --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.restclient; + +import org.apache.kylin.common.KylinConfig; + +/** + */ +public abstract class AbstractCache { + +protected final KylinConfig config; +protected final String syncEntity; + +protected AbstractCache(KylinConfig config, String syncEntity) { +this.config = config; +this.syncEntity = syncEntity; +} + +public Broadcaster getBroadcaster() { +return Broadcaster.getInstance(config); +} + +public abstract void put(K key, V value); + +public abstract void putLocal(K key, V value); + +public abstract void remove(K key); + +public abstract void removeLocal(K key); + +public abstract void clear(); + +public abstract int size(); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/478066d0/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java b/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java deleted file mode 100644 index 584131d..000 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by ap
kylin git commit: half way
Repository: kylin Updated Branches: refs/heads/KYLIN-2033 [created] 8201e8352 half way Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8201e835 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8201e835 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8201e835 Branch: refs/heads/KYLIN-2033 Commit: 8201e83520a17b3ed64e29b5bd10ab77019bb56f Parents: 5a049a5 Author: Li Yang Authored: Thu Sep 22 18:42:37 2016 +0800 Committer: Li Yang Committed: Thu Sep 22 18:42:37 2016 +0800 -- .../kylin/common/restclient/AbstractCache.java | 50 ++ .../common/restclient/AbstractRestCache.java| 52 -- .../kylin/common/restclient/Broadcaster.java| 164 +++ .../restclient/CaseInsensitiveStringCache.java | 5 +- .../kylin/common/restclient/RestClient.java | 4 +- .../common/restclient/SingleValueCache.java | 16 +- .../org/apache/kylin/cube/CubeDescManager.java | 43 - .../java/org/apache/kylin/cube/CubeManager.java | 34 +++- .../apache/kylin/metadata/MetadataManager.java | 88 +- .../kylin/metadata/project/ProjectManager.java | 52 +- .../kylin/storage/hybrid/HybridManager.java | 28 +++- .../engine/streaming/StreamingManager.java | 18 +- .../kylin/rest/controller/CacheController.java | 23 ++- .../apache/kylin/rest/service/CacheService.java | 110 +++-- .../apache/kylin/rest/service/CubeService.java | 30 .../kylin/rest/service/CacheServiceTest.java| 15 +- .../kylin/source/kafka/KafkaConfigManager.java | 18 +- .../storage/hbase/util/CubeMigrationCLI.java| 2 +- 18 files changed, 536 insertions(+), 216 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kylin/blob/8201e835/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java b/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java new file mode 100644 index 000..42692ea --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractCache.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.restclient; + +import org.apache.kylin.common.KylinConfig; + +/** + */ +public abstract class AbstractCache { + +protected final KylinConfig config; +protected final String syncEntity; + +protected AbstractCache(KylinConfig config, String syncEntity) { +this.config = config; +this.syncEntity = syncEntity; +} + +public Broadcaster getBroadcaster() { +return Broadcaster.getInstance(config); +} + +public abstract void put(K key, V value); + +public abstract void putLocal(K key, V value); + +public abstract void remove(K key); + +public abstract void removeLocal(K key); + +public abstract void clear(); + +public abstract int size(); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8201e835/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java -- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java b/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java deleted file mode 100644 index 584131d..000 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *
[1/2] kylin git commit: half way
Repository: kylin Updated Branches: refs/heads/liyang-dimenc [created] 650aea4e5 http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java -- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java new file mode 100644 index 000..40e9f6d --- /dev/null +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.dimension; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +import org.apache.kylin.common.util.Bytes; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FixedLenDimEnc extends DimensionEncoding { + +private static Logger logger = LoggerFactory.getLogger(FixedLenDimEnc.class); + +// row key fixed length place holder +public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9; + +private final int fixedLen; + +transient private int avoidVerbose = 0; + +public FixedLenDimEnc(int len) { +this.fixedLen = len; +} + +@Override +public int getLengthOfEncoding() { +return fixedLen; +} + +@Override +public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) { +if (value == null) { +Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL); +return; +} + +if (valueLen > fixedLen) { +if (avoidVerbose++ % 1 == 0) { +logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueLen + ", will truncate, value string: " + Bytes.toString(value, 0, valueLen) + " times:" + avoidVerbose); +} +} + +int n = Math.min(valueLen, fixedLen); +System.arraycopy(value, 0, output, outputOffset, n); + +if (n < fixedLen) { +Arrays.fill(output, outputOffset + n, outputOffset + fixedLen, ROWKEY_PLACE_HOLDER_BYTE); +} +} + +@Override +public String decode(byte[] bytes, int offset, int len) { +if (isNull(bytes, offset, len)) { +return null; +} + +while (len > 0 && bytes[offset + len - 1] == ROWKEY_PLACE_HOLDER_BYTE) +len--; + +return Bytes.toString(bytes, offset, len); +} + +@Override +public DataTypeSerializer asDataTypeSerializer() { +return new FixedLenSerializer(); +} + +public class FixedLenSerializer extends DataTypeSerializer { +// be thread-safe and avoid repeated obj creation +private ThreadLocal current = new ThreadLocal(); + +private byte[] currentBuf() { +byte[] buf = current.get(); +if (buf == null) { +buf = new byte[fixedLen]; +current.set(buf); +} +return buf; +} + +@Override +public void serialize(Object value, ByteBuffer out) { +byte[] buf = currentBuf(); +byte[] bytes = value == null ? null : Bytes.toBytes(value.toString()); +encode(bytes, bytes == null ? 0 : bytes.length, buf, 0); +out.put(buf); +} + +@Override +public Object deserialize(ByteBuffer in) { +byte[] buf = currentBuf(); +in.get(buf); +return decode(buf, 0, buf.length); +} + +@Override +public int peekLength(ByteBuffer in) { +return fixedLen; +} + +@Override +public int maxLength() { +return fixedLen; +} + +@Override +public int getStorageBytesEstimate() { +return fixedLen; +} + +@Override +public Object valueOf(String str) { +return str; +} +}; + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/650aea4e/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java ---
[2/2] kylin git commit: half way
half way Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/650aea4e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/650aea4e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/650aea4e Branch: refs/heads/liyang-dimenc Commit: 650aea4e5fc33d59a022c6a4d89f1dc049295133 Parents: 4dd1b34 Author: Li, Yang Authored: Fri Feb 26 14:43:59 2016 +0800 Committer: Li, Yang Committed: Fri Feb 26 14:43:59 2016 +0800 -- .../apache/kylin/common/util/Dictionary.java| 232 --- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../java/org/apache/kylin/cube/CubeSegment.java | 2 +- .../kylin/cube/gridtable/CubeCodeSystem.java| 141 ++- .../kylin/cube/gridtable/CubeGridTable.java | 2 +- .../kylin/cube/gridtable/FixLenSerializer.java | 110 - .../cube/gridtable/TrimmedCubeCodeSystem.java | 148 +++- .../inmemcubing/AbstractInMemCubeBuilder.java | 2 +- .../cube/inmemcubing/DoggedCubeBuilder.java | 2 +- .../cube/inmemcubing/InMemCubeBuilder.java | 2 +- .../InMemCubeBuilderInputConverter.java | 3 +- .../kylin/cube/kv/AbstractRowKeyEncoder.java| 4 +- .../org/apache/kylin/cube/kv/RowConstants.java | 2 - .../apache/kylin/cube/kv/RowKeyColumnIO.java| 22 +- .../org/apache/kylin/cube/util/CubingUtils.java | 2 +- .../kylin/gridtable/DefaultGTComparator.java| 2 +- .../kylin/cube/DictionaryManagerTest.java | 2 +- .../DoggedCubeBuilderStressTest.java| 2 +- .../cube/inmemcubing/DoggedCubeBuilderTest.java | 2 +- .../cube/inmemcubing/InMemCubeBuilderTest.java | 2 +- .../kylin/gridtable/DictGridTableTest.java | 2 +- .../apache/kylin/dict/DateStrDictionary.java| 2 +- .../org/apache/kylin/dict/DictCodeSystem.java | 4 +- .../apache/kylin/dict/DictionaryGenerator.java | 2 +- .../org/apache/kylin/dict/DictionaryInfo.java | 2 +- .../kylin/dict/DictionaryInfoSerializer.java| 2 +- .../apache/kylin/dict/DictionaryManager.java| 2 +- .../apache/kylin/dict/DictionarySerializer.java | 2 +- .../org/apache/kylin/dict/IDictionaryAware.java | 2 +- .../dict/MultipleDictionaryValueEnumerator.java | 2 +- .../apache/kylin/dict/TimeStrDictionary.java| 2 +- .../org/apache/kylin/dict/TrieDictionary.java | 2 +- .../dict/TupleFilterDictionaryTranslater.java | 3 +- .../apache/kylin/dict/lookup/SnapshotTable.java | 2 +- .../apache/kylin/dict/NumberDictionaryTest.java | 2 +- .../org/apache/kylin/dimension/Dictionary.java | 231 ++ .../kylin/dimension/DictionaryDimEnc.java | 133 +++ .../kylin/dimension/DimensionEncoding.java | 62 + .../apache/kylin/dimension/FixedLenDimEnc.java | 136 +++ .../apache/kylin/measure/MeasureIngester.java | 2 +- .../org/apache/kylin/measure/MeasureType.java | 2 +- .../kylin/measure/basic/BigDecimalIngester.java | 2 +- .../kylin/measure/basic/DoubleIngester.java | 2 +- .../kylin/measure/basic/LongIngester.java | 2 +- .../kylin/measure/bitmap/BitmapMeasureType.java | 2 +- .../ExtendedColumnMeasureType.java | 2 +- .../kylin/measure/hllc/HLLCMeasureType.java | 2 +- .../kylin/measure/topn/TopNMeasureType.java | 2 +- .../storage/translate/ColumnValueRange.java | 2 +- .../storage/translate/ColumnValueRangeTest.java | 2 +- .../engine/mr/steps/BaseCuboidMapperBase.java | 2 +- .../engine/mr/steps/InMemCuboidMapper.java | 2 +- .../engine/mr/steps/MergeCuboidMapper.java | 2 +- .../engine/mr/steps/MergeCuboidMapperTest.java | 2 +- .../apache/kylin/engine/spark/SparkCubing.java | 2 +- .../streaming/OneOffStreamingBuilder.java | 2 +- .../engine/streaming/StreamingBatchBuilder.java | 2 +- .../streaming/cube/StreamingCubeBuilder.java| 2 +- .../apache/kylin/invertedindex/IISegment.java | 2 +- .../index/CompressedValueContainer.java | 2 +- .../invertedindex/index/RawTableRecord.java | 4 +- .../apache/kylin/invertedindex/index/Slice.java | 2 +- .../kylin/invertedindex/index/SliceBuilder.java | 2 +- .../kylin/invertedindex/index/TableRecord.java | 2 +- .../invertedindex/index/TableRecordInfo.java| 2 +- .../invertedindex/model/IIKeyValueCodec.java| 2 +- .../invertedindex/util/IIDictionaryBuilder.java | 2 +- .../kylin/invertedindex/IIInstanceTest.java | 2 +- .../invertedindex/InvertedIndexLocalTest.java | 2 +- .../hbase/common/coprocessor/AggrKey.java | 4 +- .../common/coprocessor/FilterDecorator.java | 5 +- .../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +- .../hbase/cube/v1/CubeTupleConverter.java | 2 +- .../v1/coprocessor/observer/ObserverTuple.java | 2 +- .../hbase/cube/v2/CubeTupleConverter.java | 2 +- .../en