http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java deleted file mode 100644 index 86b0543..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java +++ /dev/null @@ -1,342 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.metadata.realization.RealizationType; -import org.apache.kylin.metadata.realization.SQLDigest; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * @author honma - */ -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class IIInstance extends RootPersistentEntity implements IRealization { - - public static IIInstance create(String iiName, String projectName, IIDesc iiDesc) { - IIInstance iii = new IIInstance(); - - iii.setConfig(iiDesc.getConfig()); - iii.setName(iiName); - iii.setDescName(iiDesc.getName()); - iii.setCreateTimeUTC(System.currentTimeMillis()); - iii.setStatus(RealizationStatusEnum.DISABLED); - iii.updateRandomUuid(); - - return iii; - } - - @JsonIgnore - private KylinConfig config; - @JsonProperty("name") - private String name; - @JsonProperty("owner") - private String owner; - @JsonProperty("version") - private String version; // user info only, we don't do version control - @JsonProperty("descriptor") - private String descName; - // Mark cube priority for query - @JsonProperty("cost") - private int cost = 50; - @JsonProperty("status") - private RealizationStatusEnum status; - - @JsonManagedReference - @JsonProperty("segments") - private List<IISegment> segments = new ArrayList<IISegment>(); - - @JsonProperty("create_time_utc") - private long createTimeUTC; - - private String projectName; - - public long getAllocatedEndDate() { - if (null == segments || segments.size() == 0) { - return 0; - } - - Collections.sort(segments); - - return segments.get(segments.size() - 1).getDateRangeEnd(); - } - - public long getAllocatedStartDate() { - if (null == segments || segments.size() == 0) { - return 0; - } - - Collections.sort(segments); - - return segments.get(0).getDateRangeStart(); - } - - public IIDesc getDescriptor() { - return IIDescManager.getInstance(config).getIIDesc(descName); - } - - public boolean isReady() { - return getStatus() == RealizationStatusEnum.READY; - } - - public String getResourcePath() { - return concatResourcePath(name); - } - - public static String concatResourcePath(String cubeName) { - return ResourceStore.II_RESOURCE_ROOT + "/" + cubeName + ".json"; - } - - @Override - public String toString() { - return getCanonicalName(); - } - - // ============================================================================ - - @JsonProperty("size_kb") - public long getSizeKB() { - long sizeKb = 0L; - - for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeKb += cubeSegment.getSizeKB(); - } - - return sizeKb; - } - - @JsonProperty("input_records_count") - public long getInputRecordCount() { - long sizeRecordCount = 0L; - - for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeRecordCount += cubeSegment.getInputRecords(); - } - - return sizeRecordCount; - } - - @JsonProperty("input_records_size") - public long getInputRecordSize() { - long sizeRecordSize = 0L; - - for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeRecordSize += cubeSegment.getInputRecordsSize(); - } - - return sizeRecordSize; - } - - public KylinConfig getConfig() { - return config; - } - - public void setConfig(KylinConfig config) { - this.config = config; - } - - @Override - public String getCanonicalName() { - return getType() + "[name=" + name + "]"; - } - - @Override - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getDescName() { - return descName; - } - - public void setDescName(String descName) { - this.descName = descName; - } - - public int getCost() { - return cost; - } - - public RealizationStatusEnum getStatus() { - return status; - } - - public void setStatus(RealizationStatusEnum status) { - this.status = status; - } - - public IISegment getFirstSegment() { - if (this.segments == null || this.segments.size() == 0) { - return null; - } else { - return this.segments.get(0); - } - } - - public List<IISegment> getSegments() { - return segments; - } - - public List<IISegment> getSegments(SegmentStatusEnum status) { - List<IISegment> result = new ArrayList<IISegment>(); - - for (IISegment segment : segments) { - if (segment.getStatus() == status) { - result.add(segment); - } - } - - return result; - } - - public IISegment getSegment(String name, SegmentStatusEnum status) { - for (IISegment segment : segments) { - if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) { - return segment; - } - } - - return null; - } - - public void setSegments(List<IISegment> segments) { - this.segments = segments; - } - - public long getCreateTimeUTC() { - return createTimeUTC; - } - - public void setCreateTimeUTC(long createTimeUTC) { - this.createTimeUTC = createTimeUTC; - } - - @Override - public CapabilityResult isCapable(SQLDigest digest) { - CapabilityResult result = IICapabilityChecker.check(this, digest); - if (result.capable) { - result.cost = getCost(digest); - } else { - result.cost = -1; - } - return result; - } - - public int getCost(SQLDigest digest) { - return 0; - } - - @Override - public RealizationType getType() { - return RealizationType.INVERTED_INDEX; - } - - @Override - public List<TblColRef> getAllColumns() { - return getDescriptor().listAllColumns(); - } - - @Override - public String getFactTable() { - return getDescriptor().getFactTableName(); - } - - @Override - public List<MeasureDesc> getMeasures() { - return getDescriptor().getMeasures(); - } - - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - @Override - public long getDateRangeStart() { - List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); - - long startTime = Long.MAX_VALUE; - for (IISegment seg : readySegs) { - if (seg.getDateRangeStart() < startTime) - startTime = seg.getDateRangeStart(); - } - - return startTime; - } - - @Override - public long getDateRangeEnd() { - - List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); - - long endTime = 0; - for (IISegment seg : readySegs) { - if (seg.getDateRangeEnd() > endTime) - endTime = seg.getDateRangeEnd(); - } - - return endTime; - } - - @Override - public List<TblColRef> getAllDimensions() { - return getDescriptor().listAllDimensions(); - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java deleted file mode 100644 index f7e70f4..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang3.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.restclient.Broadcaster; -import org.apache.kylin.common.restclient.CaseInsensitiveStringCache; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.DictionaryInfo; -import org.apache.kylin.dict.DictionaryManager; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationConstants; -import org.apache.kylin.metadata.realization.IRealizationProvider; -import org.apache.kylin.metadata.realization.RealizationType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author honma - */ -public class IIManager implements IRealizationProvider { - - private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - private static int HBASE_TABLE_LENGTH = 10; - - private static final Serializer<IIInstance> II_SERIALIZER = new JsonSerializer<IIInstance>(IIInstance.class); - - private static final Logger logger = LoggerFactory.getLogger(IIManager.class); - - // static cached instances - private static final ConcurrentHashMap<KylinConfig, IIManager> CACHE = new ConcurrentHashMap<KylinConfig, IIManager>(); - - public static IIManager getInstance(KylinConfig config) { - IIManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (IIManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new IIManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init IIManager from " + config, e); - } - } - } - - public static void clearCache() { - CACHE.clear(); - } - - // ============================================================================ - - private KylinConfig config; - // ii name ==> IIInstance - private CaseInsensitiveStringCache<IIInstance> iiMap = new CaseInsensitiveStringCache<IIInstance>(Broadcaster.TYPE.INVERTED_INDEX); - - // for generation hbase table name of a new segment - private HashSet<String> usedStorageLocation = new HashSet<String>(); - - private IIManager(KylinConfig config) throws IOException { - logger.info("Initializing IIManager with config " + config); - this.config = config; - - loadAllIIInstance(); - } - - public List<IIInstance> listAllIIs() { - return new ArrayList<IIInstance>(iiMap.values()); - } - - public IIInstance getII(String iiName) { - iiName = iiName.toUpperCase(); - return iiMap.get(iiName); - } - - public List<IIInstance> getIIsByDesc(String descName) { - - List<IIInstance> list = listAllIIs(); - List<IIInstance> result = new ArrayList<IIInstance>(); - Iterator<IIInstance> it = list.iterator(); - while (it.hasNext()) { - IIInstance ci = it.next(); - if (descName.equalsIgnoreCase(ci.getDescName())) { - result.add(ci); - } - } - return result; - } - - public void buildInvertedIndexDictionary(IISegment iiSeg, String factColumnsPath) throws IOException { - logger.info("Start building ii dictionary"); - DictionaryManager dictMgr = getDictionaryManager(); - IIDesc iiDesc = iiSeg.getIIInstance().getDescriptor(); - for (TblColRef column : iiDesc.listAllColumns()) { - logger.info("Dealing with column {}", column); - if (iiDesc.isMetricsCol(column)) { - continue; - } - - DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath); - iiSeg.putDictResPath(column, dict.getResourcePath()); - } - saveResource(iiSeg.getIIInstance()); - } - - /** - * return null if no dictionary for given column - */ - public Dictionary<?> getDictionary(IISegment iiSeg, TblColRef col) { - DictionaryInfo info = null; - try { - DictionaryManager dictMgr = getDictionaryManager(); - // logger.info("Using metadata url " + metadataUrl + - // " for DictionaryManager"); - String dictResPath = iiSeg.getDictResPath(col); - if (dictResPath == null) - return null; - - info = dictMgr.getDictionaryInfo(dictResPath); - if (info == null) - throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid II state; II segment" + iiSeg + ", col " + col); - } catch (IOException e) { - throw new IllegalStateException("Failed to get dictionary for II segment" + iiSeg + ", col" + col, e); - } - - return info.getDictionaryObject(); - } - - public IIInstance createII(IIInstance ii) throws IOException { - - if (this.getII(ii.getName()) != null) - throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists."); - - // other logic is the same as update. - return updateII(ii); - } - - public IIInstance updateII(IIInstance ii) throws IOException { - logger.info("Updating II instance '" + ii.getName()); - - // save resource - saveResource(ii); - - logger.info("II with " + ii.getSegments().size() + " segments is saved"); - - return ii; - } - - public void loadIICache(String iiName) { - try { - loadIIInstance(IIInstance.concatResourcePath(iiName)); - } catch (IOException e) { - logger.error(e.getLocalizedMessage(), e); - } - } - - public void removeIICache(IIInstance ii) { - iiMap.remove(ii.getName()); - - for (IISegment segment : ii.getSegments()) { - usedStorageLocation.remove(segment.getName()); - } - } - - public void removeIILocalCache(String name) { - iiMap.removeLocal(name); - //TODO - // for (IISegment segment : ii.getSegments()) { - // usedStorageLocation.remove(segment.getName()); - // } - } - - private void saveResource(IIInstance ii) throws IOException { - ResourceStore store = getStore(); - store.putResource(ii.getResourcePath(), ii, II_SERIALIZER); - this.afterIIUpdated(ii); - } - - private void afterIIUpdated(IIInstance updatedII) { - iiMap.put(updatedII.getName(), updatedII); - } - - /** - * @param IIInstance - * @param startDate (pass 0 if full build) - * @param endDate (pass 0 if full build) - * @return - */ - public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) { - IISegment segment = new IISegment(); - String incrementalSegName = IISegment.getSegmentName(startDate, endDate); - segment.setUuid(UUID.randomUUID().toString()); - segment.setName(incrementalSegName); - segment.setCreateTimeUTC(System.currentTimeMillis()); - segment.setDateRangeStart(startDate); - segment.setDateRangeEnd(endDate); - segment.setStatus(SegmentStatusEnum.NEW); - segment.setStorageLocationIdentifier(generateStorageLocation()); - - segment.setIIInstance(IIInstance); - - return segment; - } - - private String generateStorageLocation() { - String namePrefix = IRealizationConstants.IIHbaseStorageLocationPrefix; - String tableName = ""; - do { - StringBuffer sb = new StringBuffer(); - sb.append(namePrefix); - for (int i = 0; i < HBASE_TABLE_LENGTH; i++) { - int idx = (int) (Math.random() * ALPHA_NUM.length()); - sb.append(ALPHA_NUM.charAt(idx)); - } - tableName = sb.toString(); - } while (this.usedStorageLocation.contains(tableName)); - - return tableName; - } - - private void loadAllIIInstance() throws IOException { - ResourceStore store = getStore(); - List<String> paths = store.collectResourceRecursively(ResourceStore.II_RESOURCE_ROOT, ".json"); - - logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT)); - - for (String path : paths) { - loadIIInstance(path); - } - - logger.debug("Loaded " + paths.size() + " II(s)"); - } - - private synchronized IIInstance loadIIInstance(String path) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading IIInstance " + store.getReadableResourcePath(path)); - - IIInstance IIInstance = null; - try { - IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER); - IIInstance.setConfig(config); - - if (StringUtils.isBlank(IIInstance.getName())) - throw new IllegalStateException("IIInstance name must not be blank"); - - iiMap.putLocal(IIInstance.getName(), IIInstance); - - for (IISegment segment : IIInstance.getSegments()) { - usedStorageLocation.add(segment.getName()); - } - - return IIInstance; - } catch (Exception e) { - logger.error("Error during load ii instance " + path, e); - return null; - } - } - - private DictionaryManager getDictionaryManager() { - return DictionaryManager.getInstance(config); - } - - private ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - @Override - public RealizationType getRealizationType() { - return RealizationType.INVERTED_INDEX; - } - - @Override - public IRealization getRealization(String name) { - return getII(name); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java deleted file mode 100644 index 77a876c..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex; - -import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.ISegment; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonBackReference; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; - -/** - * @author honma - */ - -// TODO: remove segment concept for II, append old hbase table -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class IISegment implements Comparable<IISegment>, ISegment { - - @JsonBackReference - private IIInstance iiInstance; - @JsonProperty("uuid") - private String uuid; - @JsonProperty("name") - private String name; - @JsonProperty("storage_location_identifier") - private String storageLocationIdentifier; // HTable name - @JsonProperty("date_range_start") - private long dateRangeStart; - @JsonProperty("date_range_end") - private long dateRangeEnd; - @JsonProperty("status") - private SegmentStatusEnum status; - @JsonProperty("size_kb") - private long sizeKB; - @JsonProperty("input_records") - private long inputRecords; - @JsonProperty("input_records_size") - private long inputRecordsSize; - @JsonProperty("last_build_time") - private long lastBuildTime; - @JsonProperty("last_build_job_id") - private String lastBuildJobID; - - @JsonProperty("create_time_utc") - private long createTimeUTC; - - @JsonProperty("binary_signature") - private String binarySignature; // a hash of schema and dictionary ID, - // used for sanity check - - @JsonProperty("dictionaries") - private ConcurrentHashMap<String, String> dictionaries; // table/column ==> - // dictionary - // resource path - - private transient TableRecordInfo tableRecordInfo; - - /** - * @param startDate - * @param endDate - * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else - * returns "yyyyMMddHHmmss_yyyyMMddHHmmss" - */ - public static String getSegmentName(long startDate, long endDate) { - if (startDate == 0 && endDate == 0) { - return "FULL_BUILD"; - } - - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - - return dateFormat.format(startDate) + "_" + dateFormat.format(endDate); - } - - public IIDesc getIIDesc() { - return getIIInstance().getDescriptor(); - } - - // ============================================================================ - - public String getUuid() { - return uuid; - } - - public void setUuid(String id) { - this.uuid = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public long getDateRangeStart() { - return dateRangeStart; - } - - public void setDateRangeStart(long dateRangeStart) { - this.dateRangeStart = dateRangeStart; - } - - public long getDateRangeEnd() { - return dateRangeEnd; - } - - public void setDateRangeEnd(long dateRangeEnd) { - this.dateRangeEnd = dateRangeEnd; - } - - public SegmentStatusEnum getStatus() { - return status; - } - - public void setStatus(SegmentStatusEnum status) { - this.status = status; - } - - public long getSizeKB() { - return sizeKB; - } - - public void setSizeKB(long sizeKB) { - this.sizeKB = sizeKB; - } - - public long getInputRecords() { - return inputRecords; - } - - public void setInputRecords(long inputRecords) { - this.inputRecords = inputRecords; - } - - public long getInputRecordsSize() { - return inputRecordsSize; - } - - public void setInputRecordsSize(long inputRecordsSize) { - this.inputRecordsSize = inputRecordsSize; - } - - public long getLastBuildTime() { - return lastBuildTime; - } - - public void setLastBuildTime(long lastBuildTime) { - this.lastBuildTime = lastBuildTime; - } - - public String getLastBuildJobID() { - return lastBuildJobID; - } - - public void setLastBuildJobID(String lastBuildJobID) { - this.lastBuildJobID = lastBuildJobID; - } - - public String getBinarySignature() { - return binarySignature; - } - - public void setBinarySignature(String binarySignature) { - this.binarySignature = binarySignature; - } - - public IIInstance getIIInstance() { - return iiInstance; - } - - public void setIIInstance(IIInstance iiInstance) { - this.iiInstance = iiInstance; - } - - public String getStorageLocationIdentifier() { - return storageLocationIdentifier; - } - - public Map<String, String> getDictionaries() { - if (dictionaries == null) - dictionaries = new ConcurrentHashMap<String, String>(); - return dictionaries; - } - - public Collection<String> getDictionaryPaths() { - return getDictionaries().values(); - } - - public String getDictResPath(TblColRef col) { - return getDictionaries().get(dictKey(col)); - } - - public void putDictResPath(TblColRef col, String dictResPath) { - getDictionaries().put(dictKey(col), dictResPath); - } - - private String dictKey(TblColRef col) { - return col.getTable() + "/" + col.getName(); - } - - /** - * @param storageLocationIdentifier the storageLocationIdentifier to set - */ - public void setStorageLocationIdentifier(String storageLocationIdentifier) { - this.storageLocationIdentifier = storageLocationIdentifier; - } - - @Override - public int compareTo(IISegment other) { - if (this.dateRangeEnd < other.dateRangeEnd) { - return -1; - } else if (this.dateRangeEnd > other.dateRangeEnd) { - return 1; - } else { - return 0; - } - } - - private TableRecordInfo getTableRecordInfo() { - if (tableRecordInfo == null) - tableRecordInfo = new TableRecordInfo(this); - return tableRecordInfo; - } - - public List<TblColRef> getColumns() { - return this.getTableRecordInfo().getColumns(); - } - - @Override - public String toString() { - return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString(); - } - - @Override - public int getColumnLength(TblColRef col) { - - int index = getTableRecordInfo().findColumn(col); - return getTableRecordInfo().getDigest().length(index); - } - - @Override - public Dictionary<String> getDictionary(TblColRef col) { - - int index = getTableRecordInfo().findColumn(col); - return getTableRecordInfo().dict(index); - } - - public long getCreateTimeUTC() { - return createTimeUTC; - } - - public void setCreateTimeUTC(long createTimeUTC) { - this.createTimeUTC = createTimeUTC; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java deleted file mode 100644 index 164e2b9..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; -import org.roaringbitmap.RoaringBitmap; - -/** - * @author yangli9 - */ -public class BitMapContainer implements ColumnValueContainer { - - int valueLen; - int nValues; - int size; - RoaringBitmap[] sets; - boolean closedForChange; - - transient byte[] temp; - - public BitMapContainer(TableRecordInfoDigest digest, int col) { - this.valueLen = digest.length(col); - this.size = 0; - this.nValues = digest.getMaxID(col) + 1; - this.sets = null; - this.closedForChange = false; - - this.temp = new byte[valueLen]; - } - - @Override - public void append(ImmutableBytesWritable valueBytes) { - int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen); - append(value); - } - - public void append(int value) { - checkUpdateMode(); - if (value == Dictionary.NULL_ID[valueLen]) { - value = nValues; // set[nValues] holds NULL - } - sets[value].add(size); - size++; - } - - @Override - public void getValueAt(int i, ImmutableBytesWritable valueBytes) { - int value = getValueIntAt(i); - BytesUtil.writeUnsigned(value, temp, 0, valueLen); - valueBytes.set(temp, 0, valueLen); - } - - @Override - public RoaringBitmap getBitMap(Integer startId, Integer endId) { - if (startId == null && endId == null) { - return sets[this.nValues]; - } - - int start = 0; - int end = this.nValues - 1; - if (startId != null) { - start = startId; - } - if (endId != null) { - end = endId; - } - - return RoaringBitmap.or(Arrays.copyOfRange(sets, start, end + 1)); - } - - @SuppressWarnings("unused") - private RoaringBitmap getBitMap(int valueId) { - if (valueId >= 0 && valueId <= getMaxValueId()) - return sets[valueId]; - else - return sets[this.nValues]; - } - - @Override - public int getMaxValueId() { - return this.nValues - 1; - } - - public int getValueIntAt(int i) { - for (int v = 0; v < nValues; v++) { - if (sets[v].contains(i)) { - return v; - } - } - // if v is not in [0..nValues-1], then it must be nValue (NULL) - return Dictionary.NULL_ID[valueLen]; - } - - private void checkUpdateMode() { - if (isClosedForChange()) { - throw new IllegalStateException(); - } - if (sets == null) { - sets = new RoaringBitmap[nValues + 1]; - for (int i = 0; i <= nValues; i++) { - sets[i] = new RoaringBitmap(); - } - } - } - - private boolean isClosedForChange() { - return closedForChange; - } - - @Override - public void closeForChange() { - closedForChange = true; - } - - @Override - public int getSize() { - return size; - } - - public List<ImmutableBytesWritable> toBytes() { - if (isClosedForChange() == false) - closeForChange(); - - List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1); - for (int i = 0; i <= nValues; i++) { - r.add(setToBytes(sets[i])); - } - return r; - } - - public void fromBytes(List<ImmutableBytesWritable> bytes) { - assert nValues + 1 == bytes.size(); - sets = new RoaringBitmap[nValues + 1]; - size = 0; - for (int i = 0; i <= nValues; i++) { - sets[i] = bytesToSet(bytes.get(i)); - size += sets[i].getCardinality(); - } - closedForChange = true; - } - - private ImmutableBytesWritable setToBytes(RoaringBitmap set) { - // Serializing a bitmap to a byte array can be expected to be expensive, this should not be commonly done. - // If the purpose is to save the data to disk or to a network, then a direct serialization would be - // far more efficient. If the purpose is to enforce immutability, it is an expensive way to do it. - set.runOptimize(); //to improve compression - final byte[] array = new byte[set.serializedSizeInBytes()]; - try { - set.serialize(new java.io.DataOutputStream(new java.io.OutputStream() { - int c = 0; - - @Override - public void close() { - } - - @Override - public void flush() { - } - - @Override - public void write(int b) { - array[c++] = (byte) b; - } - - @Override - public void write(byte[] b) { - write(b, 0, b.length); - } - - @Override - public void write(byte[] b, int off, int l) { - System.arraycopy(b, off, array, c, l); - c += l; - } - })); - } catch (IOException ioe) { - // should never happen because we write to a byte array - throw new RuntimeException("unexpected error while serializing to a byte array"); - } - - return new ImmutableBytesWritable(array); - } - - private RoaringBitmap bytesToSet(final ImmutableBytesWritable bytes) { - // converting a byte array to a bitmap can be expected to be expensive, hopefully this is not a common operation! - RoaringBitmap set = new RoaringBitmap(); - if ((bytes.get() != null) && (bytes.getLength() > 0)) { - // here we could use an ImmutableRoaringBitmap and just "map" it. - // instead, we do a full deserialization - // Note: we deserializing a Roaring bitmap, there is no need to know the length, the format is self-describing - try { - set.deserialize(new java.io.DataInputStream(new java.io.InputStream() { - byte[] array = bytes.get(); - int c = bytes.getOffset(); - - @Override - public int read() { - return array[c++] & 0xff; - } - - @Override - public int read(byte b[]) { - return read(b, 0, b.length); - } - - @Override - public int read(byte[] b, int off, int l) { - System.arraycopy(array, c, b, off, l); - c += l; - return l; - } - })); - } catch (IOException ioe) { - // should never happen because we read from a byte array - throw new RuntimeException("unexpected error while deserializing from a byte array"); - } - } - return set; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (closedForChange ? 1231 : 1237); - result = prime * result + nValues; - result = prime * result + Arrays.hashCode(sets); - result = prime * result + size; - result = prime * result + valueLen; - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - BitMapContainer other = (BitMapContainer) obj; - if (closedForChange != other.closedForChange) - return false; - if (nValues != other.nValues) - return false; - if (!Arrays.equals(sets, other.sets)) - return false; - if (size != other.size) - return false; - if (valueLen != other.valueLen) - return false; - return true; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java deleted file mode 100644 index a5ce1bd..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; - -import org.roaringbitmap.RoaringBitmap; - -/** - * @author yangli9 - */ -public interface ColumnValueContainer { - - void append(ImmutableBytesWritable valueBytes); - - void closeForChange(); - - int getSize(); - - // works only after closeForChange() - void getValueAt(int i, ImmutableBytesWritable valueBytes); - - RoaringBitmap getBitMap(Integer startId, Integer endId); - - int getMaxValueId(); - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java deleted file mode 100644 index 334457c..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; - -import com.ning.compress.lzf.LZFDecoder; -import com.ning.compress.lzf.LZFEncoder; -import org.roaringbitmap.RoaringBitmap; - -/** - * @author yangli9 - */ -public class CompressedValueContainer implements ColumnValueContainer { - int valueLen; - int nValues; - int cap; - int size; - byte[] uncompressed; - byte[] compressed; - - public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) { - this.valueLen = digest.length(col); - this.nValues = digest.getMaxID(col) + 1; - this.cap = cap; - this.size = 0; - this.uncompressed = null; - this.compressed = null; - } - - @Override - public void append(ImmutableBytesWritable valueBytes) { - checkUpdateMode(); - System.arraycopy(valueBytes.get(), valueBytes.getOffset(), uncompressed, valueLen * size, valueLen); - size++; - } - - @Override - public void getValueAt(int i, ImmutableBytesWritable valueBytes) { - valueBytes.set(uncompressed, valueLen * i, valueLen); - } - - @Override - public RoaringBitmap getBitMap(Integer startId, Integer endId) { - RoaringBitmap ret = new RoaringBitmap(); - int nullId = Dictionary.NULL_ID[valueLen]; - - if (startId == null && endId == null) { - //entry for getting null values - for (int i = 0; i < size; ++i) { - int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen); - if (nullId == valueID) { - ret.add(i); - } - } - return ret; - } - - //normal values - for (int i = 0; i < size; ++i) { - int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen); - if (valueID == nullId) { - continue; - } - - if (startId != null && valueID < startId) { - continue; - } - - if (endId != null && valueID > endId) { - continue; - } - - ret.add(i); - } - return ret; - - } - - @Override - public int getMaxValueId() { - return nValues - 1; - } - - private void checkUpdateMode() { - if (isClosedForChange()) { - throw new IllegalArgumentException(); - } - if (uncompressed == null) { - uncompressed = new byte[valueLen * cap]; - } - } - - private boolean isClosedForChange() { - return compressed != null; - } - - @Override - public void closeForChange() { - checkUpdateMode(); - try { - compressed = LZFEncoder.encode(uncompressed, 0, valueLen * size); - } catch (Exception e) { - throw new RuntimeException("LZF encode failure", e); - } - } - - @Override - public int getSize() { - return size; - } - - public ImmutableBytesWritable toBytes() { - if (isClosedForChange() == false) - closeForChange(); - return new ImmutableBytesWritable(compressed); - } - - public void fromBytes(ImmutableBytesWritable bytes) { - try { - uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength()); - } catch (IOException e) { - throw new RuntimeException("LZF decode failure", e); - } - size = cap = uncompressed.length / valueLen; - compressed = BytesUtil.EMPTY_BYTE_ARRAY; // mark closed - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + size; - result = prime * result + valueLen; - result = prime * result + Arrays.hashCode(uncompressed); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - CompressedValueContainer other = (CompressedValueContainer) obj; - if (size != other.size) - return false; - if (valueLen != other.valueLen) - return false; - if (!Bytes.equals(uncompressed, 0, size * valueLen, uncompressed, 0, size * valueLen)) - return false; - return true; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java deleted file mode 100644 index b9f963e..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.util.Arrays; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec; - -/** - * Created by honma on 11/10/14. - */ -public class RawTableRecord implements Cloneable { - TableRecordInfoDigest digest; - private byte[] buf; // consecutive column value IDs (encoded by dictionary) - - public RawTableRecord(TableRecordInfoDigest info) { - this.digest = info; - this.buf = new byte[info.getByteFormLen()]; - reset(); - } - - public RawTableRecord(RawTableRecord another) { - this.digest = another.digest; - this.buf = Bytes.copy(another.buf); - } - - public void reset() { - Arrays.fill(buf, Dictionary.NULL); - } - - public boolean isMetric(int col) { - return digest.isMetrics(col); - } - - public FixedLenMeasureCodec<LongWritable> codec(int col) { - return digest.codec(col); - } - - public int length(int col) { - return digest.length(col); - } - - public int getColumnCount() { - return digest.getColumnCount(); - } - - public void setValueID(int col, int id) { - BytesUtil.writeUnsigned(id, buf, digest.offset(col), digest.length(col)); - } - - public int getValueID(int col) { - return BytesUtil.readUnsigned(buf, digest.offset(col), digest.length(col)); - } - - public void setValueMetrics(int col, LongWritable value) { - digest.codec(col).write(value, buf, digest.offset(col)); - } - - public String getValueMetric(int col) { - digest.codec(col).read(buf, digest.offset(col)); - return (String) digest.codec(col).getValue(); - } - - public byte[] getBytes() { - return buf; - } - - //TODO is it possible to avoid copying? - public void setBytes(byte[] bytes, int offset, int length) { - assert buf.length == length; - System.arraycopy(bytes, offset, buf, 0, length); - } - - public void setValueBytes(int col, ImmutableBytesWritable bytes) { - System.arraycopy(bytes.get(), bytes.getOffset(), buf, digest.offset(col), digest.length(col)); - } - - public void getValueBytes(int col, ImmutableBytesWritable bytes) { - bytes.set(buf, digest.offset(col), digest.length(col)); - } - - @Override - public Object clone() { - return new RawTableRecord(this); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(buf); - // result = prime * result + ((digest == null) ? 0 : digest.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - RawTableRecord other = (RawTableRecord) obj; - if (!Arrays.equals(buf, other.buf)) - return false; - return true; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java deleted file mode 100644 index 0fd1bf9..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; - -public class ShardingHash { - - static HashFunction hashFunc = Hashing.murmur3_128(); - - public static long hashInt(int integer) { - return hashFunc.newHasher().putInt(integer).hash().asLong(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java deleted file mode 100644 index 3443fc6..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.util.List; - -import com.google.common.collect.Lists; - -public class ShardingSliceBuilder { - - SliceBuilder[] builders; - - public ShardingSliceBuilder(TableRecordInfo info) { - int sharding = info.getDescriptor().getSharding(); - builders = new SliceBuilder[sharding]; - for (short i = 0; i < sharding; i++) { - builders[i] = new SliceBuilder(info, i); - } - } - - // NOTE: record must be appended in time order - public Slice append(TableRecord rec) { - short shard = rec.getShard(); - return builders[shard].append(rec); - } - - public List<Slice> close() { - List<Slice> result = Lists.newArrayList(); - for (SliceBuilder builder : builders) { - Slice slice = builder.close(); - if (slice != null) - result.add(slice); - } - return result; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java deleted file mode 100644 index 2a53864..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.util.Iterator; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; - -import org.roaringbitmap.RoaringBitmap; -import org.roaringbitmap.IntIterator; - -/** - * Within a partition (per timestampGranularity), records are further sliced - * (per sliceLength) to fit into HBASE cell. - * - * @author yangli9 - */ -public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> { - - TableRecordInfoDigest info; - int nColumns; - - short shard; - long timestamp; - int nRecords; - ColumnValueContainer[] containers; - - public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) { - this.info = digest; - this.nColumns = digest.getColumnCount(); - - this.shard = shard; - this.timestamp = timestamp; - this.nRecords = containers[0].getSize(); - this.containers = containers; - - assert nColumns == containers.length; - for (int i = 0; i < nColumns; i++) { - assert nRecords == containers[i].getSize(); - } - } - - public int getRecordCount() { - return this.nRecords; - } - - public short getShard() { - return shard; - } - - public long getTimestamp() { - return timestamp; - } - - public ColumnValueContainer[] getColumnValueContainers() { - return containers; - } - - public ColumnValueContainer getColumnValueContainer(int col) { - return containers[col]; - } - - public Iterator<RawTableRecord> iterateWithBitmap(final RoaringBitmap resultBitMap) { - if (resultBitMap == null) { - return this.iterator(); - } else { - final RawTableRecord rec = info.createTableRecordBytes(); - final ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - return new Iterator<RawTableRecord>() { - IntIterator iter = resultBitMap.getIntIterator(); - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public RawTableRecord next() { - int i = iter.next(); - for (int col = 0; col < nColumns; col++) { - containers[col].getValueAt(i, temp); - rec.setValueBytes(col, temp); - } - return rec; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - }; - } - } - - @Override - public Iterator<RawTableRecord> iterator() { - return new Iterator<RawTableRecord>() { - int i = 0; - RawTableRecord rec = info.createTableRecordBytes(); - ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - @Override - public boolean hasNext() { - return i < nRecords; - } - - @Override - public RawTableRecord next() { - for (int col = 0; col < nColumns; col++) { - containers[col].getValueAt(i, temp); - rec.setValueBytes(col, temp); - } - i++; - return rec; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - }; - } - - /* - * (non-Javadoc) - * - * @see java.lang.Object#hashCode() - */ - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((info == null) ? 0 : info.hashCode()); - result = prime * result + shard; - result = prime * result + (int) (timestamp ^ (timestamp >>> 32)); - return result; - } - - /* - * (non-Javadoc) - * - * @see java.lang.Object#equals(java.lang.Object) - */ - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - Slice other = (Slice) obj; - if (info == null) { - if (other.info != null) - return false; - } else if (!info.equals(other.info)) - return false; - if (shard != other.shard) - return false; - if (timestamp != other.timestamp) - return false; - return true; - } - - @Override - public int compareTo(Slice o) { - int comp = this.shard - o.shard; - if (comp != 0) - return comp; - - comp = (int) (this.timestamp - o.timestamp); - return comp; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java deleted file mode 100644 index d794708..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; - -/** - * @author yangli9 - */ -public class SliceBuilder { - - TableRecordInfo info; - private int nColumns; - int nRecordsCap; - - short shard; - long sliceTimestamp; - int nRecords; - private ColumnValueContainer[] containers; - - transient ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - public SliceBuilder(TableRecordInfo info, short shard) { - this.info = info; - this.nColumns = info.getDigest().getColumnCount(); - this.nRecordsCap = Math.max(1, info.getDescriptor().getSliceSize()); - - this.shard = shard; - this.sliceTimestamp = Long.MIN_VALUE; - this.nRecords = 0; - this.containers = null; - - doneSlice(); // init containers - } - - private Slice doneSlice() { - Slice r = null; - if (nRecords > 0) { - for (int i = 0; i < nColumns; i++) { - containers[i].closeForChange(); - } - r = new Slice(info.getDigest(), shard, sliceTimestamp, containers); - } - - // reset for next slice - nRecords = 0; - containers = new ColumnValueContainer[nColumns]; - for (int i : info.getDescriptor().getValueColumns()) { - containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); - } - for (int i : info.getDescriptor().getMetricsColumns()) { - containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); - } - - return r; - - } - - // NOTE: record must be appended in time order - public Slice append(TableRecord rec) { - if (rec.getShard() != shard) - throw new IllegalStateException(); - - Slice doneSlice = null; - - if (isFull()) { - doneSlice = doneSlice(); - } - - if (nRecords == 0) { - sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp()); - } - - nRecords++; - for (int i = 0; i < nColumns; i++) { - rec.getValueBytes(i, temp); - containers[i].append(temp); - } - - return doneSlice; - } - - private long increaseSliceTimestamp(long timestamp) { - if (timestamp < sliceTimestamp) - throw new IllegalStateException(); - - if (timestamp == sliceTimestamp) - return ++timestamp; // ensure slice timestamp increases - else - return timestamp; - } - - public Slice close() { - Slice doneSlice = doneSlice(); - this.sliceTimestamp = Long.MIN_VALUE; - this.nRecords = 0; - return doneSlice; - } - - private boolean isFull() { - return nRecords >= nRecordsCap; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java deleted file mode 100644 index 09a9244..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.util.List; - -import org.apache.commons.lang.ObjectUtils; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.kylin.common.util.DateFormat; - -import com.google.common.collect.Lists; - -/** - * @author yangli9, honma - * <p/> - * TableRecord extends RawTableRecord by decorating it with a - * TableRecordInfo - */ -public class TableRecord implements Cloneable { - - private TableRecordInfo info; - private RawTableRecord rawRecord; - - public TableRecord(RawTableRecord rawRecord, TableRecordInfo info) { - this.info = info; - this.rawRecord = rawRecord; - } - - public TableRecord(TableRecord another) { - this.info = another.info; - this.rawRecord = (RawTableRecord) another.rawRecord.clone(); - } - - @Override - public Object clone() { - return new TableRecord(this); - } - - public void reset() { - rawRecord.reset(); - } - - public byte[] getBytes() { - return rawRecord.getBytes(); - } - - public void setBytes(byte[] bytes, int offset, int length) { - rawRecord.setBytes(bytes, offset, length); - } - - public long getTimestamp() { - String str = getValueString(info.getTimestampColumn()); - return DateFormat.stringToMillis(str); - } - - public int length(int col) { - return rawRecord.length(col); - } - - public List<String> getOriginTableColumnValues() { - List<String> ret = Lists.newArrayList(); - for (int i = 0; i < info.nColumns; ++i) { - ret.add(getValueString(i)); - } - return ret; - } - - public void setValueString(int col, String value) { - if (rawRecord.isMetric(col)) { - LongWritable v = rawRecord.codec(col).valueOf(value); - setValueMetrics(col, v); - } else { - int id = info.dict(col).getIdFromValue(value); - rawRecord.setValueID(col, id); - } - } - - /** - * get value of columns which belongs to the original table columns. - * i.e. columns like min_xx, max_yy will never appear - */ - public String getValueString(int col) { - if (rawRecord.isMetric(col)) - return getValueMetric(col); - else - return info.dict(col).getValueFromId(rawRecord.getValueID(col)); - } - - public void getValueBytes(int col, ImmutableBytesWritable bytes) { - rawRecord.getValueBytes(col, bytes); - } - - private void setValueMetrics(int col, LongWritable value) { - rawRecord.setValueMetrics(col, value); - } - - private String getValueMetric(int col) { - return rawRecord.getValueMetric(col); - } - - public short getShard() { - int timestampID = rawRecord.getValueID(info.getTimestampColumn()); - return (short) (Math.abs(ShardingHash.hashInt(timestampID)) % info.getDescriptor().getSharding()); - } - - @Override - public String toString() { - StringBuilder buf = new StringBuilder("["); - for (int col = 0; col < rawRecord.getColumnCount(); col++) { - if (col > 0) - buf.append(","); - buf.append(getValueString(col)); - } - buf.append("]"); - return buf.toString(); - } - - @Override - public int hashCode() { - if (rawRecord != null) { - return rawRecord.hashCode(); - } else { - return 0; - } - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - TableRecord other = (TableRecord) obj; - return ObjectUtils.equals(other.rawRecord, this.rawRecord); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java deleted file mode 100644 index c41a70c..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.io.IOException; -import java.util.List; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.DictionaryManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.TblColRef; - -/** - * @author yangli9 - * <p/> - * TableRecordInfo stores application-aware knowledges, while - * TableRecordInfoDigest only stores byte level knowleges - */ -public class TableRecordInfo { - - final IISegment seg; - final IIDesc desc; - final int nColumns; - final List<TblColRef> allColumns; - - final FixedLenMeasureCodec<?>[] measureSerializers; - final Dictionary<?>[] dictionaries; - - final TableRecordInfoDigest digest; - - public TableRecordInfo(IISegment iiSegment) { - - seg = iiSegment; - desc = seg.getIIInstance().getDescriptor(); - allColumns = desc.listAllColumns(); - nColumns = allColumns.size(); - dictionaries = new Dictionary<?>[nColumns]; - measureSerializers = new FixedLenMeasureCodec<?>[nColumns]; - - DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig()); - int index = 0; - for (TblColRef tblColRef : desc.listAllColumns()) { - ColumnDesc col = tblColRef.getColumn(); - if (desc.isMetricsCol(index)) { - measureSerializers[index] = FixedLenMeasureCodec.get(col.getType()); - } else { - String dictPath = seg.getDictResPath(tblColRef); - try { - dictionaries[index] = dictMgr.getDictionary(dictPath); - } catch (IOException e) { - throw new RuntimeException("dictionary " + dictPath + " does not exist ", e); - } - } - index++; - } - - digest = createDigest(); - } - - public TableRecordInfoDigest getDigest() { - return digest; - } - - private TableRecordInfoDigest createDigest() { - // isMetric - boolean[] isMetric = new boolean[nColumns]; - for (int i = 0; i < nColumns; ++i) { - isMetric[i] = desc.isMetricsCol(i); - } - - // lengths - int[] lengths = new int[nColumns]; - for (int i = 0; i < nColumns; ++i) { - lengths[i] = isMetric[i] ? measureSerializers[i].getLength() : dictionaries[i].getSizeOfId(); - } - - // dict max id - int[] dictMaxIds = new int[nColumns]; - for (int i = 0; i < nColumns; ++i) { - if (!isMetric[i]) - dictMaxIds[i] = dictionaries[i].getMaxId(); - } - - // offsets - int pos = 0; - int[] offsets = new int[nColumns]; - for (int i = 0; i < nColumns; i++) { - offsets[i] = pos; - pos += lengths[i]; - } - - int byteFormLen = pos; - - return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, measureSerializers); - } - - public TableRecord createTableRecord() { - return new TableRecord(digest.createTableRecordBytes(), this); - } - - public IIDesc getDescriptor() { - return desc; - } - - public List<TblColRef> getColumns() { - return allColumns; - } - - public int findColumn(TblColRef col) { - return desc.findColumn(col); - } - - public int findFactTableColumn(String columnName) { - if (columnName == null) - return -1; - for (int i = 0; i < allColumns.size(); ++i) { - TblColRef tblColRef = allColumns.get(i); - if (tblColRef.isSameAs(desc.getFactTableName(), columnName)) { - return i; - } - } - return -1; - } - - // dimensions go with dictionary - @SuppressWarnings("unchecked") - public Dictionary<String> dict(int col) { - // yes, all dictionaries are string based - return (Dictionary<String>) dictionaries[col]; - } - - public int getTimestampColumn() { - return desc.getTimestampColumn(); - } - - /* - * (non-Javadoc) - * - * @see java.lang.Object#hashCode() - */ - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((seg == null) ? 0 : seg.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - TableRecordInfo other = (TableRecordInfo) obj; - if (seg == null) { - if (other.seg != null) - return false; - } else if (!seg.equals(other.seg)) - return false; - return true; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java deleted file mode 100644 index 0ed58b0..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.nio.ByteBuffer; - -import org.apache.hadoop.io.LongWritable; -import org.apache.kylin.common.util.BytesSerializer; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec; -import org.apache.kylin.metadata.datatype.DataType; - -/** - * Created by honma on 11/10/14. - */ -public class TableRecordInfoDigest { - - private int nColumns; - private int byteFormLen; - - private int[] offsets;// column offset in byte form row - private int[] dictMaxIds;// max id for each of the dict - private int[] lengths;// length of each encoded dict - private boolean[] isMetric;// whether it's metric or dict - - protected FixedLenMeasureCodec<?>[] measureSerializers; - - public TableRecordInfoDigest(int nColumns, int byteFormLen, int[] offsets, int[] dictMaxIds, int[] lengths, boolean[] isMetric, FixedLenMeasureCodec<?>[] measureSerializers) { - this.nColumns = nColumns; - this.byteFormLen = byteFormLen; - this.offsets = offsets; - this.dictMaxIds = dictMaxIds; - this.lengths = lengths; - this.isMetric = isMetric; - this.measureSerializers = measureSerializers; - } - - public TableRecordInfoDigest() { - } - - public int getByteFormLen() { - return byteFormLen; - } - - public boolean isMetrics(int col) { - return isMetric[col]; - } - - public int getColumnCount() { - return nColumns; - } - - public int offset(int col) { - return offsets[col]; - } - - public int length(int col) { - return lengths[col]; - } - - public int getMaxID(int col) { - return dictMaxIds[col]; - } - - public int getMetricCount() { - int ret = 0; - for (int i = 0; i < nColumns; ++i) { - if (isMetrics(i)) { - ret++; - } - } - return ret; - } - - public RawTableRecord createTableRecordBytes() { - return new RawTableRecord(this); - } - - // metrics go with fixed-len codec - @SuppressWarnings("unchecked") - public FixedLenMeasureCodec<LongWritable> codec(int col) { - // yes, all metrics are long currently - return (FixedLenMeasureCodec<LongWritable>) measureSerializers[col]; - } - - public static byte[] serialize(TableRecordInfoDigest o) { - ByteBuffer buf = ByteBuffer.allocate(Serializer.SERIALIZE_BUFFER_SIZE); - serializer.serialize(o, buf); - byte[] result = new byte[buf.position()]; - System.arraycopy(buf.array(), 0, result, 0, buf.position()); - return result; - } - - public static TableRecordInfoDigest deserialize(byte[] bytes) { - return serializer.deserialize(ByteBuffer.wrap(bytes)); - } - - public static TableRecordInfoDigest deserialize(ByteBuffer buffer) { - return serializer.deserialize(buffer); - } - - private static final Serializer serializer = new Serializer(); - - private static class Serializer implements BytesSerializer<TableRecordInfoDigest> { - - @Override - public void serialize(TableRecordInfoDigest value, ByteBuffer out) { - BytesUtil.writeVInt(value.nColumns, out); - BytesUtil.writeVInt(value.byteFormLen, out); - BytesUtil.writeIntArray(value.offsets, out); - BytesUtil.writeIntArray(value.dictMaxIds, out); - BytesUtil.writeIntArray(value.lengths, out); - BytesUtil.writeBooleanArray(value.isMetric, out); - - for (int i = 0; i < value.measureSerializers.length; ++i) { - if (value.isMetrics(i)) { - BytesUtil.writeAsciiString(value.measureSerializers[i].getDataType().toString(), out); - } else { - BytesUtil.writeAsciiString(null, out); - } - } - } - - @Override - public TableRecordInfoDigest deserialize(ByteBuffer in) { - TableRecordInfoDigest result = new TableRecordInfoDigest(); - result.nColumns = BytesUtil.readVInt(in); - result.byteFormLen = BytesUtil.readVInt(in); - result.offsets = BytesUtil.readIntArray(in); - result.dictMaxIds = BytesUtil.readIntArray(in); - result.lengths = BytesUtil.readIntArray(in); - result.isMetric = BytesUtil.readBooleanArray(in); - - result.measureSerializers = new FixedLenMeasureCodec<?>[result.nColumns]; - for (int i = 0; i < result.nColumns; ++i) { - String typeStr = BytesUtil.readAsciiString(in); - if (typeStr == null) { - result.measureSerializers[i] = null; - } else { - result.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getInstance(typeStr)); - } - } - - return result; - } - - } -}
