http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java deleted file mode 100644 index fa7e51b..0000000 --- a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java +++ /dev/null @@ -1,646 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.dataGen; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.util.Array; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.DimensionDesc; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.model.JoinDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.TblColRef; - -/** - * Created by hongbin on 5/20/14. - */ -public class FactTableGenerator { - CubeInstance cube = null; - CubeDesc desc = null; - ResourceStore store = null; - String factTableName = null; - - GenConfig genConf = null; - - Random r = null; - - String cubeName; - long randomSeed; - int rowCount; - int unlinkableRowCount; - int unlinkableRowCountMax; - double conflictRatio; - double linkableRatio; - - // the names of lookup table columns which is in relation with fact - // table(appear as fk in fact table) - TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - // possible values of lookupTableKeys, extracted from existing lookup - // tables. - // The key is in the format of tablename/columnname - TreeMap<String, ArrayList<String>> feasibleValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - // lookup table name -> sets of all composite keys - TreeMap<String, HashSet<Array<String>>> lookupTableCompositeKeyValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - private void init(String cubeName, int rowCount, double conflictRaio, double linkableRatio, long randomSeed) { - this.rowCount = rowCount; - this.conflictRatio = conflictRaio; - this.cubeName = cubeName; - this.randomSeed = randomSeed; - this.linkableRatio = linkableRatio; - - this.unlinkableRowCountMax = (int) (this.rowCount * (1 - linkableRatio)); - this.unlinkableRowCount = 0; - - r = new Random(randomSeed); - - KylinConfig config = KylinConfig.getInstanceFromEnv(); - cube = CubeManager.getInstance(config).getCube(cubeName); - desc = cube.getDescriptor(); - factTableName = desc.getFactTable(); - store = ResourceStore.getStore(config); - } - - /* - * users can specify the value preference for each column - */ - private void loadConfig() { - try { - InputStream configStream = store.getResource("/data/data_gen_config.json").inputStream; - this.genConf = GenConfig.loadConfig(configStream); - - if (configStream != null) - configStream.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - private void loadLookupTableValues(String lookupTableName, LinkedList<String> columnNames, int distinctRowCount) throws Exception { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - - // only deal with composite keys - if (columnNames.size() > 1 && !lookupTableCompositeKeyValues.containsKey(lookupTableName)) { - lookupTableCompositeKeyValues.put(lookupTableName, new HashSet<Array<String>>()); - } - - InputStream tableStream = null; - BufferedReader tableReader = null; - try { - TreeMap<String, Integer> zeroBasedInice = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - for (String columnName : columnNames) { - ColumnDesc cDesc = MetadataManager.getInstance(config).getTableDesc(lookupTableName).findColumnByName(columnName); - zeroBasedInice.put(columnName, cDesc.getZeroBasedIndex()); - } - - String path = "/data/" + lookupTableName + ".csv"; - tableStream = store.getResource(path).inputStream; - tableReader = new BufferedReader(new InputStreamReader(tableStream)); - tableReader.mark(0); - int rowCount = 0; - int curRowNum = 0; - String curRow; - - while (tableReader.readLine() != null) - rowCount++; - - HashSet<Integer> rows = new HashSet<Integer>(); - distinctRowCount = (distinctRowCount < rowCount) ? distinctRowCount : rowCount; - while (rows.size() < distinctRowCount) { - rows.add(r.nextInt(rowCount)); - } - - // reopen the stream - tableReader.close(); - tableStream.close(); - tableStream = null; - tableReader = null; - - tableStream = store.getResource(path).inputStream; - tableReader = new BufferedReader(new InputStreamReader(tableStream)); - - while ((curRow = tableReader.readLine()) != null) { - if (rows.contains(curRowNum)) { - String[] tokens = curRow.split(","); - - String[] comboKeys = null; - int index = 0; - if (columnNames.size() > 1) - comboKeys = new String[columnNames.size()]; - - for (String columnName : columnNames) { - int zeroBasedIndex = zeroBasedInice.get(columnName); - if (!feasibleValues.containsKey(lookupTableName + "/" + columnName)) - feasibleValues.put(lookupTableName + "/" + columnName, new ArrayList<String>()); - feasibleValues.get(lookupTableName + "/" + columnName).add(tokens[zeroBasedIndex]); - - if (columnNames.size() > 1) { - comboKeys[index] = tokens[zeroBasedIndex]; - index++; - } - } - - if (columnNames.size() > 1) { - Array<String> wrap = new Array<String>(comboKeys); - if (lookupTableCompositeKeyValues.get(lookupTableName).contains(wrap)) { - throw new Exception("The composite key already exist in the lookup table"); - } - lookupTableCompositeKeyValues.get(lookupTableName).add(wrap); - } - } - curRowNum++; - } - - if (tableStream != null) - tableStream.close(); - if (tableReader != null) - tableReader.close(); - - } catch (IOException e) { - e.printStackTrace(); - System.exit(1); - } - } - - // prepare the candidate values for each joined column - private void prepare() throws Exception { - // load config - loadConfig(); - - TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - - for (DimensionDesc dim : desc.getDimensions()) { - for (TblColRef col : dim.getColumnRefs()) { - if (col.getTable().equals(factTableName)) - factTableColumns.add(col.getName()); - } - - JoinDesc join = dim.getJoin(); - if (join != null) { - String lookupTable = dim.getTable(); - for (String column : join.getPrimaryKey()) { - if (!lookupTableKeys.containsKey(lookupTable)) { - lookupTableKeys.put(lookupTable, new LinkedList<String>()); - } - - if (!lookupTableKeys.get(lookupTable).contains(column)) - lookupTableKeys.get(lookupTable).add(column); - } - } - } - - int distinctRowCount = (int) (this.rowCount / this.conflictRatio); - distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount; - // lookup tables - for (String lookupTable : lookupTableKeys.keySet()) { - this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount); - } - } - - private List<DimensionDesc> getSortedDimentsionDescs() { - List<DimensionDesc> dimensions = desc.getDimensions(); - Collections.sort(dimensions, new Comparator<DimensionDesc>() { - @Override - public int compare(DimensionDesc o1, DimensionDesc o2) { - JoinDesc j1 = o2.getJoin(); - JoinDesc j2 = o1.getJoin(); - return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0); - } - }); - return dimensions; - } - - /** - * Generate the fact table and return it as text - * - * @return - * @throws Exception - */ - private String cookData() throws Exception { - // the columns on the fact table can be classified into three groups: - // 1. foreign keys - TreeMap<String, String> factTableCol2LookupCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - // 2. metrics or directly used dimensions - TreeSet<String> usedCols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - // 3. others, not referenced anywhere - - TreeMap<String, String> lookupCol2factTableCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - - // find fact table columns in fks - List<DimensionDesc> dimensions = getSortedDimentsionDescs(); - for (DimensionDesc dim : dimensions) { - JoinDesc jDesc = dim.getJoin(); - if (jDesc != null) { - String[] fks = jDesc.getForeignKey(); - String[] pks = jDesc.getPrimaryKey(); - int num = fks.length; - for (int i = 0; i < num; ++i) { - String value = dim.getTable() + "/" + pks[i]; - - lookupCol2factTableCol.put(value, fks[i]); - - if (factTableCol2LookupCol.containsKey(fks[i])) { - if (!factTableCol2LookupCol.get(fks[i]).equals(value)) { - System.out.println("Warning: Disambiguation on the mapping of column " + fks[i] + ", " + factTableCol2LookupCol.get(fks[i]) + "(chosen) or " + value); - continue; - } - } - factTableCol2LookupCol.put(fks[i], value); - } - } - //else, deal with it in next roung - } - - // find fact table columns in direct dimension - // DO NOT merge this with the previous loop - for (DimensionDesc dim : dimensions) { - JoinDesc jDesc = dim.getJoin(); - if (jDesc == null) { - // column on fact table used directly as a dimension - for (String aColumn : dim.getColumn()) { - if (!factTableCol2LookupCol.containsKey(aColumn)) - usedCols.add(aColumn); - } - } - } - - // find fact table columns in measures - for (MeasureDesc mDesc : desc.getMeasures()) { - List<TblColRef> pcols = mDesc.getFunction().getParameter().getColRefs(); - if (pcols != null) { - for (TblColRef col : pcols) { - if (!factTableCol2LookupCol.containsKey(col.getName())) - usedCols.add(col.getName()); - } - } - } - - return createTable(this.rowCount, factTableCol2LookupCol, lookupCol2factTableCol, usedCols); - } - - private String normToTwoDigits(int v) { - if (v < 10) - return "0" + v; - else - return Integer.toString(v); - } - - private String randomPick(ArrayList<String> candidates) { - int index = r.nextInt(candidates.size()); - return candidates.get(index); - } - - private String createRandomCell(ColumnDesc cDesc, ArrayList<String> range) throws Exception { - DataType type = cDesc.getType(); - if (type.isStringFamily()) { - throw new Exception("Can't handle range values for string"); - - } else if (type.isIntegerFamily()) { - int low = Integer.parseInt(range.get(0)); - int high = Integer.parseInt(range.get(1)); - return Integer.toString(r.nextInt(high - low) + low); - - } else if (type.isDouble()) { - double low = Double.parseDouble(range.get(0)); - double high = Double.parseDouble(range.get(1)); - return String.format("%.4f", r.nextDouble() * (high - low) + low); - - } else if (type.isFloat()) { - float low = Float.parseFloat(range.get(0)); - float high = Float.parseFloat(range.get(1)); - return String.format("%.4f", r.nextFloat() * (high - low) + low); - - } else if (type.isDecimal()) { - double low = Double.parseDouble(range.get(0)); - double high = Double.parseDouble(range.get(1)); - return String.format("%.4f", r.nextDouble() * (high - low) + low); - - } else if (type.isDateTimeFamily()) { - - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); - Date start = format.parse(range.get(0)); - Date end = format.parse(range.get(1)); - long diff = end.getTime() - start.getTime(); - Date temp = new Date(start.getTime() + (long) (diff * r.nextDouble())); - Calendar cal = Calendar.getInstance(); - cal.setTime(temp); - // first day - cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek()); - - return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH)); - } else { - System.out.println("The data type " + type + "is not recognized"); - System.exit(1); - } - return null; - } - - private String createRandomCell(ColumnDesc cDesc) { - String type = cDesc.getTypeName(); - String s = type.toLowerCase(); - if (s.equals("string") || s.equals("char") || s.equals("varchar")) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 2; i++) { - sb.append((char) ('a' + r.nextInt(10)));// there are 10*10 - // possible strings - } - return sb.toString(); - } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) { - return Integer.toString(r.nextInt(128)); - } else if (s.equals("double")) { - return String.format("%.4f", r.nextDouble() * 100); - } else if (s.equals("float")) { - return String.format("%.4f", r.nextFloat() * 100); - } else if (s.equals("decimal")) { - return String.format("%.4f", r.nextDouble() * 100); - } else if (s.equals("date")) { - long date20131231 = 61349312153265L; - long date20010101 = 60939158400000L; - long diff = date20131231 - date20010101; - Date temp = new Date(date20010101 + (long) (diff * r.nextDouble())); - Calendar cal = Calendar.getInstance(); - cal.setTime(temp); - // first day - cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek()); - - return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH)); - } else { - System.out.println("The data type " + type + "is not recognized"); - System.exit(1); - } - return null; - } - - private String createDefaultsCell(String type) { - String s = type.toLowerCase(); - if (s.equals("string") || s.equals("char") || s.equals("varchar")) { - return "abcde"; - } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) { - return "0"; - } else if (s.equals("double")) { - return "0"; - } else if (s.equals("float")) { - return "0"; - } else if (s.equals("decimal")) { - return "0"; - } else if (s.equals("date")) { - return "1970-01-01"; - } else { - System.out.println("The data type " + type + "is not recognized"); - System.exit(1); - } - return null; - } - - private void printColumnMappings(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) { - - System.out.println("======================================================================="); - System.out.format("%-30s %s", "FACT_TABLE_COLUMN", "MAPPING"); - System.out.println(); - System.out.println(); - for (Map.Entry<String, String> entry : factTableCol2LookupCol.entrySet()) { - System.out.format("%-30s %s", entry.getKey(), entry.getValue()); - System.out.println(); - } - for (String key : usedCols) { - System.out.format("%-30s %s", key, "Random Values"); - System.out.println(); - } - for (String key : defaultColumns) { - System.out.format("%-30s %s", key, "Default Values"); - System.out.println(); - } - System.out.println("======================================================================="); - - System.out.println("Parameters:"); - System.out.println(); - System.out.println("CubeName: " + cubeName); - System.out.println("RowCount: " + rowCount); - System.out.println("ConflictRatio: " + conflictRatio); - System.out.println("LinkableRatio: " + linkableRatio); - System.out.println("Seed: " + randomSeed); - System.out.println(); - System.out.println("The number of actual unlinkable fact rows is: " + this.unlinkableRowCount); - System.out.println("You can vary the above parameters to generate different datasets."); - System.out.println(); - } - - // Any row in the column must finally appear in the flatten big table. - // for single-column joins the generated row is guaranteed to have a match - // in lookup table - // for composite keys we'll need an extra check - private boolean matchAllCompositeKeys(TreeMap<String, String> lookupCol2FactTableCol, LinkedList<String> columnValues) { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - - for (String lookupTable : lookupTableKeys.keySet()) { - if (lookupTableKeys.get(lookupTable).size() == 1) - continue; - - String[] comboKey = new String[lookupTableKeys.get(lookupTable).size()]; - int index = 0; - for (String column : lookupTableKeys.get(lookupTable)) { - String key = lookupTable + "/" + column; - String factTableCol = lookupCol2FactTableCol.get(key); - int cardinal = MetadataManager.getInstance(config).getTableDesc(factTableName).findColumnByName(factTableCol).getZeroBasedIndex(); - comboKey[index] = columnValues.get(cardinal); - - index++; - } - Array<String> wrap = new Array<String>(comboKey); - if (!lookupTableCompositeKeyValues.get(lookupTable).contains(wrap)) { - // System.out.println("Try " + wrap + " Failed, continue..."); - return false; - } - } - return true; - } - - private String createCell(ColumnDesc cDesc) throws Exception { - ColumnConfig cConfig = null; - - if ((cConfig = genConf.getColumnConfigByName(cDesc.getName())) == null) { - // if the column is not configured, use random values - return (createRandomCell(cDesc)); - - } else { - // the column has a configuration - if (!cConfig.isAsRange() && !cConfig.isExclusive() && r.nextBoolean()) { - // if the column still allows random values - return (createRandomCell(cDesc)); - - } else { - // use specified values - ArrayList<String> valueSet = cConfig.getValueSet(); - if (valueSet == null || valueSet.size() == 0) - throw new Exception("Did you forget to specify value set for " + cDesc.getName()); - - if (!cConfig.isAsRange()) { - return (randomPick(valueSet)); - } else { - if (valueSet.size() != 2) - throw new Exception("Only two values can be set for range values, the column: " + cDesc.getName()); - - return (createRandomCell(cDesc, valueSet)); - } - } - - } - } - - private LinkedList<String> createRow(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) throws Exception { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - LinkedList<String> columnValues = new LinkedList<String>(); - - for (ColumnDesc cDesc : MetadataManager.getInstance(config).getTableDesc(factTableName).getColumns()) { - - String colName = cDesc.getName(); - - if (factTableCol2LookupCol.containsKey(colName)) { - - // if the current column is a fk column in fact table - ArrayList<String> candidates = this.feasibleValues.get(factTableCol2LookupCol.get(colName)); - - columnValues.add(candidates.get(r.nextInt(candidates.size()))); - } else if (usedCols.contains(colName)) { - - // if the current column is a metric column in fact table - columnValues.add(createCell(cDesc)); - } else { - - // otherwise this column is not useful in OLAP - columnValues.add(createDefaultsCell(cDesc.getTypeName())); - defaultColumns.add(colName); - } - } - - return columnValues; - } - - /** - * return the text of table contents(one line one row) - * - * @param rowCount - * @param factTableCol2LookupCol - * @param lookupCol2FactTableCol - * @param usedCols - * @return - * @throws Exception - */ - private String createTable(int rowCount, TreeMap<String, String> factTableCol2LookupCol, TreeMap<String, String> lookupCol2FactTableCol, TreeSet<String> usedCols) throws Exception { - try { - TreeSet<String> defaultColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < rowCount;) { - - LinkedList<String> columnValues = createRow(factTableCol2LookupCol, usedCols, defaultColumns); - - if (!matchAllCompositeKeys(lookupCol2FactTableCol, columnValues)) { - if (unlinkableRowCount < unlinkableRowCountMax) { - unlinkableRowCount++; - } else { - continue; - } - } - - for (String c : columnValues) - sb.append(c + ","); - sb.deleteCharAt(sb.length() - 1); - sb.append(System.getProperty("line.separator")); - - i++; - - // System.out.println("Just generated the " + i + "th record"); - } - - printColumnMappings(factTableCol2LookupCol, usedCols, defaultColumns); - - return sb.toString(); - - } catch (IOException e) { - e.printStackTrace(); - System.exit(1); - } - - return null; - } - - /** - * Randomly create a fact table and return the table content - * - * @param cubeName name of the cube - * @param rowCount expected row count generated - * @param linkableRatio the percentage of fact table rows that can be linked with all - * lookup table by INNER join - * @param randomSeed random seed - */ - public static String generate(String cubeName, String rowCount, String linkableRatio, String randomSeed, String joinType) throws Exception { - - if (cubeName == null) - cubeName = "test_kylin_cube_with_slr_ready"; - if (rowCount == null) - rowCount = "10000"; - if (linkableRatio == null) - linkableRatio = "0.6"; - - //if (randomSeed == null) - // don't give it value - - // String conflictRatio = "5";//this parameter do not allow configuring - // any more - - FactTableGenerator generator = new FactTableGenerator(); - long seed; - if (randomSeed != null) { - seed = Long.parseLong(randomSeed); - } else { - Random r = new Random(); - seed = r.nextLong(); - } - - generator.init(cubeName, Integer.parseInt(rowCount), 5, Double.parseDouble(linkableRatio), seed); - generator.prepare(); - return generator.cookData(); - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java b/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java deleted file mode 100644 index f7d7341..0000000 --- a/job/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.dataGen; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; - -import org.apache.kylin.common.util.JsonUtil; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; - -/** - * Created by honma on 5/29/14. - */ -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class GenConfig { - - @JsonProperty("columnConfigs") - private ArrayList<ColumnConfig> columnConfigs; - - private HashMap<String, ColumnConfig> cache = new HashMap<String, ColumnConfig>(); - - public ArrayList<ColumnConfig> getColumnConfigs() { - return columnConfigs; - } - - public void setColumnConfigs(ArrayList<ColumnConfig> columnConfigs) { - this.columnConfigs = columnConfigs; - } - - public ColumnConfig getColumnConfigByName(String columnName) { - columnName = columnName.toLowerCase(); - - if (cache.containsKey(columnName)) - return cache.get(columnName); - - for (ColumnConfig cConfig : columnConfigs) { - if (cConfig.getColumnName().toLowerCase().equals(columnName)) { - cache.put(columnName, cConfig); - return cConfig; - } - } - cache.put(columnName, null); - return null; - } - - public static GenConfig loadConfig(InputStream stream) { - try { - GenConfig config = JsonUtil.readValue(stream, GenConfig.class); - return config; - } catch (JsonMappingException e) { - e.printStackTrace(); - } catch (JsonParseException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - return null; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java deleted file mode 100644 index 8d22efb..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; - -import java.io.File; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class BaseCuboidJobTest extends LocalFileMetadataTestCase { - - public static void setLocalMR(Configuration conf) { - conf.set("fs.default.name", "file:///"); - conf.set("mapreduce.framework.name", "local"); - conf.set("mapreduce.application.framework.path", ""); - - // for local runner out-of-memory issue - conf.set("mapreduce.task.io.sort.mb", "10"); - } - - private Configuration conf; - - @Before - public void setup() throws Exception { - conf = new Configuration(); - BaseCuboidJobTest.setLocalMR(conf); - - createTestMetadata(); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void testJob() throws Exception { - String input = "src/test/resources/data/flat_table/"; - String output = "target/test-output/base_cuboid/"; - String cubeName = "test_kylin_cube_with_slr_1_new_segment"; - String segmentName = "20130331080000_20131212080000"; - String jobname = "base_cuboid_job"; - String level = "0"; - FileUtil.fullyDelete(new File(output)); - - String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level }; - assertEquals("Job failed", 0, ToolRunner.run(conf, new BaseCuboidJob(), args)); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java deleted file mode 100644 index f87bb0a..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperPerformanceTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.Reader; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapreduce.Mapper.Context; -import org.apache.hadoop.util.ReflectionUtils; -import org.junit.Ignore; -import org.junit.Test; - -/** - * @author yangli9 - * - */ -@SuppressWarnings({ "rawtypes", "unchecked" }) -public class BaseCuboidMapperPerformanceTest { - - String metadataUrl = "hbase:yadesk00:2181:/hbase-unsecure"; - String cubeName = "test_kylin_cube_with_slr"; - Path srcPath = new Path("/download/test_kylin_cube_with_slr_intermediate_table_64mb.seq"); - - @Ignore("convenient trial tool for dev") - @Test - public void test() throws IOException, InterruptedException { - Configuration hconf = new Configuration(); - BaseCuboidMapper mapper = new BaseCuboidMapper(); - Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null); - - mapper.setup(context); - - Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath)); - Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); - Text value = new Text(); - - while (reader.next(key, value)) { - mapper.map(key, value, context); - } - - reader.close(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java deleted file mode 100644 index 58d093a..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidMapperTest.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.kv.RowKeyDecoder; -import org.apache.kylin.job.constant.BatchConstants; -import org.apache.kylin.measure.MeasureCodec; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class BaseCuboidMapperTest extends LocalFileMetadataTestCase { - - MapDriver<Text, Text, Text, Text> mapDriver; - - @Before - public void setUp() throws Exception { - createTestMetadata(); - - // hack for distributed cache - FileUtils.deleteDirectory(new File("../job/meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); - - BaseCuboidMapper<Text> mapper = new BaseCuboidMapper<Text>(); - mapDriver = MapDriver.newMapDriver(mapper); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - FileUtils.deleteDirectory(new File("../job/meta")); - } - - @Test - public void testMapperWithHeader() throws Exception { - String cubeName = "test_kylin_cube_with_slr_1_new_segment"; - String segmentName = "20130331080000_20131212080000"; - mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); - mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); - // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, - // metadata); - mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.3322")); - List<Pair<Text, Text>> result = mapDriver.run(); - - CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); - CubeInstance cube = cubeMgr.getCube(cubeName); - - assertEquals(1, result.size()); - Text rowkey = result.get(0).getFirst(); - byte[] key = rowkey.getBytes(); - byte[] header = Bytes.head(key, 26); - byte[] sellerId = Bytes.tail(header, 18); - byte[] cuboidId = Bytes.head(header, 8); - byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26); - - RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment()); - decoder.decode(key); - assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString()); - - assertTrue(Bytes.toString(sellerId).startsWith("123456789")); - assertEquals(511, Bytes.toLong(cuboidId)); - assertEquals(22, restKey.length); - - verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33", "1", "22"); - } - - private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String... valueStr) { - MeasureCodec codec = new MeasureCodec(measures); - Object[] values = new Object[measures.size()]; - codec.decode(ByteBuffer.wrap(valueBytes.getBytes()), values); - assertTrue(new BigDecimal(valueStr[0]).equals(values[0])); - assertTrue(new BigDecimal(valueStr[1]).equals(values[1])); - assertTrue(new BigDecimal(valueStr[2]).equals(values[2])); - assertTrue(new LongWritable(Long.valueOf(valueStr[3])).equals(values[3])); - assertTrue(new LongWritable(Long.valueOf(valueStr[4])).equals(values[4])); - } - - @Test - public void testMapperWithNull() throws Exception { - String cubeName = "test_kylin_cube_with_slr_1_new_segment"; - String segmentName = "20130331080000_20131212080000"; - mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); - mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); - // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, - // metadata); - mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N22")); - List<Pair<Text, Text>> result = mapDriver.run(); - - CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); - CubeInstance cube = cubeMgr.getCube(cubeName); - - assertEquals(1, result.size()); - Text rowkey = result.get(0).getFirst(); - byte[] key = rowkey.getBytes(); - byte[] header = Bytes.head(key, 26); - byte[] sellerId = Bytes.tail(header, 18); - byte[] cuboidId = Bytes.head(header, 8); - byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26); - - RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment()); - decoder.decode(key); - assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString()); - - assertTrue(Bytes.toString(sellerId).startsWith("123456789")); - assertEquals(511, Bytes.toLong(cuboidId)); - assertEquals(22, restKey.length); - - verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0", "1", "22"); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java deleted file mode 100644 index a8e29e6..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.io.SequenceFile.Reader; -import org.apache.hadoop.io.SequenceFile.Writer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.io.compress.CompressionCodecFactory; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * @author yangli9 - * - */ -public class CopySeq { - - public static void main(String[] args) throws IOException { - copyTo64MB(args[0], args[1]); - } - - public static void copyTo64MB(String src, String dst) throws IOException { - Configuration hconf = new Configuration(); - Path srcPath = new Path(src); - Path dstPath = new Path(dst); - - FileSystem fs = FileSystem.get(hconf); - long srcSize = fs.getFileStatus(srcPath).getLen(); - int copyTimes = (int) (67108864 / srcSize); // 64 MB - System.out.println("Copy " + copyTimes + " times"); - - Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath)); - Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); - Text value = new Text(); - - Writer writer = SequenceFile.createWriter(hconf, Writer.file(dstPath), Writer.keyClass(key.getClass()), Writer.valueClass(Text.class), Writer.compression(CompressionType.BLOCK, getLZOCodec(hconf))); - - int count = 0; - while (reader.next(key, value)) { - for (int i = 0; i < copyTimes; i++) { - writer.append(key, value); - count++; - } - } - - System.out.println("Len: " + writer.getLength()); - System.out.println("Rows: " + count); - - reader.close(); - writer.close(); - } - - static CompressionCodec getLZOCodec(Configuration hconf) { - CompressionCodecFactory factory = new CompressionCodecFactory(hconf); - return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec"); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java deleted file mode 100644 index c9b7eba..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.nio.ByteBuffer; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Mapper.Context; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.kv.RowConstants; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.measure.MeasureCodec; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author yangli9 - * - */ -@SuppressWarnings({ "rawtypes", "unchecked" }) -public class CubeHFileMapper2Test extends LocalFileMetadataTestCase { - - String cubeName = "test_kylin_cube_with_slr_ready"; - - MeasureCodec codec; - ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); - Object[] outKV = new Object[2]; - - @Before - public void setup() throws Exception { - this.createTestMetadata(); - // hack for distributed cache - FileUtils.deleteDirectory(new File("../job/meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); - CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor(); - codec = new MeasureCodec(desc.getMeasures()); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - FileUtils.deleteDirectory(new File("../job/meta")); - } - - @Test - public void testBasic() throws Exception { - - Configuration hconf = new Configuration(); - Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV); - - CubeHFileMapper mapper = new CubeHFileMapper(); - mapper.setup(context); - - Text key = new Text("not important"); - Text value = new Text(new byte[] { 2, 2, 51, -79, 1 }); - - mapper.map(key, value, context); - - ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0]; - KeyValue outValue = (KeyValue) outKV[1]; - - assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(), outKey.getLength()) == 0); - - assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(), outValue.getValueOffset(), outValue.getValueLength()) == 0); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java deleted file mode 100644 index 3555782..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.job.constant.BatchConstants; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * @author George Song (ysong1) - * - */ -public class CubeHFileMapperTest { - - MapDriver<Text, Text, ImmutableBytesWritable, KeyValue> mapDriver; - - private String cube_name = "FLAT_ITEM_CUBE"; - - @Before - public void setUp() { - CubeHFileMapper mapper = new CubeHFileMapper(); - mapDriver = MapDriver.newMapDriver(mapper); - } - - @SuppressWarnings("deprecation") - @Test - @Ignore("not maintaining") - public void testMapper2() throws IOException { - mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name); - - mapDriver.addInput(new Text("52010tech"), new Text("35.432")); - - List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run(); - - assertEquals(2, result.size()); - - byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 }; - ImmutableBytesWritable key = new ImmutableBytesWritable(bytes); - - Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0); - Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1); - - assertEquals(key, p1.getFirst()); - assertEquals("cf1", new String(p1.getSecond().getFamily())); - assertEquals("usd_amt", new String(p1.getSecond().getQualifier())); - assertEquals("35.43", new String(p1.getSecond().getValue())); - - assertEquals(key, p2.getFirst()); - assertEquals("cf1", new String(p2.getSecond().getFamily())); - assertEquals("item_count", new String(p2.getSecond().getQualifier())); - assertEquals("2", new String(p2.getSecond().getValue())); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java deleted file mode 100644 index 51f3990..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.lang.reflect.Field; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.kv.RowConstants; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.job.constant.BatchConstants; -import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.measure.MeasureCodec; -import org.apache.kylin.measure.MeasureIngester; -import org.apache.kylin.measure.MeasureType; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class CubeReducerTest extends LocalFileMetadataTestCase { - - ReduceDriver<Text, Text, Text, Text> reduceDriver; - - ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); - - @Before - public void setUp() throws Exception { - createTestMetadata(); - - // hack for distributed cache - FileUtils.deleteDirectory(new File("../job/meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); - - CuboidReducer reducer = new CuboidReducer(); - reduceDriver = ReduceDriver.newReduceDriver(reducer); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - FileUtils.deleteDirectory(new File("../job/meta")); - } - - @Test - public void testReducer() throws Exception { - - reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); - - CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); - MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures()); - - Text key1 = new Text("72010ustech"); - List<Text> values1 = new ArrayList<Text>(); - values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 22)); - values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 23)); - values1.add(newValueText(codec, "10", "10", "10", 1, 24)); - - Text key2 = new Text("1tech"); - List<Text> values2 = new ArrayList<Text>(); - values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 12)); - values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 13)); - - Text key3 = new Text("0"); - List<Text> values3 = new ArrayList<Text>(); - values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4, 11)); - - reduceDriver.withInput(key1, values1); - reduceDriver.withInput(key2, values2); - reduceDriver.withInput(key3, values3); - - List<Pair<Text, Text>> result = reduceDriver.run(); - - Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3, 69)); - Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2, 25)); - Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4, 11)); - - assertEquals(3, result.size()); - - assertTrue(result.contains(p1)); - assertTrue(result.contains(p2)); - assertTrue(result.contains(p3)); - } - - @Test - public void testReducerOnlyAggrInBaseCuboid() throws Exception { - reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready"); - reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1); - - CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); - MeasureDesc measureDesc = cubeDesc.getMeasures().get(0); - FunctionDesc functionDesc = measureDesc.getFunction(); - Field field = FunctionDesc.class.getDeclaredField("measureType"); - field.setAccessible(true); - MeasureType origMeasureType = functionDesc.getMeasureType(); - field.set(functionDesc, new MockUpMeasureType(origMeasureType)); - - MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures()); - - Text key1 = new Text("72010ustech"); - List<Text> values1 = new ArrayList<Text>(); - values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100)); - values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200)); - values1.add(newValueText(codec, "10", "10", "10", 1, 300)); - - Text key2 = new Text("1tech"); - List<Text> values2 = new ArrayList<Text>(); - values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500)); - values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000)); - - Text key3 = new Text("0"); - List<Text> values3 = new ArrayList<Text>(); - values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0)); - - reduceDriver.withInput(key1, values1); - reduceDriver.withInput(key2, values2); - reduceDriver.withInput(key3, values3); - - List<Pair<Text, Text>> result = reduceDriver.run(); - - Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600)); - Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500)); - Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0)); - - assertEquals(3, result.size()); - - assertTrue(result.contains(p1)); - assertTrue(result.contains(p2)); - assertTrue(result.contains(p3)); - } - - private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count, int itemcount) { - Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongWritable(count), new LongWritable(itemcount) }; - - buf.clear(); - codec.encode(values, buf); - - Text t = new Text(); - t.set(buf.array(), 0, buf.position()); - return t; - } - - class MockUpMeasureType extends MeasureType { - MeasureType origMeasureType; - - public MockUpMeasureType(MeasureType origMeasureType) { - this.origMeasureType = origMeasureType; - } - - @Override - public boolean onlyAggrInBaseCuboid() { - return true; - } - - @Override - public MeasureIngester newIngester() { - return origMeasureType.newIngester(); - } - - @Override - public MeasureAggregator newAggregator() { - return origMeasureType.newAggregator(); - } - - @Override - public boolean needRewrite() { - return origMeasureType.needRewrite(); - } - - @Override - public Class<?> getRewriteCalciteAggrFunctionClass() { - return origMeasureType.getRewriteCalciteAggrFunctionClass(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java deleted file mode 100644 index 6f7db6a..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -///* -// * Copyright 2013-2014 eBay Software Foundation -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package org.apache.kylin.index.cube; -// -//import static org.junit.Assert.*; -// -//import java.io.File; -//import java.io.IOException; -// -//import org.apache.hadoop.conf.Configuration; -//import org.apache.hadoop.fs.FileUtil; -//import org.apache.hadoop.util.ToolRunner; -//import org.junit.Before; -//import org.junit.Test; -// -//import org.apache.kylin.metadata.MetadataManager; -// -///** -// * @author xjiang -// * -// */ -//public class KeyDistributionJobTest { -// -// private Configuration conf; -// -// @Before -// public void setup() throws IOException { -// conf = new Configuration(); -// BaseCuboidJobTest.setLocalMR(conf); -// } -// -// @Test -// public void testJob() throws Exception { -// final String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/"; -// final String output = "target/test-output/key_distribution/"; -// final String cubeName = "test_kylin_cube_with_slr"; -// final String metadata = MetadataManager.getMetadataUrlFromEnv(); -// -// FileUtil.fullyDelete(new File(output)); -// -// String[] args = -// { "-input", input, "-cubename", cubeName, "-output", output, "-metadata", metadata, -// "-columnpercentage", "50", "-splitnumber", "10" }; -// assertEquals("Job failed", 0, ToolRunner.run(new KeyDistributionJob(), args)); -// } -// -// } http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java deleted file mode 100644 index b4577f7..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -///* -// * Copyright 2013-2014 eBay Software Foundation -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package org.apache.kylin.index.cube; -// -//import static org.junit.Assert.*; -// -//import java.io.File; -//import java.io.IOException; -//import java.util.List; -// -//import org.apache.hadoop.io.LongWritable; -//import org.apache.hadoop.io.Text; -//import org.apache.hadoop.mrunit.mapreduce.MapDriver; -//import org.apache.hadoop.mrunit.types.Pair; -//import org.junit.Before; -//import org.junit.Test; -// -//import org.apache.kylin.index.BatchConstants; -//import org.apache.kylin.metadata.MetadataManager; -// -///** -// * @author ysong1 -// * -// */ -//public class KeyDistributionMapperTest { -// @SuppressWarnings("rawtypes") -// MapDriver mapDriver; -// String localTempDir = System.getProperty("java.io.tmpdir") + File.separator; -// -// @Before -// public void setUp() { -// KeyDistributionMapper mapper = new KeyDistributionMapper(); -// mapDriver = MapDriver.newMapDriver(mapper); -// } -// -// @SuppressWarnings("unchecked") -// @Test -// public void testMapperWithoutHeader() throws IOException { -// String matadata = MetadataManager.getMetadataUrlFromEnv(); -// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_without_slr"); -// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata); -// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7"); -// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "8"); -// -// Text inputKey1 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey2 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey3 = -// new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey4 = -// new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey5 = -// new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey6 = -// new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey7 = -// new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// -// mapDriver.addInput(inputKey1, new Text("abc")); -// mapDriver.addInput(inputKey2, new Text("abc")); -// mapDriver.addInput(inputKey3, new Text("abc")); -// mapDriver.addInput(inputKey4, new Text("abc")); -// mapDriver.addInput(inputKey5, new Text("abc")); -// mapDriver.addInput(inputKey6, new Text("abc")); -// mapDriver.addInput(inputKey7, new Text("abc")); -// -// List<Pair<Text, LongWritable>> result = mapDriver.run(); -// -// assertEquals(7, result.size()); -// -// byte[] key1 = result.get(0).getFirst().getBytes(); -// LongWritable value1 = result.get(0).getSecond(); -// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11 }, key1); -// assertEquals(2, value1.get()); -// -// byte[] key7 = result.get(6).getFirst().getBytes(); -// LongWritable value7 = result.get(6).getSecond(); -// assertArrayEquals(new byte[] { 0 }, key7); -// assertEquals(7, value7.get()); -// } -// -// @SuppressWarnings("unchecked") -// @Test -// public void testMapperWithHeader() throws IOException { -// String matadata = MetadataManager.getMetadataUrlFromEnv(); -// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr"); -// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata); -// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7"); -// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "26"); -// -// Text inputKey1 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, -// 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey2 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, -// 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey3 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, -// 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey4 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, -// 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey5 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, -// 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey6 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, -// 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// Text inputKey7 = -// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, -// 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); -// -// mapDriver.addInput(inputKey1, new Text("abc")); -// mapDriver.addInput(inputKey2, new Text("abc")); -// mapDriver.addInput(inputKey3, new Text("abc")); -// mapDriver.addInput(inputKey4, new Text("abc")); -// mapDriver.addInput(inputKey5, new Text("abc")); -// mapDriver.addInput(inputKey6, new Text("abc")); -// mapDriver.addInput(inputKey7, new Text("abc")); -// -// List<Pair<Text, LongWritable>> result = mapDriver.run(); -// -// assertEquals(7, result.size()); -// -// byte[] key1 = result.get(0).getFirst().getBytes(); -// LongWritable value1 = result.get(0).getSecond(); -// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, -// 0, 0, 0, 0, 0, 127, 11 }, key1); -// assertEquals(2, value1.get()); -// -// byte[] key7 = result.get(6).getFirst().getBytes(); -// LongWritable value7 = result.get(6).getSecond(); -// assertArrayEquals(new byte[] { 0 }, key7); -// assertEquals(7, value7.get()); -// } -// } http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java deleted file mode 100644 index 6c17c69..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; - -import java.io.File; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class MergeCuboidJobTest extends LocalFileMetadataTestCase { - - private Configuration conf; - - @Before - public void setup() throws Exception { - conf = new Configuration(); - BaseCuboidJobTest.setLocalMR(conf); - - createTestMetadata(); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - // String input = - // "src/test/resources/data/base_cuboid,src/test/resources/data/6d_cuboid"; - String output = "target/test-output/merged_cuboid"; - String cubeName = "test_kylin_cube_with_slr_ready"; - String jobname = "merge_cuboid"; - - File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base"); - FileUtils.forceDelete(baseFolder); - baseFolder.mkdir(); - FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder); - FileUtils.forceDeleteOnExit(baseFolder); - - File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d"); - FileUtils.forceDelete(eightFolder); - eightFolder.mkdir(); - FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder); - FileUtils.forceDeleteOnExit(eightFolder); - - FileUtil.fullyDelete(new File(output)); - - // CubeManager cubeManager = - // CubeManager.getInstanceFromEnv(getTestConfig()); - - String[] args = { "-input", baseFolder.getAbsolutePath() + "," + eightFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname }; - assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args)); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java deleted file mode 100644 index ea5c163..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.DictionaryGenerator; -import org.apache.kylin.dict.DictionaryInfo; -import org.apache.kylin.dict.DictionaryManager; -import org.apache.kylin.dict.TrieDictionary; -import org.apache.kylin.dict.ListDictionaryValueEnumerator; -import org.apache.kylin.dict.lookup.ReadableTable.TableSignature; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.project.ProjectManager; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author honma - */ -@SuppressWarnings("rawtypes") -public class MergeCuboidMapperTest extends LocalFileMetadataTestCase { - - private static final Logger logger = LoggerFactory.getLogger(MergeCuboidMapperTest.class); - - MapDriver<Text, Text, Text, Text> mapDriver; - CubeManager cubeManager; - CubeInstance cube; - DictionaryManager dictionaryManager; - - TblColRef lfn; - TblColRef lsi; - TblColRef ssc; - - private DictionaryInfo makeSharedDict() throws IOException { - TableSignature signature = new TableSignature(); - signature.setSize(100); - signature.setLastModifiedTime(System.currentTimeMillis()); - signature.setPath("fake_common_dict"); - - DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature); - - List<byte[]> values = new ArrayList<byte[]>(); - values.add(new byte[] { 101, 101, 101 }); - values.add(new byte[] { 102, 102, 102 }); - Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(newDictInfo, new ListDictionaryValueEnumerator(values)); - dictionaryManager.trySaveNewDict(dict, newDictInfo); - ((TrieDictionary) dict).dump(System.out); - - return newDictInfo; - } - - @Before - public void setUp() throws Exception { - - createTestMetadata(); - - logger.info("The metadataUrl is : " + getTestConfig()); - - MetadataManager.clearCache(); - CubeManager.clearCache(); - ProjectManager.clearCache(); - DictionaryManager.clearCache(); - - // hack for distributed cache - // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to - // make sure the following mapper could get latest CubeManger - FileUtils.deleteDirectory(new File("../job/meta")); - - MergeCuboidMapper mapper = new MergeCuboidMapper(); - mapDriver = MapDriver.newMapDriver(mapper); - - cubeManager = CubeManager.getInstance(getTestConfig()); - cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments"); - dictionaryManager = DictionaryManager.getInstance(getTestConfig()); - lfn = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME"); - lsi = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT"); - ssc = cube.getDescriptor().findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME"); - - DictionaryInfo sharedDict = makeSharedDict(); - - boolean isFirstSegment = true; - for (CubeSegment segment : cube.getSegments()) { - - TableSignature signature = new TableSignature(); - signature.setSize(100); - signature.setLastModifiedTime(System.currentTimeMillis()); - signature.setPath("fake_dict_for" + lfn.getName() + segment.getName()); - - DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature); - - List<byte[]> values = new ArrayList<byte[]>(); - values.add(new byte[] { 97, 97, 97 }); - if (isFirstSegment) - values.add(new byte[] { 99, 99, 99 }); - else - values.add(new byte[] { 98, 98, 98 }); - Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(newDictInfo, new ListDictionaryValueEnumerator(values)); - dictionaryManager.trySaveNewDict(dict, newDictInfo); - ((TrieDictionary) dict).dump(System.out); - - segment.putDictResPath(lfn, newDictInfo.getResourcePath()); - segment.putDictResPath(lsi, sharedDict.getResourcePath()); - segment.putDictResPath(ssc, sharedDict.getResourcePath()); - - // cubeManager.saveResource(segment.getCubeInstance()); - // cubeManager.afterCubeUpdated(segment.getCubeInstance()); - cubeManager.updateCube(cube); - - isFirstSegment = false; - } - - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - FileUtils.deleteDirectory(new File("../job/meta")); - } - - @Test - public void test() throws IOException, ParseException { - - String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments"; - - CubeSegment newSeg = cubeManager.mergeSegments(cube, 0L, 1386835200000L, false); - String segmentName = newSeg.getName(); - - final Dictionary<?> dictionary = cubeManager.getDictionary(newSeg, lfn); - assertTrue(dictionary == null); - // ((TrieDictionary) dictionary).dump(System.out); - - // hack for distributed cache - // File metaDir = new File("../job/meta"); - // FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), metaDir); - // - // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); - // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); - // // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL, - // // "../job/meta"); - // - // byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 }; - // byte[] value = new byte[] { 1, 2, 3 }; - // byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 }; - // byte[] newvalue = new byte[] { 1, 2, 3 }; - // - // mapDriver.withInput(new Text(key), new Text(value)); - // mapDriver.withOutput(new Text(newkey), new Text(newvalue)); - // mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid")); - // - // mapDriver.runTest(); - } -}
