Reverted changes to logic in core module and refactored HBaseColumn class so it now extends the AbstractColumn class (and implement the Column class).
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/2008c7eb Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/2008c7eb Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/2008c7eb Branch: refs/heads/master Commit: 2008c7eb641c647ffe2588a1119b5d52f8fc9809 Parents: f15d027 Author: arjansh <[email protected]> Authored: Mon Apr 30 11:49:52 2018 +0200 Committer: arjansh <[email protected]> Committed: Mon Apr 30 11:49:52 2018 +0200 ---------------------------------------------------------------------- .../metamodel/AbstractUpdateCallback.java | 4 +- .../metamodel/data/AbstractRowBuilder.java | 33 +----- .../insert/AbstractRowInsertionBuilder.java | 10 +- hbase/pom.xml | 111 ------------------ .../org/apache/metamodel/hbase/HBaseColumn.java | 110 +++++++++++++++--- .../hbase/HBaseRowInsertionBuilder.java | 6 +- .../metamodel/hbase/HBaseUpdateCallback.java | 24 +--- .../hbase/qualifiers/FindQualifiersDriver.java | 116 ------------------- 8 files changed, 110 insertions(+), 304 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/core/src/main/java/org/apache/metamodel/AbstractUpdateCallback.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/metamodel/AbstractUpdateCallback.java b/core/src/main/java/org/apache/metamodel/AbstractUpdateCallback.java index b41b481..4ff7279 100644 --- a/core/src/main/java/org/apache/metamodel/AbstractUpdateCallback.java +++ b/core/src/main/java/org/apache/metamodel/AbstractUpdateCallback.java @@ -117,7 +117,7 @@ public abstract class AbstractUpdateCallback implements UpdateCallback { return update(getTable(tableName)); } - protected Table getTable(String tableName) { + private Table getTable(String tableName) { Table table = getDataContext().getTableByQualifiedLabel(tableName); if (table == null) { throw new IllegalArgumentException("No such table: " + tableName); @@ -159,7 +159,7 @@ public abstract class AbstractUpdateCallback implements UpdateCallback { UnsupportedOperationException { return new DeleteAndInsertBuilder(this, table); } - + public UpdateSummary getUpdateSummary() { return DefaultUpdateSummary.unknownUpdates(); } http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/core/src/main/java/org/apache/metamodel/data/AbstractRowBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/metamodel/data/AbstractRowBuilder.java b/core/src/main/java/org/apache/metamodel/data/AbstractRowBuilder.java index 4fdbc59..144017c 100644 --- a/core/src/main/java/org/apache/metamodel/data/AbstractRowBuilder.java +++ b/core/src/main/java/org/apache/metamodel/data/AbstractRowBuilder.java @@ -40,10 +40,6 @@ public abstract class AbstractRowBuilder<RB extends RowBuilder<?>> implements Ro this(table.getColumns()); } - public AbstractRowBuilder(Table table, int numberOfValues) { - this(table.getColumns(), numberOfValues); - } - public AbstractRowBuilder(List<Column> columns) { _columns = columns.toArray(new Column[columns.size()]); _explicitNulls = new boolean[_columns.length]; @@ -51,14 +47,6 @@ public abstract class AbstractRowBuilder<RB extends RowBuilder<?>> implements Ro _styles = new Style[_columns.length]; } - public AbstractRowBuilder(List<Column> columns, int numberOfValues) { - _columns = columns.toArray(new Column[columns.size()]); - _explicitNulls = new boolean[numberOfValues]; - _values = new Object[numberOfValues]; - _styles = new Style[numberOfValues]; - setColumns(columns); - } - /** * Gets a boolean array indicating if any of the values have been explicitly * set to null (as opposed to just not set) @@ -83,8 +71,7 @@ public abstract class AbstractRowBuilder<RB extends RowBuilder<?>> implements Ro @Override public final Row toRow() { - return new DefaultRow(new SimpleDataSetHeader(Arrays.stream(_columns).map(SelectItem::new).collect(Collectors - .toList())), _values); + return new DefaultRow(new SimpleDataSetHeader(Arrays.stream(_columns).map(SelectItem::new).collect(Collectors.toList())), _values); } @Override @@ -159,22 +146,4 @@ public abstract class AbstractRowBuilder<RB extends RowBuilder<?>> implements Ro } return false; } - - public void setColumns(List<Column> columns) { - if (columns.size() != _columns.length) { - throw new IllegalArgumentException("The amount of columns don't match"); - } - for (int i = 0; i < _columns.length; i++) { - _columns[i] = columns.get(i); - } - } - - public void setValues(Object[] values) { - if (values.length != _values.length) { - throw new IllegalArgumentException("The amount of values don't match"); - } - for (int i = 0; i < values.length; i++) { - _values[i] = values[i]; - } - } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/core/src/main/java/org/apache/metamodel/insert/AbstractRowInsertionBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/metamodel/insert/AbstractRowInsertionBuilder.java b/core/src/main/java/org/apache/metamodel/insert/AbstractRowInsertionBuilder.java index 185f942..58c0da8 100644 --- a/core/src/main/java/org/apache/metamodel/insert/AbstractRowInsertionBuilder.java +++ b/core/src/main/java/org/apache/metamodel/insert/AbstractRowInsertionBuilder.java @@ -18,8 +18,6 @@ */ package org.apache.metamodel.insert; -import java.util.List; - import org.apache.metamodel.UpdateCallback; import org.apache.metamodel.data.AbstractRowBuilder; import org.apache.metamodel.data.Row; @@ -27,6 +25,8 @@ import org.apache.metamodel.query.SelectItem; import org.apache.metamodel.schema.Column; import org.apache.metamodel.schema.Table; +import java.util.List; + /** * Abstract implementation of the {@link RowInsertionBuilder} interface, * provided as a convenience to {@link RowInsertable} implementations. Handles @@ -44,12 +44,6 @@ public abstract class AbstractRowInsertionBuilder<U extends UpdateCallback> exte _table = table; } - public AbstractRowInsertionBuilder(U updateCallback, Table table, int numberOfValues) { - super(table, numberOfValues); - _updateCallback = updateCallback; - _table = table; - } - @Override public Table getTable() { return _table; http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/hbase/pom.xml b/hbase/pom.xml index 47101ba..3a1febe 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -118,113 +118,6 @@ <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> </exclusion> - <!-- <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </exclusion> --> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <!-- <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core-asl</artifactId> - </exclusion> --> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-jaxrs</artifactId> - </exclusion> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-xc</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - <version>${hbase.version}</version> - <exclusions> - <exclusion> - <groupId>jdk.tools</groupId> - <artifactId>jdk.tools</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <artifactId>log4j</artifactId> - <groupId>log4j</groupId> - </exclusion> - <exclusion> - <artifactId>commons-logging</artifactId> - <groupId>commons-logging</groupId> - </exclusion> - <exclusion> - <artifactId>jetty</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <artifactId>jetty-util</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <groupId>com.github.stephenc.findbugs</groupId> - <artifactId>findbugs-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>javax.servlet</groupId> - <artifactId>servlet-api</artifactId> - </exclusion> - <exclusion> - <groupId>google-collections</groupId> - <artifactId>google-collections</artifactId> - </exclusion> - <exclusion> - <groupId>net.sourceforge.collections</groupId> - <artifactId>collections-generic</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-server</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-json</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-api-2.1</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-all</artifactId> - </exclusion> - <exclusion> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </exclusion> <exclusion> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> @@ -245,10 +138,6 @@ <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-xc</artifactId> </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>servlet-api-2.5</artifactId> - </exclusion> </exclusions> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java index abe37d1..067b59f 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java @@ -18,33 +18,117 @@ */ package org.apache.metamodel.hbase; -public final class HBaseColumn { - private String columnFamily; - private String qualifier; +import org.apache.metamodel.schema.AbstractColumn; +import org.apache.metamodel.schema.ColumnType; +import org.apache.metamodel.schema.ColumnTypeImpl; +import org.apache.metamodel.schema.SuperColumnType; +import org.apache.metamodel.schema.Table; - public HBaseColumn() { - columnFamily = ""; - qualifier = ""; +public final class HBaseColumn extends AbstractColumn { + private final String columnFamily; + private final String qualifier; + private final Table table; + private final boolean primaryKey; + private final ColumnType columnType; + private final int columnNumber; + + public HBaseColumn(String columnFamily, Table table) { + this(columnFamily, null, table, -1); + } + + public HBaseColumn(String columnFamily, String qualifier, Table table) { + this(columnFamily, qualifier, table, -1); + } + + public HBaseColumn(String columnFamily, Table table, int columnNumber) { + this(columnFamily, null, table, columnNumber); } - public HBaseColumn(String columnFamily, String qualifier) { + public HBaseColumn(String columnFamily, String qualifier, Table table, int columnNumber) { + if (columnFamily == null) { + throw new IllegalArgumentException("Column family isn't allowed to be null."); + } else if (table == null) { + throw new IllegalArgumentException("Table isn't allowed to be null."); + } + this.columnFamily = columnFamily; this.qualifier = qualifier; + this.table = table; + this.columnNumber = columnNumber; + + primaryKey = HBaseDataContext.FIELD_ID.equals(columnFamily); + + if (primaryKey || qualifier != null) { + columnType = new ColumnTypeImpl("BYTE[]", SuperColumnType.LITERAL_TYPE); + } else { + columnType = ColumnType.LIST; + } } public String getColumnFamily() { return columnFamily; } - public void setColumnFamily(String columnFamily) { - this.columnFamily = columnFamily; - } - public String getQualifier() { return qualifier; } - public void setQualifier(String qualifier) { - this.qualifier = qualifier; + @Override + public String getName() { + if (qualifier == null) { + return columnFamily; + } + return columnFamily + ":" + qualifier; + } + + @Override + public int getColumnNumber() { + return columnNumber; + } + + @Override + public ColumnType getType() { + return columnType; + } + + @Override + public Table getTable() { + return table; + } + + @Override + public Boolean isNullable() { + return !primaryKey; + } + + @Override + public String getRemarks() { + return null; + } + + @Override + public Integer getColumnSize() { + return null; + } + + @Override + public String getNativeType() { + // TODO: maybe change if no qualifier is present (and not identifier column). + return "byte[]"; + } + + @Override + public boolean isIndexed() { + return false; + } + + @Override + public boolean isPrimaryKey() { + return primaryKey; + } + + @Override + public String getQuote() { + return null; } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java index 358d743..d77cd04 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java @@ -26,8 +26,8 @@ public class HBaseRowInsertionBuilder extends AbstractRowInsertionBuilder<HBaseU private final HBaseColumn[] _outputColumns; - public HBaseRowInsertionBuilder(HBaseUpdateCallback updateCallback, Table table, HBaseColumn[] outputColumns) { - super(updateCallback, table, outputColumns.length); + public HBaseRowInsertionBuilder(HBaseUpdateCallback updateCallback, HBaseTable table, HBaseColumn[] outputColumns) { + super(updateCallback, table.setColumns(outputColumns)); _outputColumns = outputColumns; } @@ -44,7 +44,7 @@ public class HBaseRowInsertionBuilder extends AbstractRowInsertionBuilder<HBaseU int indexOfTablesColumn = 0; while (!matchingColumnFound && indexOfTablesColumn < table.getColumnCount()) { - if (outputColumns[i].getColumnFamily().equals(table.getColumn(indexOfTablesColumn).getName())) { + if (outputColumns[i].equals(table.getColumn(indexOfTablesColumn))) { matchingColumnFound = true; } else { indexOfTablesColumn++; http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java index 271aa87..6cc1522 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java @@ -82,22 +82,12 @@ public class HBaseUpdateCallback extends AbstractUpdateCallback implements Updat throw new UnsupportedOperationException("Use insertInto(String tableName, HBaseColumn[] outputColumns)"); } - public HBaseRowInsertionBuilder insertInto(String tableName, HBaseColumn[] outputColumns) - throws IllegalArgumentException, IllegalStateException, UnsupportedOperationException { - Table table = getTable(tableName); - return insertInto(table, outputColumns); - } - - public HBaseRowInsertionBuilder insertInto(Table table, HBaseColumn[] outputColumns) - throws IllegalArgumentException, IllegalStateException, UnsupportedOperationException { - validateTable(table); - return new HBaseRowInsertionBuilder(this, table, outputColumns); - } - - private void validateTable(Table table) { - if (!(table instanceof HBaseTable)) { - throw new IllegalArgumentException("Not a valid HBase table: " + table); + public HBaseRowInsertionBuilder insertInto(Table table, HBaseColumn[] columns) + throws IllegalArgumentException { + if (table instanceof HBaseTable) { + return new HBaseRowInsertionBuilder(this, (HBaseTable) table, columns); } + throw new IllegalArgumentException("Not an HBase table: " + table); } protected synchronized void writeRow(HBaseTable hBaseTable, HBaseColumn[] outputColumns, Object[] values) { @@ -120,10 +110,6 @@ public class HBaseUpdateCallback extends AbstractUpdateCallback implements Updat throw new UnsupportedOperationException(); } - public boolean tableAlreadyExists(String tableName) { - return _dataContext.getMainSchema().getTableByName(tableName) == null ? false : true; - } - public HBaseConfiguration getConfiguration() { return _configuration; } http://git-wip-us.apache.org/repos/asf/metamodel/blob/2008c7eb/hbase/src/main/java/org/apache/metamodel/hbase/qualifiers/FindQualifiersDriver.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/qualifiers/FindQualifiersDriver.java b/hbase/src/main/java/org/apache/metamodel/hbase/qualifiers/FindQualifiersDriver.java deleted file mode 100644 index 9292d06..0000000 --- a/hbase/src/main/java/org/apache/metamodel/hbase/qualifiers/FindQualifiersDriver.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.metamodel.hbase.qualifiers; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.KeyOnlyFilter; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; -import org.apache.hadoop.hbase.mapreduce.TableMapper; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.metamodel.hbase.HBaseConfiguration; - -public class FindQualifiersDriver extends Configured implements Tool { - - static class OnlyColumnNameMapper extends TableMapper<Text, Text> { - @Override - protected void map(ImmutableBytesWritable key, Result value, final Context context) throws IOException, - InterruptedException { - CellScanner cellScanner = value.cellScanner(); - while (cellScanner.advance()) { - - Cell cell = cellScanner.current(); - byte[] q = Bytes.copy(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); - - context.write(new Text(q), new Text()); - } - } - } - - static class OnlyColumnNameReducer extends Reducer<Text, Text, Text, Text> { - - @Override - protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, - InterruptedException { - context.write(new Text(key), new Text()); - } - } - - @Override - public int run(String[] args) throws Exception { - Path outputPath = new Path("output/"); - byte[] tableName = new String("ietsanders2").getBytes(); - byte[] columnFamilyName = new String("data").getBytes(); - - Configuration configuration = createConfig(); - FileSystem fileSystem = FileSystem.get(configuration); - fileSystem.delete(outputPath, true); - - Job job = Job.getInstance(configuration, "Distinct_columns"); - job.setJarByClass(this.getClass()); - - Scan scan = new Scan(); - scan.setBatch(500); - scan.addFamily(columnFamilyName); - scan.setFilter(new KeyOnlyFilter()); // scan only key part of KeyValue (raw, column family, column) - scan.setCacheBlocks(false); // don't set to true for MR jobs - - TextOutputFormat.setOutputPath(job, outputPath); - - TableMapReduceUtil.initTableMapperJob(tableName, scan, OnlyColumnNameMapper.class, // mapper - Text.class, // mapper output key - Text.class, // mapper output value - job); - - job.setNumReduceTasks(1); - job.setReducerClass(OnlyColumnNameReducer.class); - - return job.waitForCompletion(true) ? 0 : 1; - } - - protected Configuration createConfig() { - Configuration config = org.apache.hadoop.hbase.HBaseConfiguration.create(); - config.set("hbase.zookeeper.quorum", "bigdatavm"); - config.set("hbase.zookeeper.property.clientPort", Integer.toString(HBaseConfiguration.DEFAULT_ZOOKEEPER_PORT)); - config.set("hbase.client.retries.number", Integer.toString(HBaseConfiguration.DEFAULT_HBASE_CLIENT_RETRIES)); - config.set("zookeeper.session.timeout", Integer.toString(HBaseConfiguration.DEFAULT_ZOOKEEPER_SESSION_TIMEOUT)); - config.set("zookeeper.recovery.retry", Integer.toString(HBaseConfiguration.DEFAULT_ZOOKEEPER_RECOVERY_RETRIES)); - return config; - } - - public static void main(String[] args) throws Exception { - int exitCode = ToolRunner.run(new FindQualifiersDriver(), args); - System.exit(exitCode); - } -}
