HBase improvements: refactoring, new unittests
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/3ada7eb4 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/3ada7eb4 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/3ada7eb4 Branch: refs/heads/master Commit: 3ada7eb48e0599b507024b1b6c10be3b7cceb63f Parents: 38d8bc3 Author: Gerard Dellemann <[email protected]> Authored: Thu May 24 16:12:20 2018 +0200 Committer: Gerard Dellemann <[email protected]> Committed: Thu May 24 16:12:20 2018 +0200 ---------------------------------------------------------------------- .../org/apache/metamodel/hbase/HBaseClient.java | 164 ++++++++++++++ .../org/apache/metamodel/hbase/HBaseColumn.java | 28 ++- .../hbase/HBaseCreateTableBuilder.java | 94 ++++++-- .../hbase/HBaseRowDeletionBuilder.java | 24 ++- .../hbase/HBaseRowInsertionBuilder.java | 89 ++++---- .../org/apache/metamodel/hbase/HBaseTable.java | 90 ++++++-- .../metamodel/hbase/HBaseTableDropBuilder.java | 25 ++- .../metamodel/hbase/HBaseUpdateCallback.java | 99 +++------ .../org/apache/metamodel/hbase/HBaseWriter.java | 147 ------------- .../metamodel/hbase/HBaseDataContextTest.java | 98 +++------ .../apache/metamodel/hbase/HBaseTestCase.java | 58 ++++- .../hbase/HBaseUpdateCallbackTest.java | 213 +++++++++++++++++++ 12 files changed, 742 insertions(+), 387 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java new file mode 100644 index 0000000..1e957fc --- /dev/null +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.hbase; + +import java.io.IOException; +import java.util.Set; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.metamodel.MetaModelException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class can perform client-operations on a HBase datastore + */ +public final class HBaseClient { + + private static final Logger logger = LoggerFactory.getLogger(HBaseClient.class); + + private final Connection _connection; + + public HBaseClient(Connection connection) { + this._connection = connection; + } + + /** + * Write a single row of values to a HBase table + * @param hBaseTable + * @param columns + * @param values + * @throws IOException + */ + public void writeRow(HBaseTable hBaseTable, HBaseColumn[] columns, Object[] values) throws IOException { + try (final Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName()))) { + int indexOfIdColumn = getIndexOfIdColumn(columns); + + // Create a put with the values of indexOfIdColumn as rowkey + final Put put = new Put(Bytes.toBytes(values[indexOfIdColumn].toString())); + + // Add the other values to the put + for (int i = 0; i < columns.length; i++) { + if (i != indexOfIdColumn) { + put.addColumn(Bytes.toBytes(columns[i].getColumnFamily()), Bytes.toBytes(columns[i].getQualifier()), + Bytes.toBytes(values[i].toString())); + } + } + // Add the put to the table + table.put(put); + } + } + + /** + * Gets the index of the ID-column + * Throws an {@link MetaModelException} when no ID-column is found. + * @param columns + * @return + */ + private int getIndexOfIdColumn(HBaseColumn[] columns) { + int indexOfIdColumn = 0; + boolean idColumnFound = false; + while (!idColumnFound && indexOfIdColumn < columns.length) { + if (columns[indexOfIdColumn].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) { + idColumnFound = true; + } else { + indexOfIdColumn++; + } + } + if (!idColumnFound) { + throw new MetaModelException("The ID Column family was not found"); + } + return indexOfIdColumn; + } + + /** + * Delete 1 row based on the key + * @param hBaseTable + * @param key + * @throws IOException + */ + public void deleteRow(HBaseTable hBaseTable, Object key) throws IOException { + try (final Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName()));) { + if (rowExists(table, key) == true) { + table.delete(new Delete(Bytes.toBytes(key.toString()))); + } else { + logger.warn("Rowkey with value " + key.toString() + " doesn't exist in the table"); + } + } + } + + /** + * Checks in the HBase datastore if a row exists based on the key + * @param table + * @param key + * @return boolean + * @throws IOException + */ + private boolean rowExists(Table table, Object key) throws IOException { + final Get get = new Get(Bytes.toBytes(key.toString())); + return !table.get(get).isEmpty(); + } + + /** + * Creates a HBase table based on a tableName and it's columnFamilies + * @param tableName + * @param columnFamilies + * @throws IOException + */ + public void createTable(String tableName, Set<String> columnFamilies) throws IOException { + try (final Admin admin = _connection.getAdmin()) { + final TableName hBasetableName = TableName.valueOf(tableName); + final HTableDescriptor tableDescriptor = new HTableDescriptor(hBasetableName); + // Add all columnFamilies to the tableDescriptor. + for (final String columnFamilie : columnFamilies) { + // The ID-column isn't needed because, it will automatically be created. + if (!columnFamilie.equals(HBaseDataContext.FIELD_ID)) { + tableDescriptor.addFamily(new HColumnDescriptor(columnFamilie)); + } + } + admin.createTable(tableDescriptor); + final HTableDescriptor[] tables = admin.listTables(); + if (tables.length != 1 && Bytes.equals(hBasetableName.getName(), tables[0].getTableName().getName())) { + throw new IOException("Failed create of table"); + } + } + } + + /** + * Disable and drop a table from a HBase datastore + * @param tableName + * @throws IOException + */ + public void dropTable(String tableName) throws IOException { + try (final Admin admin = _connection.getAdmin()) { + final TableName hBasetableName = TableName.valueOf(tableName); + admin.disableTable(hBasetableName); // A table must be disabled first, before it can be deleted + admin.deleteTable(hBasetableName); + } + } +} http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java index 067b59f..bd57c1c 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java @@ -18,6 +18,9 @@ */ package org.apache.metamodel.hbase; +import java.util.LinkedHashSet; +import java.util.Set; + import org.apache.metamodel.schema.AbstractColumn; import org.apache.metamodel.schema.ColumnType; import org.apache.metamodel.schema.ColumnTypeImpl; @@ -32,32 +35,32 @@ public final class HBaseColumn extends AbstractColumn { private final ColumnType columnType; private final int columnNumber; - public HBaseColumn(String columnFamily, Table table) { + public HBaseColumn(final String columnFamily, final Table table) { this(columnFamily, null, table, -1); } - public HBaseColumn(String columnFamily, String qualifier, Table table) { + public HBaseColumn(final String columnFamily, final String qualifier, final Table table) { this(columnFamily, qualifier, table, -1); } - public HBaseColumn(String columnFamily, Table table, int columnNumber) { + public HBaseColumn(final String columnFamily, final Table table, final int columnNumber) { this(columnFamily, null, table, columnNumber); } - public HBaseColumn(String columnFamily, String qualifier, Table table, int columnNumber) { + public HBaseColumn(final String columnFamily, final String qualifier, final Table table, final int columnNumber) { if (columnFamily == null) { throw new IllegalArgumentException("Column family isn't allowed to be null."); } else if (table == null) { throw new IllegalArgumentException("Table isn't allowed to be null."); } - + this.columnFamily = columnFamily; this.qualifier = qualifier; this.table = table; this.columnNumber = columnNumber; primaryKey = HBaseDataContext.FIELD_ID.equals(columnFamily); - + if (primaryKey || qualifier != null) { columnType = new ColumnTypeImpl("BYTE[]", SuperColumnType.LITERAL_TYPE); } else { @@ -131,4 +134,17 @@ public final class HBaseColumn extends AbstractColumn { public String getQuote() { return null; } + + /** + * Creates a set of columnFamilies out of an array of hbaseColumns + * @param hbaseColumns + * @return {@link LinkedHashSet} + */ + public static Set<String> getColumnFamilies(HBaseColumn[] hbaseColumns) { + final LinkedHashSet<String> columnFamilies = new LinkedHashSet<String>(); + for (int i = 0; i < hbaseColumns.length; i++) { + columnFamilies.add(hbaseColumns[i].getColumnFamily()); + } + return columnFamilies; + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java index e1f1257..ffdc20a 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java @@ -19,53 +19,103 @@ package org.apache.metamodel.hbase; import java.io.IOException; -import java.util.LinkedHashSet; +import java.util.Iterator; import java.util.Set; import org.apache.metamodel.MetaModelException; import org.apache.metamodel.create.AbstractTableCreationBuilder; import org.apache.metamodel.schema.MutableSchema; -import org.apache.metamodel.schema.MutableTable; import org.apache.metamodel.schema.Schema; import org.apache.metamodel.schema.Table; import org.apache.metamodel.util.SimpleTableDef; +/** + * A builder-class to create tables in a HBase datastore + */ public class HBaseCreateTableBuilder extends AbstractTableCreationBuilder<HBaseUpdateCallback> { - private Set<String> columnFamilies; + private Set<String> _columnFamilies; + + public HBaseCreateTableBuilder(HBaseUpdateCallback updateCallback, Schema schema, String name) { + this(updateCallback, schema, name, null); + } + /** + * Create a {@link HBaseCreateTableBuilder}. + * Throws an {@link IllegalArgumentException} if the schema isn't a {@link MutableSchema}. + * @param updateCallback + * @param schema + * @param name + * @param columnFamilies + */ public HBaseCreateTableBuilder(HBaseUpdateCallback updateCallback, Schema schema, String name, - HBaseColumn[] outputColumns) { + Set<String> columnFamilies) { super(updateCallback, schema, name); if (!(schema instanceof MutableSchema)) { - throw new IllegalArgumentException("Not a valid schema: " + schema); - } - columnFamilies = new LinkedHashSet<String>(); - for (int i = 0; i < outputColumns.length; i++) { - columnFamilies.add(outputColumns[i].getColumnFamily()); + throw new IllegalArgumentException("Not a mutable schema: " + schema); } + this._columnFamilies = columnFamilies; } @Override - public Table execute() throws MetaModelException { - final MutableTable table = getTable(); - final SimpleTableDef emptyTableDef = new SimpleTableDef(table.getName(), columnFamilies.toArray( - new String[columnFamilies.size()])); + public Table execute() { + checkColumnFamilies(_columnFamilies); - final HBaseUpdateCallback updateCallback = (HBaseUpdateCallback) getUpdateCallback(); + final Table table = getTable(); + // Add the table to the datastore try { - final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(updateCallback - .getConfiguration())); - HbaseWriter.createTable(table.getName(), columnFamilies); + final HBaseClient hBaseClient = getUpdateCallback().getHBaseClient(); + hBaseClient.createTable(table.getName(), _columnFamilies); } catch (IOException e) { - e.printStackTrace(); + throw new MetaModelException(e); } - final MutableSchema schema = (MutableSchema) table.getSchema(); - schema.addTable(new HBaseTable(updateCallback.getDataContext(), emptyTableDef, schema, - HBaseConfiguration.DEFAULT_ROW_KEY_TYPE)); - return schema.getTableByName(table.getName()); + // Update the schema + addNewTableToSchema(table); + return getSchema().getTableByName(table.getName()); } + /** + * Check if the new table has columnFamilies and if the ID-column is included. + * Throws a {@link MetaModelException} if a check fails. + * @param columnFamilies + */ + private void checkColumnFamilies(Set<String> columnFamilies) { + if (columnFamilies == null || columnFamilies.size() == 0) { + throw new MetaModelException("Creating a table without columnFamilies"); + } + boolean idColumnFound = false; + final Iterator<String> iterator = columnFamilies.iterator(); + while (!idColumnFound && iterator.hasNext()) { + if (iterator.next().equals(HBaseDataContext.FIELD_ID)) { + idColumnFound = true; + } + } + if (!idColumnFound) { + throw new MetaModelException("ColumnFamily: " + HBaseDataContext.FIELD_ID + " not found"); + } + } + + /** + * Set the columnFamilies + * @param columnFamilies + */ + public void setColumnFamilies(Set<String> columnFamilies) { + this._columnFamilies = columnFamilies; + } + + /** + * Add the new {@link Table} to the {@link MutableSchema} + * @param table + * @param updateCallback + * @return {@link MutableSchema} + */ + private void addNewTableToSchema(final Table table) { + final MutableSchema schema = (MutableSchema) getSchema(); + final SimpleTableDef emptyTableDef = new SimpleTableDef(table.getName(), _columnFamilies.toArray( + new String[_columnFamilies.size()])); + schema.addTable(new HBaseTable((HBaseDataContext) getUpdateCallback().getDataContext(), emptyTableDef, schema, + HBaseConfiguration.DEFAULT_ROW_KEY_TYPE)); + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java index 925537a..3ba4ef8 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java @@ -18,30 +18,38 @@ */ package org.apache.metamodel.hbase; +import java.io.IOException; + import org.apache.metamodel.MetaModelException; import org.apache.metamodel.delete.AbstractRowDeletionBuilder; import org.apache.metamodel.schema.Table; +/** + * A builder-class to delete rows in a HBase datastore + */ public class HBaseRowDeletionBuilder extends AbstractRowDeletionBuilder { - private HBaseUpdateCallback _updateCallback; + private HBaseClient _hBaseClient; private Object _key; - public HBaseRowDeletionBuilder(HBaseUpdateCallback updateCallback, Table table) { + public HBaseRowDeletionBuilder(final HBaseClient hBaseWriter, final Table table) { super(table); - if (updateCallback == null) { + if (hBaseWriter == null) { throw new IllegalArgumentException("UpdateCallback cannot be null"); } - _updateCallback = updateCallback; - + this._hBaseClient = hBaseWriter; } @Override - public void execute() throws MetaModelException { + public synchronized void execute() { if (_key == null) { - throw new IllegalArgumentException("Key cannot be null"); + throw new MetaModelException("Key cannot be null"); + } + try { + _hBaseClient.deleteRow((HBaseTable) getTable(), _key); + } catch (IOException e) { + throw new MetaModelException(e); } - _updateCallback.deleteRow((HBaseTable) getTable(), _key); } public void setKey(Object _key) { http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java index d77cd04..1c633f7 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java @@ -18,59 +18,70 @@ */ package org.apache.metamodel.hbase; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Set; + import org.apache.metamodel.MetaModelException; import org.apache.metamodel.insert.AbstractRowInsertionBuilder; -import org.apache.metamodel.schema.Table; +/** + * A builder-class to insert rows in a HBase datastore + */ public class HBaseRowInsertionBuilder extends AbstractRowInsertionBuilder<HBaseUpdateCallback> { - private final HBaseColumn[] _outputColumns; + private HBaseColumn[] _hbaseColumns; - public HBaseRowInsertionBuilder(HBaseUpdateCallback updateCallback, HBaseTable table, HBaseColumn[] outputColumns) { - super(updateCallback, table.setColumns(outputColumns)); - _outputColumns = outputColumns; - } - - @Override - public void execute() throws MetaModelException { - checkForMatchingColumnFamilies(getTable(), _outputColumns); - getUpdateCallback().writeRow((HBaseTable) getTable(), _outputColumns, getValues()); + public HBaseRowInsertionBuilder(final HBaseUpdateCallback updateCallback, final HBaseTable table) { + super(updateCallback, table); + checkTable(updateCallback, table); } - private void checkForMatchingColumnFamilies(Table table, HBaseColumn[] outputColumns) { - for (int i = 0; i < outputColumns.length; i++) { - if (!outputColumns[i].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) { - boolean matchingColumnFound = false; - int indexOfTablesColumn = 0; - - while (!matchingColumnFound && indexOfTablesColumn < table.getColumnCount()) { - if (outputColumns[i].equals(table.getColumn(indexOfTablesColumn))) { - matchingColumnFound = true; - } else { - indexOfTablesColumn++; - } - } - - if (!matchingColumnFound) { - throw new IllegalArgumentException(String.format( - "OutputColumnFamily: %s doesn't exist in the schema of the table", outputColumns[i] - .getColumnFamily())); - } - } + /** + * Check if the table exits and it's columnFamilies exist + * If the table doesn't exist, then a {@link MetaModelException} is thrown + * @param updateCallback + * @param tableGettingInserts + */ + private void checkTable(final HBaseUpdateCallback updateCallback, final HBaseTable tableGettingInserts) { + final HBaseTable tableInSchema = (HBaseTable) updateCallback.getDataContext().getDefaultSchema().getTableByName( + tableGettingInserts.getName()); + if (tableInSchema == null) { + throw new MetaModelException("Trying to insert data into table: " + tableGettingInserts.getName() + + ", which doesn't exist yet"); } + tableInSchema.checkForNotMatchingColumns(tableGettingInserts.getColumnNames()); } - public HBaseColumn[] getOutputColumns() { - return _outputColumns; + /** + * Set the hbaseColumns. Checks if the columnFamilies exist in the table. + * @param hbaseColumns a {@link IllegalArgumentException} is thrown when this parameter is null or empty + */ + public void setHbaseColumns(HBaseColumn[] hbaseColumns) { + if (hbaseColumns == null || hbaseColumns.length == 0) { + throw new IllegalArgumentException("List of hbaseColumns is null or empty"); + } + final Set<String> columnFamilies = HBaseColumn.getColumnFamilies(hbaseColumns); + final HBaseTable tableInSchema = (HBaseTable) getTable(); + final ArrayList<String> columnFamiliesAsList = new ArrayList<String>(); + columnFamiliesAsList.addAll(columnFamilies); + tableInSchema.checkForNotMatchingColumns(columnFamiliesAsList); + this._hbaseColumns = hbaseColumns; } - public void setOutputColumns(HBaseColumn[] outputColumns) { - if (outputColumns.length != _outputColumns.length) { - throw new IllegalArgumentException("The amount of outputColumns don't match"); + @Override + public synchronized void execute() { + if (_hbaseColumns == null || _hbaseColumns.length == 0) { + throw new MetaModelException("The hbaseColumns-array is null or empty"); + } + if (getValues() == null || getValues().length == 0) { + throw new MetaModelException("The values-array is null or empty"); } - for (int i = 0; i < outputColumns.length; i++) { - _outputColumns[i] = outputColumns[i]; + try { + final HBaseClient hBaseClient = getUpdateCallback().getHBaseClient(); + hBaseClient.writeRow((HBaseTable) getTable(), _hbaseColumns, getValues()); + } catch (IOException e) { + throw new MetaModelException(e); } } - } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java index def7c9c..fee2f5d 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java @@ -43,49 +43,69 @@ final class HBaseTable extends MutableTable { private final transient HBaseDataContext _dataContext; private final transient ColumnType _defaultRowKeyColumnType; + /** + * Creates an HBaseTable. If the tableDef variable doesn't include the ID column (see {@link HBaseDataContext#FIELD_ID}). + * Then it's first inserted. + * @param dataContext + * @param tableDef Table definition. The tableName, columnNames and columnTypes variables are used. + * @param schema {@link MutableSchema} where the table belongs to. + * @param defaultRowKeyColumnType This variable determines the {@link ColumnType}, + * used when the tableDef doesn't include the ID column (see {@link HBaseDataContext#FIELD_ID}). + */ public HBaseTable(HBaseDataContext dataContext, SimpleTableDef tableDef, MutableSchema schema, ColumnType defaultRowKeyColumnType) { super(tableDef.getName(), TableType.TABLE, schema); _dataContext = dataContext; _defaultRowKeyColumnType = defaultRowKeyColumnType; + // Add the columns final String[] columnNames = tableDef.getColumnNames(); if (columnNames == null || columnNames.length == 0) { logger.info("No user-defined columns specified for table {}. Columns will be auto-detected."); } else { - - final ColumnType[] types = tableDef.getColumnTypes(); - int columnNumber = 1; - - for (int i = 0; i < columnNames.length; i++) { - String columnName = columnNames[i]; - if (HBaseDataContext.FIELD_ID.equals(columnName)) { - final ColumnType type = types[i]; - final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, type) - .setPrimaryKey(true).setColumnNumber(columnNumber).setTable(this); - addColumn(idColumn); - columnNumber++; + final ColumnType[] columnTypes = tableDef.getColumnTypes(); + + // Find the ID-Column + boolean idColumnFound = false; + int indexOfIDColumn = 0; + while (!idColumnFound && indexOfIDColumn < columnNames.length) { + if (columnNames[indexOfIDColumn].equals(HBaseDataContext.FIELD_ID)) { + idColumnFound = true; + } else { + indexOfIDColumn++; } } - if (columnNumber == 1) { - // insert a default definition of the id column - final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, defaultRowKeyColumnType) - .setPrimaryKey(true).setColumnNumber(columnNumber).setTable(this); - addColumn(idColumn); - columnNumber++; + int columnNumber = indexOfIDColumn + 1; // ColumnNumbers start from 1 + + // Add the ID-Column, even if the column wasn't included in columnNames + ColumnType columnType; + if (idColumnFound) { + columnType = columnTypes[indexOfIDColumn]; + } else { + columnType = defaultRowKeyColumnType; } + final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, columnType) + .setPrimaryKey(true) + .setColumnNumber(columnNumber) + .setTable(this); + addColumn(idColumn); + // Add the other columns for (int i = 0; i < columnNames.length; i++) { final String columnName = columnNames[i]; - + if (idColumnFound) { + columnNumber = i + 1; // ColumnNumbers start from 1 + } else { + columnNumber = i + 2; // ColumnNumbers start from 1 + the ID-column has just been created + } if (!HBaseDataContext.FIELD_ID.equals(columnName)) { - final ColumnType type = types[i]; + final ColumnType type = columnTypes[i]; final MutableColumn column = new MutableColumn(columnName, type); column.setTable(this); column.setColumnNumber(columnNumber); - columnNumber++; addColumn(column); + columnNumber++; } } } @@ -100,7 +120,9 @@ final class HBaseTable extends MutableTable { int columnNumber = 1; final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, _defaultRowKeyColumnType) - .setPrimaryKey(true).setColumnNumber(columnNumber).setTable(this); + .setPrimaryKey(true) + .setColumnNumber(columnNumber) + .setTable(this); addColumn(idColumn); columnNumber++; @@ -124,4 +146,28 @@ final class HBaseTable extends MutableTable { } return columnsInternal; } + + /** + * Check if a list of columnNames all exist in this table + * If a column doesn't exist, then a {@link MetaModelException} is thrown + * @param columnNamesOfCheckedTable + */ + public void checkForNotMatchingColumns(final List<String> columnNamesOfCheckedTable) { + final List<String> columnsNamesOfExistingTable = getColumnNames(); + for (String columnNameOfCheckedTable : columnNamesOfCheckedTable) { + boolean matchingColumnFound = false; + int i = 0; + while (!matchingColumnFound && i < columnsNamesOfExistingTable.size()) { + if (columnNameOfCheckedTable.equals(columnsNamesOfExistingTable.get(i))) { + matchingColumnFound = true; + } else { + i++; + } + } + if (!matchingColumnFound) { + throw new MetaModelException(String.format("ColumnFamily: %s doesn't exist in the schema of the table", + columnNameOfCheckedTable)); + } + } + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java index 5ab38d9..374e325 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java @@ -18,20 +18,39 @@ */ package org.apache.metamodel.hbase; +import java.io.IOException; + import org.apache.metamodel.MetaModelException; import org.apache.metamodel.drop.AbstractTableDropBuilder; +import org.apache.metamodel.schema.MutableSchema; import org.apache.metamodel.schema.Table; +/** + * A builder-class to drop tables in a HBase datastore + */ public class HBaseTableDropBuilder extends AbstractTableDropBuilder { private final HBaseUpdateCallback _updateCallback; - public HBaseTableDropBuilder(Table table, HBaseUpdateCallback updateCallback) { + public HBaseTableDropBuilder(final Table table, final HBaseUpdateCallback updateCallback) { super(table); + if (updateCallback.getDataContext().getDefaultSchema().getTableByName(table.getName()) == null) { + throw new MetaModelException("Trying to delete a table that doesn't exist in the datastore."); + } _updateCallback = updateCallback; } @Override - public void execute() throws MetaModelException { - _updateCallback.dropTableExecute(getTable()); + public void execute() { + try { + // Remove from the datastore + final HBaseClient hBaseClient = _updateCallback.getHBaseClient(); + final Table table = getTable(); + hBaseClient.dropTable(table.getName()); + + // Remove from schema + ((MutableSchema) table.getSchema()).removeTable(table); + } catch (IOException e) { + throw new MetaModelException(e); + } } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java index 39670db..e548229 100644 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java +++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java @@ -18,7 +18,7 @@ */ package org.apache.metamodel.hbase; -import java.io.IOException; +import java.util.Set; import org.apache.metamodel.AbstractUpdateCallback; import org.apache.metamodel.UpdateCallback; @@ -26,32 +26,35 @@ import org.apache.metamodel.create.TableCreationBuilder; import org.apache.metamodel.delete.RowDeletionBuilder; import org.apache.metamodel.drop.TableDropBuilder; import org.apache.metamodel.insert.RowInsertionBuilder; -import org.apache.metamodel.schema.MutableSchema; import org.apache.metamodel.schema.Schema; import org.apache.metamodel.schema.Table; +/** + * This class is used to build objects to do client-operations on a HBase datastore + */ public class HBaseUpdateCallback extends AbstractUpdateCallback implements UpdateCallback { - private final HBaseConfiguration _configuration; - - private final HBaseDataContext _dataContext; + private final HBaseClient _hBaseClient; public HBaseUpdateCallback(HBaseDataContext dataContext) { super(dataContext); - _configuration = dataContext.getConfiguration(); - _dataContext = dataContext; + _hBaseClient = new HBaseClient(dataContext.getConnection()); } @Override - public TableCreationBuilder createTable(Schema schema, String name) throws IllegalArgumentException, - IllegalStateException { - throw new UnsupportedOperationException( - "Use createTable(Schema schema, String name, HBaseColumn[] outputColumns)"); + public TableCreationBuilder createTable(Schema schema, String name) { + return new HBaseCreateTableBuilder(this, schema, name); } - public HBaseCreateTableBuilder createTable(Schema schema, String name, HBaseColumn[] outputColumns) - throws IllegalArgumentException, IllegalStateException { - return new HBaseCreateTableBuilder(this, schema, name, outputColumns); + /** + * Initiates the building of a table creation operation. + * @param schema the schema to create the table in + * @param name the name of the new table + * @param columnFamilies the columnFamilies of the new table + * @return {@link HBaseCreateTableBuilder} + */ + public HBaseCreateTableBuilder createTable(Schema schema, String name, Set<String> columnFamilies) { + return new HBaseCreateTableBuilder(this, schema, name, columnFamilies); } @Override @@ -60,78 +63,34 @@ public class HBaseUpdateCallback extends AbstractUpdateCallback implements Updat } @Override - public TableDropBuilder dropTable(Table table) throws IllegalArgumentException, IllegalStateException, - UnsupportedOperationException { + public TableDropBuilder dropTable(Table table) { return new HBaseTableDropBuilder(table, this); } - public void dropTableExecute(Table table) { - try { - final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(_configuration)); - HbaseWriter.dropTable(table.getName()); - MutableSchema schema = (MutableSchema) table.getSchema(); - schema.removeTable(table); - } catch (IOException e) { - e.printStackTrace(); - } - } - @Override - public RowInsertionBuilder insertInto(Table table) throws IllegalArgumentException, IllegalStateException, - UnsupportedOperationException { - throw new UnsupportedOperationException("Use insertInto(Table table, HBaseColumn[] outputColumns)"); - } - - public HBaseRowInsertionBuilder insertInto(Table table, HBaseColumn[] columns) throws IllegalArgumentException { + public RowInsertionBuilder insertInto(Table table) { if (table instanceof HBaseTable) { - return new HBaseRowInsertionBuilder(this, (HBaseTable) table, columns); - } - throw new IllegalArgumentException("Not an HBase table: " + table); - } - - protected synchronized void writeRow(HBaseTable hBaseTable, HBaseColumn[] outputColumns, Object[] values) { - try { - final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(_configuration)); - HbaseWriter.writeRow(hBaseTable, outputColumns, values); - } catch (IOException e) { - e.printStackTrace(); + return new HBaseRowInsertionBuilder(this, (HBaseTable) table); + } else { + throw new IllegalArgumentException("Not an HBase table: " + table); } } @Override public boolean isDeleteSupported() { - return false; + return true; } @Override - public RowDeletionBuilder deleteFrom(Table table) throws IllegalArgumentException, IllegalStateException, - UnsupportedOperationException { - throw new UnsupportedOperationException( - "Use deleteFrom(HBaseUpdateCallback updateCallback, Table table, Object key)"); - } - - public HBaseRowDeletionBuilder deleteFrom(HBaseUpdateCallback updateCallback, Table table) - throws IllegalArgumentException { + public RowDeletionBuilder deleteFrom(Table table) { if (table instanceof HBaseTable) { - return new HBaseRowDeletionBuilder(this, (HBaseTable) table); + return new HBaseRowDeletionBuilder(_hBaseClient, (HBaseTable) table); + } else { + throw new IllegalArgumentException("Not an HBase table: " + table); } - throw new IllegalArgumentException("Not an HBase table: " + table); - } - - protected synchronized void deleteRow(HBaseTable hBaseTable, Object key) { - try { - final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(_configuration)); - HbaseWriter.deleteRow(hBaseTable, key); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public HBaseConfiguration getConfiguration() { - return _configuration; } - public HBaseDataContext getDataContext() { - return _dataContext; + public HBaseClient getHBaseClient() { + return _hBaseClient; } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java ---------------------------------------------------------------------- diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java deleted file mode 100644 index ad63597..0000000 --- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.metamodel.hbase; - -import java.io.IOException; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.metamodel.MetaModelException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public final class HBaseWriter extends Configured { - - private static final Logger logger = LoggerFactory.getLogger(HBaseWriter.class); - - private final Connection _connection; - - public HBaseWriter(Configuration configuration) throws IOException { - _connection = ConnectionFactory.createConnection(configuration); - } - - public void writeRow(HBaseTable hBaseTable, HBaseColumn[] outputColumns, Object[] values) throws IOException { - try { - Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName())); - try { - int indexOfIdColumn = 0; - boolean idColumnFound = false; - while (!idColumnFound && indexOfIdColumn < outputColumns.length) { - if (outputColumns[indexOfIdColumn].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) { - idColumnFound = true; - } else { - indexOfIdColumn++; - } - } - if (!idColumnFound) { - throw new MetaModelException("The ID Column family was not found"); - } - - Put put = new Put(Bytes.toBytes(values[indexOfIdColumn].toString())); - - for (int i = 0; i < outputColumns.length; i++) { - if (!outputColumns[i].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) { - put.addColumn(Bytes.toBytes(outputColumns[i].getColumnFamily()), Bytes.toBytes(outputColumns[i] - .getQualifier()), Bytes.toBytes(values[i].toString())); - } - } - table.put(put); - } finally { - table.close(); - } - } finally { - _connection.close(); - } - } - - public void deleteRow(HBaseTable hBaseTable, Object key) throws IOException { - try { - Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName())); - try { - if (rowExists(table, key) == true) { - table.delete(new Delete(Bytes.toBytes(key.toString()))); - } else { - logger.warn("Rowkey with value " + key.toString() + " doesn't exist in the table"); - } - } finally { - table.close(); - } - } finally { - _connection.close(); - } - } - - private boolean rowExists(Table table, Object key) throws IOException { - Get get = new Get(Bytes.toBytes(key.toString())); - return (table.get(get).isEmpty()) == true ? false : true; - } - - public void createTable(String tableName, Set<String> columnFamilies) throws IOException { - try { - // Create table - Admin admin = _connection.getAdmin(); - try { - TableName hBasetableName = TableName.valueOf(tableName); - HTableDescriptor tableDescriptor = new HTableDescriptor(hBasetableName); - for (String columnFamilie : columnFamilies) { - if (!columnFamilie.equals(HBaseDataContext.FIELD_ID)) { - tableDescriptor.addFamily(new HColumnDescriptor(columnFamilie)); - } - } - admin.createTable(tableDescriptor); - HTableDescriptor[] tables = admin.listTables(); - if (tables.length != 1 && Bytes.equals(hBasetableName.getName(), tables[0].getTableName().getName())) { - throw new IOException("Failed create of table"); - } - } finally { - admin.close(); - } - } finally { - _connection.close(); - } - - } - - public void dropTable(String tableName) throws IOException { - try { - Admin admin = _connection.getAdmin(); - try { - TableName hBasetableName = TableName.valueOf(tableName); - admin.disableTable(hBasetableName); - admin.deleteTable(hBasetableName); - } finally { - admin.close(); - } - } finally { - _connection.close(); - } - } -} http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java ---------------------------------------------------------------------- diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java index fb21b46..3872611 100644 --- a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java +++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java @@ -18,6 +18,7 @@ */ package org.apache.metamodel.hbase; +import java.io.IOException; import java.util.Arrays; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -32,43 +33,10 @@ import org.apache.metamodel.util.SimpleTableDef; public class HBaseDataContextTest extends HBaseTestCase { - // Table - private static final String EXAMPLE_TABLE_NAME = "table_for_junit"; - - // ColumnFamilies - private static final String CF_FOO = "foo"; - private static final String CF_BAR = "bar"; - - // Qualifiers - private static final String Q_HELLO = "hello"; - private static final String Q_HI = "hi"; - private static final String Q_HEY = "hey"; - private static final String Q_BAH = "bah"; - - // RowKeys - private static final String RK_1 = "junit1"; - private static final String RK_2 = "junit2"; - - private static final int NUMBER_OF_ROWS = 2; - - // Values - private static final String V_WORLD = "world"; - private static final String V_THERE = "there"; - private static final String V_YO = "yo"; - private static final byte[] V_123_BYTE_ARRAY = new byte[] { 1, 2, 3 }; - private static final String V_YOU = "you"; - - private HBaseDataContext _dataContext; - @Override protected void setUp() throws Exception { super.setUp(); if (isConfigured()) { - final String zookeeperHostname = getZookeeperHostname(); - final int zookeeperPort = getZookeeperPort(); - final HBaseConfiguration configuration = new HBaseConfiguration(zookeeperHostname, zookeeperPort, - ColumnType.VARCHAR); - _dataContext = new HBaseDataContext(configuration); createTableNatively(); } } @@ -80,7 +48,7 @@ public class HBaseDataContextTest extends HBaseTestCase { } // test the schema exploration - final Table table = _dataContext.getDefaultSchema().getTableByName(EXAMPLE_TABLE_NAME); + final Table table = getDataContext().getDefaultSchema().getTableByName(TABLE_NAME); assertNotNull(table); assertEquals("[" + HBaseDataContext.FIELD_ID + ", " + CF_BAR + ", " + CF_FOO + "]", Arrays.toString(table @@ -92,7 +60,7 @@ public class HBaseDataContextTest extends HBaseTestCase { insertRecordsNatively(); // query using regular configuration - final DataSet dataSet1 = _dataContext.query().from(EXAMPLE_TABLE_NAME).selectAll().execute(); + final DataSet dataSet1 = getDataContext().query().from(TABLE_NAME).selectAll().execute(); try { assertTrue(dataSet1.next()); assertEquals("Row[values=[" + RK_1 + ", {" + Q_HEY + "=" + V_YO + "," + Q_HI + "=" + V_THERE + "}, {" @@ -111,14 +79,14 @@ public class HBaseDataContextTest extends HBaseTestCase { final String columnName3 = CF_BAR + ":" + Q_HEY; final String[] columnNames = new String[] { columnName1, columnName2, columnName3 }; final ColumnType[] columnTypes = new ColumnType[] { ColumnType.MAP, ColumnType.VARCHAR, ColumnType.VARCHAR }; - final SimpleTableDef[] tableDefinitions = new SimpleTableDef[] { new SimpleTableDef(EXAMPLE_TABLE_NAME, + final SimpleTableDef[] tableDefinitions = new SimpleTableDef[] { new SimpleTableDef(TABLE_NAME, columnNames, columnTypes) }; - _dataContext = new HBaseDataContext(new HBaseConfiguration("SCH", getZookeeperHostname(), getZookeeperPort(), - tableDefinitions, ColumnType.VARCHAR)); + setDataContext(new HBaseDataContext(new HBaseConfiguration("SCH", getZookeeperHostname(), getZookeeperPort(), + tableDefinitions, ColumnType.VARCHAR))); - final DataSet dataSet2 = _dataContext + final DataSet dataSet2 = getDataContext() .query() - .from(EXAMPLE_TABLE_NAME) + .from(TABLE_NAME) .select(columnName1, columnName2, columnName3) .execute(); try { @@ -134,7 +102,7 @@ public class HBaseDataContextTest extends HBaseTestCase { } // query count - final DataSet dataSet3 = _dataContext.query().from(EXAMPLE_TABLE_NAME).selectCount().execute(); + final DataSet dataSet3 = getDataContext().query().from(TABLE_NAME).selectCount().execute(); try { assertTrue(dataSet3.next()); assertEquals("Row[values=[" + NUMBER_OF_ROWS + "]]", dataSet3.getRow().toString()); @@ -144,9 +112,9 @@ public class HBaseDataContextTest extends HBaseTestCase { } // query only id - final DataSet dataSet4 = _dataContext + final DataSet dataSet4 = getDataContext() .query() - .from(EXAMPLE_TABLE_NAME) + .from(TABLE_NAME) .select(HBaseDataContext.FIELD_ID) .execute(); @@ -161,9 +129,9 @@ public class HBaseDataContextTest extends HBaseTestCase { } // primary key lookup query - using GET - final DataSet dataSet5 = _dataContext + final DataSet dataSet5 = getDataContext() .query() - .from(EXAMPLE_TABLE_NAME) + .from(TABLE_NAME) .select(HBaseDataContext.FIELD_ID) .where(HBaseDataContext.FIELD_ID) .eq(RK_1) @@ -178,9 +146,8 @@ public class HBaseDataContextTest extends HBaseTestCase { } } - private void insertRecordsNatively() throws Exception { - final org.apache.hadoop.hbase.client.Table hTable = _dataContext.getHTable(EXAMPLE_TABLE_NAME); - try { + private void insertRecordsNatively() throws IOException, InterruptedException { + try (final org.apache.hadoop.hbase.client.Table hTable = getDataContext().getHTable(TABLE_NAME)) { final Put put1 = new Put(RK_1.getBytes()); put1.addColumn(CF_FOO.getBytes(), Q_HELLO.getBytes(), V_WORLD.getBytes()); put1.addColumn(CF_BAR.getBytes(), Q_HI.getBytes(), V_THERE.getBytes()); @@ -192,27 +159,26 @@ public class HBaseDataContextTest extends HBaseTestCase { final Object[] result = new Object[NUMBER_OF_ROWS]; hTable.batch(Arrays.asList(put1, put2), result); - } finally { - hTable.close(); } } - private void createTableNatively() throws Exception { - final TableName tableName = TableName.valueOf(EXAMPLE_TABLE_NAME); - - // check if the table exists - if (_dataContext.getAdmin().isTableAvailable(tableName)) { - System.out.println("Unittest table already exists: " + EXAMPLE_TABLE_NAME); - // table already exists - return; + private void createTableNatively() throws IOException { + try (Admin admin = getDataContext().getAdmin()) { + final TableName tableName = TableName.valueOf(TABLE_NAME); + + // Check if the table exists + if (admin.isTableAvailable(tableName)) { + // table already exists + System.out.println("Unittest table already exists: " + TABLE_NAME); + } else { + // Create table + System.out.println("Creating table"); + final HTableDescriptor tableDescriptor = new HTableDescriptor(tableName); + tableDescriptor.addFamily(new HColumnDescriptor(CF_FOO.getBytes())); + tableDescriptor.addFamily(new HColumnDescriptor(CF_BAR.getBytes())); + admin.createTable(tableDescriptor); + System.out.println("Created table"); + } } - - Admin admin = _dataContext.getAdmin(); - System.out.println("Creating table"); - final HTableDescriptor tableDescriptor = new HTableDescriptor(tableName); - tableDescriptor.addFamily(new HColumnDescriptor(CF_FOO.getBytes())); - tableDescriptor.addFamily(new HColumnDescriptor(CF_BAR.getBytes())); - admin.createTable(tableDescriptor); - System.out.println("Created table"); } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java ---------------------------------------------------------------------- diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java index 1f7f162..b9524a8 100644 --- a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java +++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java @@ -22,14 +22,45 @@ import java.io.File; import java.io.FileReader; import java.util.Properties; +import org.apache.metamodel.schema.ColumnType; + import junit.framework.TestCase; public abstract class HBaseTestCase extends TestCase { + // TableName + protected static final String TABLE_NAME = "table_for_junit"; + + // ColumnFamilies + protected static final int NUMBER_OF_CFS = 3; // foo + bar + ID + protected static final String CF_FOO = "foo"; + protected static final String CF_BAR = "bar"; + + // Qualifiers + protected static final String Q_HELLO = "hello"; + protected static final String Q_HI = "hi"; + protected static final String Q_HEY = "hey"; + protected static final String Q_BAH = "bah"; + + // Number of rows + protected static final int NUMBER_OF_ROWS = 2; + + // RowKeys + protected static final String RK_1 = "junit1"; + protected static final String RK_2 = "junit2"; + + // RowValues + protected static final String V_WORLD = "world"; + protected static final String V_THERE = "there"; + protected static final String V_YO = "yo"; + protected static final byte[] V_123_BYTE_ARRAY = new byte[] { 1, 2, 3 }; + protected static final String V_YOU = "you"; + private String zookeeperHostname; private int zookeeperPort; private boolean _configured; - + private HBaseDataContext _dataContext; + @Override protected void setUp() throws Exception { super.setUp(); @@ -43,13 +74,18 @@ public abstract class HBaseTestCase extends TestCase { if (zookeeperPortPropertyValue != null && !zookeeperPortPropertyValue.isEmpty()) { zookeeperPort = Integer.parseInt(zookeeperPortPropertyValue); } - + _configured = (zookeeperHostname != null && !zookeeperHostname.isEmpty()); } else { _configured = false; } + if (isConfigured()) { + final HBaseConfiguration configuration = new HBaseConfiguration(zookeeperHostname, zookeeperPort, + ColumnType.VARCHAR); + setDataContext(new HBaseDataContext(configuration)); + } } - + private String getPropertyFilePath() { String userHome = System.getProperty("user.home"); return userHome + "/metamodel-integrationtest-configuration.properties"; @@ -63,7 +99,7 @@ public abstract class HBaseTestCase extends TestCase { public boolean isConfigured() { return _configured; } - + public String getZookeeperHostname() { return zookeeperHostname; } @@ -71,4 +107,18 @@ public abstract class HBaseTestCase extends TestCase { public int getZookeeperPort() { return zookeeperPort; } + + public HBaseDataContext getDataContext() { + return _dataContext; + } + + public void setDataContext(HBaseDataContext dataContext) { + this._dataContext = dataContext; + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + _dataContext.getConnection().close(); + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java ---------------------------------------------------------------------- diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java new file mode 100644 index 0000000..359be01 --- /dev/null +++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.hbase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.metamodel.MetaModelException; +import org.apache.metamodel.schema.ColumnType; +import org.apache.metamodel.schema.ImmutableSchema; +import org.apache.metamodel.schema.MutableSchema; +import org.apache.metamodel.schema.Table; +import org.apache.metamodel.util.SimpleTableDef; + +public class HBaseUpdateCallbackTest extends HBaseTestCase { + + private HBaseUpdateCallback updateCallback; + private MutableSchema schema; + + @Override + protected void setUp() throws Exception { + super.setUp(); + if (isConfigured()) { + updateCallback = new HBaseUpdateCallback(getDataContext()); + schema = (MutableSchema) getDataContext().getDefaultSchema(); + + if (schema.getTableByName(TABLE_NAME) != null) { + dropTableIfItExists(); + } + } + } + + public void testDropTable() throws IOException { + dropTableIfItExists(); + + try { + HBaseTable table = createHBaseTable(); + updateCallback.dropTable(table).execute(); + fail("Should get an exception that the table doesn't exist in the datastore"); + } catch (MetaModelException e) { + assertEquals("Trying to delete a table that doesn't exist in the datastore.", e.getMessage()); + } + } + + private void dropTableIfItExists() { + Table table = schema.getTableByName(TABLE_NAME); + if (table != null) { + updateCallback.dropTable(table).execute(); + // Check schema + assertNull(schema.getTableByName(TABLE_NAME)); + // Check in the datastore + try (Admin admin = getDataContext().getAdmin()) { + assertFalse(admin.tableExists(TableName.valueOf(TABLE_NAME))); + } catch (IOException e) { + fail("Should not an exception checking if the table exists"); + } + } + } + + public void testCreateTable() { + // Drop the table if it exists + dropTableIfItExists(); + + // Test 1: Create a table with an immutableSchema, should throw a IllegalArgumentException + ImmutableSchema immutableSchema = new ImmutableSchema(schema); + try { + updateCallback.createTable(immutableSchema, TABLE_NAME).execute(); + fail("Should get an exception that the schema isn't mutable"); + } catch (IllegalArgumentException e) { + assertEquals("Not a mutable schema: " + immutableSchema, e.getMessage()); + } + + // Test 2: Create a table without columnFamilies, should throw a MetaModelException + try { + updateCallback.createTable(schema, TABLE_NAME).execute(); + fail("Should get an exception that the columnFamilies haven't been set"); + } catch (MetaModelException e) { + assertEquals("Creating a table without columnFamilies", e.getMessage()); + } + + // Test 3: Create a table with columnFamilies null, should throw a MetaModelException + try { + updateCallback.createTable(schema, TABLE_NAME, null).execute(); + fail("Should get an exception that the columnFamilies haven't been set"); + } catch (MetaModelException e) { + assertEquals("Creating a table without columnFamilies", e.getMessage()); + } + + // Test 4: Create a table with columnFamilies empty, should throw a MetaModelException + try { + final LinkedHashSet<String> columnFamilies = new LinkedHashSet<String>(); + updateCallback.createTable(schema, TABLE_NAME, columnFamilies).execute(); + fail("Should get an exception that the columnFamilies haven't been set"); + } catch (MetaModelException e) { + assertEquals("Creating a table without columnFamilies", e.getMessage()); + } + + HBaseTable table = createHBaseTable(); + + // Test 5: Create a table without the ID-Column, should throw a MetaModelException + ArrayList<HBaseColumn> hBaseColumnsAsArrayList = createListWithHBaseColumnsExcludingIDColumn(table); + HBaseColumn[] hBaseColumnsAsArray = convertToHBaseColumnArray(hBaseColumnsAsArrayList); + Set<String> columnFamilies = HBaseColumn.getColumnFamilies(hBaseColumnsAsArray); + try { + HBaseCreateTableBuilder hBaseCreateTableBuilder = (HBaseCreateTableBuilder) updateCallback.createTable( + schema, TABLE_NAME); + + hBaseCreateTableBuilder.setColumnFamilies(columnFamilies); + hBaseCreateTableBuilder.execute(); + fail("Should get an exception that the ID-colum is missing"); + } catch (MetaModelException e) { + assertEquals("ColumnFamily: " + HBaseDataContext.FIELD_ID + " not found", e.getMessage()); + } + + // Test 6: Create a table including the ID-Column (columnFamilies not in constructor), should work + hBaseColumnsAsArrayList = createListWithHBaseColumnsIncludingIDColumn(table); + hBaseColumnsAsArray = convertToHBaseColumnArray(hBaseColumnsAsArrayList); + columnFamilies = HBaseColumn.getColumnFamilies(hBaseColumnsAsArray); + try { + HBaseCreateTableBuilder hBaseCreateTableBuilder = (HBaseCreateTableBuilder) updateCallback.createTable( + schema, TABLE_NAME); + + hBaseCreateTableBuilder.setColumnFamilies(HBaseColumn.getColumnFamilies(hBaseColumnsAsArray)); + hBaseCreateTableBuilder.execute(); + checkSuccesfullyInsertedTable(); + } catch (Exception e) { + fail("Should not get an exception"); + } + dropTableIfItExists(); + + // Test 7: Create a table including the ID-Column (columnFamilies in constructor), should work + try { + updateCallback.createTable(schema, TABLE_NAME, columnFamilies).execute(); + checkSuccesfullyInsertedTable(); + } catch (Exception e) { + fail("Should not get an exception"); + } + dropTableIfItExists(); + } + + private void checkSuccesfullyInsertedTable() throws IOException { + // Check the schema + assertNotNull(schema.getTableByName(TABLE_NAME)); + // Check in the datastore + try (Admin admin = getDataContext().getAdmin()) { + assertTrue(admin.tableExists(TableName.valueOf(TABLE_NAME))); + } catch (IOException e) { + fail("Should not an exception checking if the table exists"); + } + } + + // public void testInsertRows() throws IOException { + // // Drop the table if it exists + // dropTableIfItExists(); + // + // insertTable(); + // } + + private void insertTable() throws IOException { + HBaseTable table = createHBaseTable(); + ArrayList<HBaseColumn> hBaseColumnsAsArrayList = createListWithHBaseColumnsIncludingIDColumn(table); + HBaseColumn[] hBaseColumnsAsArray = convertToHBaseColumnArray(hBaseColumnsAsArrayList); + Set<String> columnFamilies = HBaseColumn.getColumnFamilies(hBaseColumnsAsArray); + updateCallback.createTable(schema, TABLE_NAME, columnFamilies).execute(); + checkSuccesfullyInsertedTable(); + } + + private HBaseTable createHBaseTable() { + String[] columnNames = new String[] { CF_FOO, CF_BAR }; + ColumnType[] columnTypes = new ColumnType[] { ColumnType.STRING, ColumnType.STRING }; + SimpleTableDef tableDef = new SimpleTableDef(TABLE_NAME, columnNames, columnTypes); + return new HBaseTable(getDataContext(), tableDef, schema, ColumnType.STRING); + } + + private static ArrayList<HBaseColumn> createListWithHBaseColumnsExcludingIDColumn(final HBaseTable table) { + ArrayList<HBaseColumn> hbaseColumns = new ArrayList<HBaseColumn>(); + hbaseColumns.add(new HBaseColumn(CF_FOO, Q_HELLO, table)); + hbaseColumns.add(new HBaseColumn(CF_FOO, Q_HI, table)); + hbaseColumns.add(new HBaseColumn(CF_BAR, Q_HEY, table)); + hbaseColumns.add(new HBaseColumn(CF_BAR, Q_BAH, table)); + return hbaseColumns; + } + + private static ArrayList<HBaseColumn> createListWithHBaseColumnsIncludingIDColumn(final HBaseTable table) { + ArrayList<HBaseColumn> hbaseColumns = createListWithHBaseColumnsExcludingIDColumn(table); + hbaseColumns.add(new HBaseColumn(HBaseDataContext.FIELD_ID, table)); + return hbaseColumns; + } + + private static HBaseColumn[] convertToHBaseColumnArray(final ArrayList<HBaseColumn> hBaseColumnsAsArrayList) { + return hBaseColumnsAsArrayList.toArray(new HBaseColumn[hBaseColumnsAsArrayList.size()]); + } +}
