http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java new file mode 100644 index 0000000..2f2e561 --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java @@ -0,0 +1,386 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb.json; + +import static org.ojai.DocumentConstants.ID_FIELD; +import static org.ojai.DocumentConstants.ID_KEY; +import io.netty.buffer.DrillBuf; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.ops.OperatorContext; +import org.apache.drill.exec.ops.OperatorStats; +import org.apache.drill.exec.physical.impl.OutputMutator; +import org.apache.drill.exec.store.AbstractRecordReader; +import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec; +import org.apache.drill.exec.store.maprdb.util.CommonFns; +import org.apache.drill.exec.vector.BaseValueVector; +import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; +import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter; +import org.apache.drill.exec.vector.complex.writer.VarCharWriter; +import org.ojai.DocumentReader; +import org.ojai.DocumentReader.EventType; +import org.ojai.DocumentStream; +import org.ojai.FieldPath; +import org.ojai.Value; +import org.ojai.store.QueryCondition; +import org.ojai.store.QueryCondition.Op; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.mapr.db.DBDocument; +import com.mapr.db.MapRDB; +import com.mapr.db.Table; +import com.mapr.db.Table.TableOption; +import com.mapr.db.exceptions.DBException; +import com.mapr.db.impl.IdCodec; +import com.mapr.db.ojai.DBDocumentReader; +import com.mapr.db.util.ByteBufs; +import com.mapr.org.apache.hadoop.hbase.util.Bytes; + +public class MaprDBJsonRecordReader extends AbstractRecordReader { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MaprDBJsonRecordReader.class); + + public static final SchemaPath ID_PATH = SchemaPath.getSimplePath(ID_KEY); + + private Table table; + private QueryCondition condition; + private FieldPath[] projectedFields; + + private String tableName; + private OperatorContext operatorContext; + private VectorContainerWriter writer; + + @SuppressWarnings("unused") + private boolean idOnly; + + private DrillBuf buffer; + + private DocumentStream<DBDocument> documentStream; + + private Iterator<DocumentReader> documentReaderIterators; + + public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, + List<SchemaPath> projectedColumns, FragmentContext context) { + buffer = context.getManagedBuffer(); + tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName(); + condition = MapRDB.newCondition().and(); + addKeyCondition(condition, Op.GREATER_OR_EQUAL, subScanSpec.getStartRow()); + addKeyCondition(condition, Op.LESS, subScanSpec.getStopRow()); + if (subScanSpec.getSerializedFilter() != null) { + condition.condition(com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter()))); + } + condition.close().build(); + setColumns(projectedColumns); + } + + private void addKeyCondition(QueryCondition condition, Op op, byte[] key) { + if (!CommonFns.isNullOrEmpty(key)) { + Value value = IdCodec.decode(key); + switch (value.getType()) { + case STRING: + condition.is(ID_FIELD, op, value.getString()); + return; + case BINARY: + condition.is(ID_FIELD, op, value.getBinary()); + return; + default: + throw new UnsupportedOperationException(""); + } + } + } + + @Override + protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) { + Set<SchemaPath> transformed = Sets.newLinkedHashSet(); + idOnly = true; // TODO: handle the case when only ID is requested. + if (!isStarQuery()) { + ArrayList<Object> projectedFieldsList = Lists.newArrayList(); + for (SchemaPath column : columns) { + if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) { + transformed.add(ID_PATH); + continue; + } + idOnly = false; + projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath())); + } + projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]); + } else { + idOnly = false; + transformed.add(ID_PATH); + } + + return transformed; + } + + @Override + public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { + this.writer = new VectorContainerWriter(output); + this.operatorContext = context; + + try { + table = MapRDB.getTable(tableName); + table.setOption(TableOption.EXCLUDEID, true); + documentStream = table.find(condition, projectedFields); + documentReaderIterators = documentStream.documentReaders().iterator(); + } catch (DBException e) { + throw new ExecutionSetupException(e); + } + } + + @Override + public int next() { + Stopwatch watch = new Stopwatch(); + watch.start(); + + writer.allocate(); + writer.reset(); + + int recordCount = 0; + + while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION) { + DBDocumentReader reader = nextDocumentReader(); + if (reader == null) break; + writer.setPosition(recordCount); + if (reader.next() != EventType.START_MAP) { + throw new IllegalStateException("The document did not start with START_MAP!"); + } + try { + MapWriter map = writer.rootAsMap(); + if (reader.getId() != null) { + switch (reader.getId().getType()) { + case BINARY: + writeBinary(map.varBinary(ID_KEY), reader.getId().getBinary()); + break; + case STRING: + writeString(map.varChar(ID_KEY), reader.getId().getString()); + break; + default: + throw new UnsupportedOperationException(reader.getId().getType() + + " is not a supported type for _id field."); + } + } + writeToMap(reader, map); + recordCount++; + } catch (IllegalStateException e) { + logger.warn(String.format("Possible schema change at _id: %s", + IdCodec.asString(reader.getId())), e); + } + } + + writer.setValueCount(recordCount); + logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), recordCount); + return recordCount; + } + + private void writeToMap(DBDocumentReader reader, MapWriter map) { + String fieldName = null; + map.start(); + outside: while (true) { + EventType event = reader.next(); + if (event == null) break outside; + switch (event) { + case FIELD_NAME: + fieldName = reader.getFieldName(); + break; + case NULL: + map.varChar(fieldName).write(null); // treat as VARCHAR for now + case BINARY: + writeBinary(map.varBinary(fieldName), reader.getBinary()); + break; + case BOOLEAN: + map.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); + break; + case STRING: + writeString(map.varChar(fieldName), reader.getString()); + break; + case BYTE: + map.tinyInt(fieldName).writeTinyInt(reader.getByte()); + break; + case SHORT: + map.smallInt(fieldName).writeSmallInt(reader.getShort()); + break; + case INT: + map.integer(fieldName).writeInt(reader.getInt()); + break; + case LONG: + map.bigInt(fieldName).writeBigInt(reader.getLong()); + break; + case FLOAT: + map.float4(fieldName).writeFloat4(reader.getFloat()); + break; + case DOUBLE: + map.float8(fieldName).writeFloat8(reader.getDouble()); + break; + case DECIMAL: + throw new UnsupportedOperationException("Decimals are currently not supported."); + case DATE: + map.date(fieldName).writeDate(reader.getDate().getTime()); + break; + case TIME: + map.time(fieldName).writeTime(reader.getTimeInt()); + break; + case TIMESTAMP: + map.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); + break; + case INTERVAL: + throw new UnsupportedOperationException("Interval is currently not supported."); + case START_MAP: + writeToMap(reader, map.map(fieldName)); + break; + case END_MAP: + break outside; + case START_ARRAY: + writeToList(reader, map.list(fieldName)); + break; + case END_ARRAY: + throw new IllegalStateException("Shouldn't get a END_ARRAY inside a map"); + default: + throw new UnsupportedOperationException("Unsupported type: " + event); + } + } + map.end(); + } + + private void writeToList(DBDocumentReader reader, ListWriter list) { + list.start(); + outside: while (true) { + EventType event = reader.next(); + if (event == null) break outside; + switch (event) { + case FIELD_NAME: + throw new IllegalStateException("Shouldn't get a field name inside a list"); + case NULL: + list.varChar().write(null); // treat as VARCHAR for now + case BINARY: + writeBinary(list.varBinary(), reader.getBinary()); + break; + case BOOLEAN: + list.bit().writeBit(reader.getBoolean() ? 1 : 0); + break; + case STRING: + writeString(list.varChar(), reader.getString()); + break; + case BYTE: + list.tinyInt().writeTinyInt(reader.getByte()); + break; + case SHORT: + list.smallInt().writeSmallInt(reader.getShort()); + break; + case INT: + list.integer().writeInt(reader.getInt()); + break; + case LONG: + list.bigInt().writeBigInt(reader.getLong()); + break; + case FLOAT: + list.float4().writeFloat4(reader.getFloat()); + break; + case DOUBLE: + list.float8().writeFloat8(reader.getDouble()); + break; + case DECIMAL: + throw new UnsupportedOperationException("Decimals are currently not supported."); + case DATE: + list.date().writeDate(reader.getDate().getTime()); + break; + case TIME: + list.time().writeTime(reader.getTimeInt()); + break; + case TIMESTAMP: + list.timeStamp().writeTimeStamp(reader.getTimestampLong()); + break; + case INTERVAL: + throw new UnsupportedOperationException("Interval is currently not supported."); + case START_MAP: + writeToMap(reader, list.map()); + break; + case END_MAP: + throw new IllegalStateException("Shouldn't get a END_MAP inside a list"); + case START_ARRAY: + writeToList(reader, list.list()); + break; + case END_ARRAY: + break outside; + default: + throw new UnsupportedOperationException("Unsupported type: " + event); + } + } + list.end(); + } + + private void writeBinary(VarBinaryWriter binaryWriter, ByteBuffer buf) { + buffer.reallocIfNeeded(buf.remaining()); + buffer.setBytes(0, buf, buf.position(), buf.remaining()); + binaryWriter.writeVarBinary(0, buf.remaining(), buffer); + } + + private void writeString(VarCharWriter varCharWriter, String string) { + final byte[] strBytes = Bytes.toBytes(string); + buffer.reallocIfNeeded(strBytes.length); + buffer.setBytes(0, strBytes); + varCharWriter.writeVarChar(0, strBytes.length, buffer); + } + + private DBDocumentReader nextDocumentReader() { + final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); + try { + if (operatorStats != null) { + operatorStats.startWait(); + } + try { + if (!documentReaderIterators.hasNext()) { + return null; + } else { + return (DBDocumentReader) documentReaderIterators.next(); + } + } finally { + if (operatorStats != null) { + operatorStats.stopWait(); + } + } + } catch (DBException e) { + throw new DrillRuntimeException(e); + } + } + + @Override + public void close() { + if (documentStream != null) { + documentStream.close(); + } + if (table != null) { + table.close(); + } + } + +}
http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java new file mode 100644 index 0000000..894e5bd --- /dev/null +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/util/CommonFns.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.maprdb.util; + +public class CommonFns { + + public static boolean isNullOrEmpty(final byte[] key) { + return key == null || key.length == 0; + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java new file mode 100644 index 0000000..cd1333a --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.hbase.HBaseTestsSuite; +import org.apache.hadoop.conf.Configuration; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; +import org.ojai.Document; +import org.ojai.DocumentStream; +import org.ojai.json.Json; + +import com.mapr.db.Admin; +import com.mapr.db.MapRDB; +import com.mapr.db.Table; +import com.mapr.drill.maprdb.tests.binary.TestMapRDBFilterPushDown; +import com.mapr.drill.maprdb.tests.binary.TestMapRDBSimple; +import com.mapr.drill.maprdb.tests.json.TestSimpleJson; + +@RunWith(Suite.class) +@SuiteClasses({ + TestMapRDBSimple.class, + TestMapRDBFilterPushDown.class, + TestSimpleJson.class +}) +public class MaprDBTestsSuite { + private static final String TMP_BUSINESS_TABLE = "/tmp/business"; + + private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0; + + private static volatile AtomicInteger initCount = new AtomicInteger(0); + private static volatile Configuration conf; + + private static Admin admin; + + @BeforeClass + public static void setupTests() throws Exception { + if (initCount.get() == 0) { + synchronized (MaprDBTestsSuite.class) { + if (initCount.get() == 0) { + HBaseTestsSuite.configure(false, true); + HBaseTestsSuite.initCluster(); + createJsonTables(); + + // Sleep to allow table data to be flushed to tables. + // Without this, the row count stats to return 0, + // causing the planner to reject optimized plans. + System.out.println("Sleeping for 5 seconds to allow table flushes"); + Thread.sleep(5000); + + conf = HBaseTestsSuite.getConf(); + initCount.incrementAndGet(); // must increment while inside the synchronized block + return; + } + } + } + initCount.incrementAndGet(); + return; + } + + @AfterClass + public static void cleanupTests() throws Exception { + synchronized (MaprDBTestsSuite.class) { + if (initCount.decrementAndGet() == 0) { + HBaseTestsSuite.tearDownCluster(); + deleteJsonTables(); + } + } + } + + private static volatile boolean pluginCreated; + + public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws Exception { + if (!pluginCreated) { + synchronized (MaprDBTestsSuite.class) { + if (!pluginCreated) { + String pluginConfStr = "{" + + " \"type\": \"file\"," + + " \"enabled\": true," + + " \"connection\": \"maprfs:///\"," + + " \"workspaces\": {" + + " \"default\": {" + + " \"location\": \"/tmp\"," + + " \"writable\": false," + + " \"defaultInputFormat\": \"maprdb\"" + + " }" + + " }," + + " \"formats\": {" + + " \"maprdb\": {" + + " \"type\": \"maprdb\"" + + " }" + + " }" + + "}"; + + FileSystemConfig pluginConfig = ctx.getConfig().getMapper().readValue(pluginConfStr, FileSystemConfig.class); + // create the plugin with "hbase" name so that we can run HBase unit tests against them + ctx.getStorage().createOrUpdate("hbase", pluginConfig, true); + } + } + } + return conf; + } + + public static boolean isDebug() { + return IS_DEBUG; + } + + public static InputStream getJsonStream(String resourceName) { + return MaprDBTestsSuite.class.getClassLoader().getResourceAsStream(resourceName); + } + + public static void createJsonTables() throws IOException { + admin = MapRDB.newAdmin(); + if (admin.tableExists(TMP_BUSINESS_TABLE)) { + admin.deleteTable(TMP_BUSINESS_TABLE); + } + + try (Table table = admin.createTable(TMP_BUSINESS_TABLE); + InputStream in = getJsonStream("json/business.json"); + DocumentStream<Document> stream = Json.newDocumentStream(in)) { + for (Document document : stream) { + table.insert(document, "business_id"); + } + table.flush(); + } + } + + public static void deleteJsonTables() { + if (admin != null) { + if (admin.tableExists(TMP_BUSINESS_TABLE)) { + admin.deleteTable(TMP_BUSINESS_TABLE); + } + admin.close(); + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java new file mode 100644 index 0000000..b049f37 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.TestHBaseFilterPushDown; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +/** + * This class does not define any test method but includes all test methods + * defined in the parent class, all of which are tested against MapRDB instead + * of HBase. + */ +@Category(ClusterTest.class) +public class TestMapRDBFilterPushDown extends TestHBaseFilterPushDown { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java new file mode 100644 index 0000000..894e64d --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.binary; + +import org.apache.drill.hbase.BaseHBaseTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +@Category(ClusterTest.class) +public class TestMapRDBSimple extends BaseHBaseTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + + @Test + public void testMe() throws Exception { + setColumnWidths(new int[] {8, 38, 38}); + final String sql = "SELECT\n" + + " *\n" + + "FROM\n" + + " hbase.`[TABLE_NAME]` tableName"; + runHBaseSQLVerifyCount(sql, 7); + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java new file mode 100644 index 0000000..c92fc44 --- /dev/null +++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.mapr.drill.maprdb.tests.json; + +import java.util.List; + +import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.exception.SchemaChangeException; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; +import com.mapr.tests.annotations.ClusterTest; + +@Category(ClusterTest.class) +public class TestSimpleJson extends BaseTestQuery { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + MaprDBTestsSuite.setupTests(); + MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + MaprDBTestsSuite.cleanupTests(); + } + + @Test + public void testMe() throws Exception { + setColumnWidths(new int[] {25, 40, 40, 40}); + final String sql = "SELECT\n" + + " _id, name, categories, full_address\n" + + "FROM\n" + + " hbase.`business` business"; + runSQLAndVerifyCount(sql, 10); + } + + protected List<QueryDataBatch> runHBaseSQLlWithResults(String sql) throws Exception { + System.out.println("Running query:\n" + sql); + return testSqlWithResults(sql); + } + + protected void runSQLAndVerifyCount(String sql, int expectedRowCount) throws Exception{ + List<QueryDataBatch> results = runHBaseSQLlWithResults(sql); + printResultAndVerifyRowCount(results, expectedRowCount); + } + + private void printResultAndVerifyRowCount(List<QueryDataBatch> results, int expectedRowCount) throws SchemaChangeException { + int rowCount = printResult(results); + if (expectedRowCount != -1) { + Assert.assertEquals(expectedRowCount, rowCount); + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/resources/hbase-site.xml ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/resources/hbase-site.xml b/contrib/format-maprdb/src/test/resources/hbase-site.xml new file mode 100644 index 0000000..92e8a86 --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/hbase-site.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>hbase.table.namespace.mappings</name> + <value>*:/tmp/</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/resources/json/business.json ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/resources/json/business.json b/contrib/format-maprdb/src/test/resources/json/business.json new file mode 100644 index 0000000..e1d46ac --- /dev/null +++ b/contrib/format-maprdb/src/test/resources/json/business.json @@ -0,0 +1,10 @@ +{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"b runch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":tru e,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"roma ntic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business"} +{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business"} http://git-wip-us.apache.org/repos/asf/drill/blob/f97a3332/contrib/format-maprdb/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/contrib/format-maprdb/src/test/resources/logback.xml b/contrib/format-maprdb/src/test/resources/logback.xml index 00cc85b..38c2fc8 100644 --- a/contrib/format-maprdb/src/test/resources/logback.xml +++ b/contrib/format-maprdb/src/test/resources/logback.xml @@ -28,7 +28,7 @@ <appender name="FILE" class="ch.qos.logback.core.FileAppender"> <!-- The property 'logback.log.dir' is defined in pom.xml --> - <file>${logback.log.dir:-./target/surefire-reports}/hbase-tests-${bySecond}.log</file> + <file>${logback.log.dir:-./target/surefire-reports}/maprdb-tests-${bySecond}.log</file> <append>false</append> <encoder> <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> @@ -41,6 +41,16 @@ </encoder> </appender> + <logger name="com.mapr" additivity="false"> + <level value="info" /> + <appender-ref ref="FILE" /> + </logger> + + <logger name="com.mapr" additivity="false"> + <level value="debug" /> + <appender-ref ref="SOCKET" /> + </logger> + <logger name="org.apache.drill" additivity="false"> <level value="info" /> <appender-ref ref="FILE" />