Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java?rev=1422104&r1=1422103&r2=1422104&view=diff ============================================================================== --- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java (original) +++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/hcat/TestMetaDataClientWrapper.java Fri Dec 14 21:41:30 2012 @@ -1,214 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.oozie.hcat; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hcatalog.api.HCatAddPartitionDesc; -import org.apache.hcatalog.api.HCatClient; -import org.apache.hcatalog.api.HCatCreateDBDesc; -import org.apache.hcatalog.api.HCatCreateTableDesc; -import org.apache.hcatalog.api.HCatPartition; -import org.apache.hcatalog.data.schema.HCatFieldSchema; -import org.apache.hcatalog.data.schema.HCatFieldSchema.Type; -import org.apache.oozie.service.MetaDataAccessorService; -import org.apache.oozie.service.Services; -import org.apache.oozie.test.XDataTestCase; -import org.junit.After; -import org.junit.Before; - -public class TestMetaDataClientWrapper extends XDataTestCase { - private Services services; - private String server = "local"; - // private String server = "thrift://localhost:11002"; // to specify the - // non-local endpoint. - private String isLocal = "true"; // false for non-local instance - - @Before - protected void setUp() throws Exception { - super.setUp(); - setSystemProperty("hive.metastore.local", isLocal); - services = new Services(); - addServiceToRun(services.getConf(), MetaDataAccessorService.class.getName()); - services.init(); - } - - @After - protected void tearDown() throws Exception { - services.destroy(); - super.tearDown(); - } - - /** - * Test if query of one specific partition works. Also check the -ve case, - * if there is not such partition - * - * @throws Exception - */ - public void testGetOnePartition() throws Exception { - MetaDataClientWrapper clWr = new MetaDataClientWrapper(); - - String db = "default"; - String table = "tablename"; - - populateTable(server, db, table); - // +ve test - Map<String, String> existPartitions = new HashMap<String, String>(); - existPartitions.put("dt", "04/12/2012"); // The same value added in - // addRecords methods - existPartitions.put("country", "brazil"); - - HCatPartition part = clWr.getOnePartition(server, db, table, existPartitions, getTestUser()); - assertNotNull(part); - assertEquals(part.getDatabaseName(), db); - assertEquals(part.getTableName(), table); - assertTrue(part.getValues().size() == 2); - - // -ve test - Map<String, String> nonexistPartitions = new HashMap<String, String>(); - nonexistPartitions.put("dt", "04/01/2012"); // The same value added in - // addRecords methods - nonexistPartitions.put("country", "brazil2"); - try { - clWr.getOnePartition(server, db, table, nonexistPartitions, getTestUser()); - fail("Should throw exception earlier."); - } - catch (Exception ex) { - // Expected - } - } - - /** - * Test the query a list of partition based on specific filter. - * - * @throws Exception - */ - public void testGetPartitionsByFilter() throws Exception { - MetaDataClientWrapper clWr = new MetaDataClientWrapper(); - String db = "default"; - String table = "tablename"; - - populateTable(server, db, table); - - String filter = "dt = '04/12/2012' AND country = 'brazil'"; - - List<HCatPartition> partList = clWr.getPartitionsByFilter(server, db, table, filter, getTestUser()); - assertNotNull(partList); - assertTrue(partList.size() == 1); - - filter = "country = 'brazil'"; - partList = clWr.getPartitionsByFilter(server, db, table, filter, getTestUser()); - assertNotNull(partList); - assertTrue(partList.size() == 2); - - filter = "country = 'mexico'"; - partList = clWr.getPartitionsByFilter(server, db, table, filter, getTestUser()); - assertNotNull(partList); - assertTrue(partList.size() == 0); - } - - /** - * Remove an existing filter. - * - * @throws Exception - */ - public void testDropOnePartition() throws Exception { - MetaDataClientWrapper clWr = new MetaDataClientWrapper(); - String db = "default"; - String table = "tablename"; - - populateTable(server, db, table); - // +ve test - Map<String, String> existPartitions = new HashMap<String, String>(); - existPartitions.put("dt", "04/12/2012"); // The same value added in - // addRecords methods - existPartitions.put("country", "brazil"); - - HCatPartition part = clWr.getOnePartition(server, db, table, existPartitions, getTestUser()); - assertNotNull(part); - assertEquals(part.getDatabaseName(), db); - assertEquals(part.getTableName(), table); - assertTrue(part.getValues().size() == 2); - - clWr.dropOnePartition(server, db, table, existPartitions, true, getTestUser()); - // -ve test - try { - HCatPartition part2 = clWr.getOnePartition(server, db, table, existPartitions, getTestUser()); - fail("Should throw exception earlier."); - } - catch (Exception ex) { - // Expected - } - } - - private void populateTable(String server, String db, String table) throws Exception { - createTable(server, db, table); - addRecords(server, db, table); - - } - - private void createTable(String server, String db, String tableName) throws Exception { - HCatClient client = services.get(MetaDataAccessorService.class).getHCatClient(server, getTestUser()); - assertNotNull(client); - // Creating a table - HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(db).ifNotExists(true).build(); - client.createDatabase(dbDesc); - ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); - cols.add(new HCatFieldSchema("userid", Type.INT, "id columns")); - cols.add(new HCatFieldSchema("viewtime", Type.BIGINT, "view time columns")); - cols.add(new HCatFieldSchema("pageurl", Type.STRING, "")); - cols.add(new HCatFieldSchema("ip", Type.STRING, "IP Address of the User")); - ArrayList<HCatFieldSchema> ptnCols = new ArrayList<HCatFieldSchema>(); - ptnCols.add(new HCatFieldSchema("dt", Type.STRING, "date column")); - ptnCols.add(new HCatFieldSchema("country", Type.STRING, "country column")); - HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(db, tableName, cols).fileFormat("sequencefile") - .partCols(ptnCols).build(); - client.dropTable(db, tableName, true); - client.createTable(tableDesc); - List<String> tables = client.listTableNamesByPattern(db, "*"); - assertTrue(tables.size() > 0); - assertTrue(tables.contains(tableName)); - List<String> dbNames = client.listDatabaseNamesByPattern(db); - assertTrue(dbNames.size() == 1); - assertTrue(dbNames.contains(db)); - } - - private void addRecords(String server, String dbName, String tableName) throws Exception { - HCatClient client = services.get(MetaDataAccessorService.class).getHCatClient(server, getTestUser()); - Map<String, String> firstPtn = new HashMap<String, String>(); - firstPtn.put("dt", "04/30/2012"); - firstPtn.put("country", "usa"); - HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(dbName, tableName, null, firstPtn).build(); - client.addPartition(addPtn); - - Map<String, String> secondPtn = new HashMap<String, String>(); - secondPtn.put("dt", "04/12/2012"); - secondPtn.put("country", "brazil"); - HCatAddPartitionDesc addPtn2 = HCatAddPartitionDesc.create(dbName, tableName, null, secondPtn).build(); - client.addPartition(addPtn2); - - Map<String, String> thirdPtn = new HashMap<String, String>(); - thirdPtn.put("dt", "04/13/2012"); - thirdPtn.put("country", "brazil"); - HCatAddPartitionDesc addPtn3 = HCatAddPartitionDesc.create(dbName, tableName, null, thirdPtn).build(); - client.addPartition(addPtn3); - } -}
Added: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java?rev=1422104&view=auto ============================================================================== --- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java (added) +++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java Fri Dec 14 21:41:30 2012 @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.test; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hcatalog.api.HCatAddPartitionDesc; +import org.apache.hcatalog.api.HCatClient; +import org.apache.hcatalog.api.HCatClient.DROP_DB_MODE; +import org.apache.hcatalog.api.HCatCreateDBDesc; +import org.apache.hcatalog.api.HCatCreateTableDesc; +import org.apache.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hcatalog.data.schema.HCatFieldSchema.Type; +import org.apache.oozie.dependency.FSURIHandler; +import org.apache.oozie.dependency.HCatURIHandler; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.URIHandlerService; +import org.apache.oozie.util.XLog; + +/** + * Base JUnit <code>TestCase</code> subclass used by all Oozie testcases that + * need Hadoop FS access and HCat access. + */ +public abstract class XHCatTestCase extends XFsTestCase { + + private static XLog LOG = XLog.getLog(XHCatTestCase.class); + private static final Random RANDOM = new Random(); + private int msPort; + private Services services; + private Configuration hadoopConf; + private HiveConf hiveConf; + private HCatClient hcatClient; + private Thread serverThread; + + @Override + protected void setUp() throws Exception { + super.setUp(); + services = new Services(); + services.getConf().set(URIHandlerService.URI_HANDLERS, + FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName()); + services.init(); + hadoopConf = createJobConf(); + LOG.info("Namenode started at " + getNameNodeUri()); + msPort = RANDOM.nextInt(100) + 30000; + startMetastoreServer(); + initHiveConf(); + hcatClient = HCatClient.create(hiveConf); + } + + private void initHiveConf() throws Exception { + hiveConf = new HiveConf(hadoopConf, this.getClass()); + hiveConf.set("hive.metastore.local", "false"); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); + hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3); + hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + } + + private void startMetastoreServer() throws Exception { + final HiveConf serverConf = new HiveConf(hadoopConf, this.getClass()); + serverThread = new Thread(new Runnable() { + @Override + public void run() { + try { + HiveMetaStore.startMetaStore(msPort, ShimLoader.getHadoopThriftAuthBridge(), serverConf); + LOG.info("Started metastore server on port " + msPort); + } + catch (Throwable e) { + LOG.error("Metastore Thrift Server threw an exception...", e); + } + } + }); + serverThread.setDaemon(true); + serverThread.start(); + Thread.sleep(15000L); + } + + @Override + protected void tearDown() throws Exception { + services.destroy(); + hcatClient.close(); + serverThread.stop(); + super.tearDown(); + } + + protected Configuration getMetaStoreConf() { + return hiveConf; + } + + protected URI getHCatURI(String db, String table, String partitions) throws URISyntaxException { + String[] parts = partitions.split(","); + StringBuilder uri = new StringBuilder(); + uri.append("hcat://localhost:").append(msPort).append("/").append(db).append("/").append(table).append("/?"); + for (String partition : parts) { + uri.append(partition).append("&"); + } + uri.deleteCharAt(uri.length() - 1); + return new URI(uri.toString()); + } + + protected void createDatabase(String db) throws Exception { + HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(db).ifNotExists(true).location(getFsTestCaseDir().toString()) + .build(); + hcatClient.createDatabase(dbDesc); + List<String> dbNames = hcatClient.listDatabaseNamesByPattern(db); + assertTrue(dbNames.contains(db)); + } + + protected void createTable(String db, String table) throws Exception { + List<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); + cols.add(new HCatFieldSchema("userid", Type.INT, "userid")); + cols.add(new HCatFieldSchema("viewtime", Type.BIGINT, "view time")); + cols.add(new HCatFieldSchema("pageurl", Type.STRING, "page url visited")); + cols.add(new HCatFieldSchema("ip", Type.STRING, "IP Address of the User")); + ArrayList<HCatFieldSchema> ptnCols = new ArrayList<HCatFieldSchema>(); + ptnCols.add(new HCatFieldSchema("year", Type.STRING, "year column")); + ptnCols.add(new HCatFieldSchema("month", Type.STRING, "month column")); + ptnCols.add(new HCatFieldSchema("dt", Type.STRING, "date column")); + ptnCols.add(new HCatFieldSchema("region", Type.STRING, "country column")); + HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(db, table, cols).fileFormat("textfile") + .partCols(ptnCols).build(); + hcatClient.createTable(tableDesc); + List<String> tables = hcatClient.listTableNamesByPattern(db, "*"); + assertTrue(tables.contains(table)); + } + + protected void dropDatabase(String db, boolean ifExists) throws Exception { + hcatClient.dropDatabase(db, ifExists, DROP_DB_MODE.CASCADE); + List<String> dbNames = hcatClient.listDatabaseNamesByPattern(db); + assertFalse(dbNames.contains(db)); + } + + protected void dropTable(String db, String table, boolean ifExists) throws Exception { + hcatClient.dropTable(db, table, ifExists); + List<String> tables = hcatClient.listTableNamesByPattern(db, "*"); + assertFalse(tables.contains(table)); + } + + protected String createPartitionDir(String db, String table, String partitionSpec) throws Exception { + String dir = getPartitionDir(db, table, partitionSpec); + getFileSystem().mkdirs(new Path(dir)); + return dir; + } + + protected String getPartitionDir(String db, String table, String partitionSpec) throws Exception { + String dir = getFsTestCaseDir() + "/" + db + "/" + table + "/" + partitionSpec.replaceAll(",", "/"); + return dir; + } + + protected void addPartition(String db, String table, String partitionSpec, String location) throws Exception { + String[] parts = partitionSpec.split(","); + Map<String, String> partitions = new HashMap<String, String>(); + for (String part : parts) { + String[] split = part.split("="); + partitions.put(split[0], split[1]); + } + HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(db, table, location, partitions).build(); + hcatClient.addPartition(addPtn); + } + + protected void dropPartition(String db, String table, String partitionSpec) throws Exception { + String[] parts = partitionSpec.split(","); + Map<String, String> partitions = new HashMap<String, String>(); + for (String part : parts) { + String[] split = part.split("="); + partitions.put(split[0], split[1]); + } + hcatClient.dropPartition(db, table, partitions, false); + } + +} Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java?rev=1422104&r1=1422103&r2=1422104&view=diff ============================================================================== --- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java (original) +++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java Fri Dec 14 21:41:30 2012 @@ -38,7 +38,7 @@ public class TestHCatURI { catch (Exception ex) { System.err.print(ex.getMessage()); } - assertEquals(uri.getServerEndPoint(), HCatURI.DEFAULT_PROTOCOL + "hcat.yahoo.com:5080"); + assertEquals(uri.getServerEndPoint(), "hcat://hcat.yahoo.com:5080"); assertEquals(uri.getDb(), "mydb"); assertEquals(uri.getTable(), "clicks"); assertEquals(uri.getPartitionValue("datastamp"), "12"); @@ -62,7 +62,7 @@ public class TestHCatURI { catch (Exception ex) { System.err.println(ex.getMessage()); } - assertEquals(uri.getServerEndPoint(), HCatURI.DEFAULT_PROTOCOL + "hcat.yahoo.com:5080"); + assertEquals(uri.getServerEndPoint(), "hcat://hcat.yahoo.com:5080"); assertEquals(uri.getDb(), "mydb"); assertEquals(uri.getTable(), "clicks"); assertEquals(uri.getPartitionValue("datastamp"), "12"); @@ -85,7 +85,7 @@ public class TestHCatURI { catch (Exception ex) { System.err.println(ex.getMessage()); } - assertEquals(uri.getServerEndPoint(), HCatURI.DEFAULT_PROTOCOL + "hcat.yahoo.com:5080"); + assertEquals(uri.getServerEndPoint(), "hcat://hcat.yahoo.com:5080"); assertEquals(uri.getDb(), "mydb"); assertEquals(uri.getTable(), "clicks"); assertEquals(uri.getPartitionValue("datastamp"), "12"); @@ -108,7 +108,7 @@ public class TestHCatURI { catch (Exception ex) { System.err.println(ex.getMessage()); } - assertEquals(uri.getServerEndPoint(), HCatURI.DEFAULT_PROTOCOL + "hcat.yahoo.com:5080"); + assertEquals(uri.getServerEndPoint(), "hcat://hcat.yahoo.com:5080"); assertEquals(uri.getDb(), "mydb"); assertEquals(uri.getTable(), "clicks"); assertEquals(uri.getPartitionValue("datastamp"), "12"); @@ -131,7 +131,7 @@ public class TestHCatURI { catch (Exception ex) { System.err.println(ex.getMessage()); } - assertEquals(uri.getServerEndPoint(), HCatURI.DEFAULT_PROTOCOL + "hcat.yahoo.com:5080"); + assertEquals(uri.getServerEndPoint(), "hcat://hcat.yahoo.com:5080"); assertEquals(uri.getDb(), "mydb"); assertEquals(uri.getTable(), "clicks"); assertEquals(uri.getPartitionValue("datastamp"), "12"); Modified: oozie/branches/hcat-intre/pom.xml URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/pom.xml?rev=1422104&r1=1422103&r2=1422104&view=diff ============================================================================== --- oozie/branches/hcat-intre/pom.xml (original) +++ oozie/branches/hcat-intre/pom.xml Fri Dec 14 21:41:30 2012 @@ -851,6 +851,7 @@ <!-- See 'testHive' profile in core/pom.xml and the Building doc--> <exclude>**/TestHive*.java</exclude> + <exclude>**/TestHCatURIHandler.java</exclude> <!-- See 'testSqoop' profile in core/pom.xml and the Building doc--> <exclude>**/TestSqoop*.java</exclude> Modified: oozie/branches/hcat-intre/release-log.txt URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1422104&r1=1422103&r2=1422104&view=diff ============================================================================== --- oozie/branches/hcat-intre/release-log.txt (original) +++ oozie/branches/hcat-intre/release-log.txt Fri Dec 14 21:41:30 2012 @@ -1,5 +1,6 @@ -- Oozie 3.4.0 release (trunk - unreleased) +OOZIE-1075 Create general scheme handler (rohini via virag) OOZIE-1071 latest EL function is based on action materialization time (rohini via virag) OOZIE-1114 Some tests don't use the Services singleton properly (rkanter) OOZIE-1106 latest and future function do not work correctly when oozie processing timezone is non UTC (rohini via tucu)
