Author: khorgath Date: Thu Aug 8 21:07:40 2013 New Revision: 1512031 URL: http://svn.apache.org/r1512031 Log: HIVE-5011 Dynamic partitioning in HCatalog broken on external tables (Sushanth Sowmyan, reviewed by Daniel Dai)
Added: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java?rev=1512031&r1=1512030&r2=1512031&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java (original) +++ hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java Thu Aug 8 21:07:40 2013 @@ -117,9 +117,11 @@ public class FosterStorageHandler extend String outputLocation; - if (Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL")) + if ((dynHash == null) + && Boolean.valueOf((String)tableDesc.getProperties().get("EXTERNAL")) && jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) { - // honor external table that specifies the location + // honor custom location for external table apart from what metadata specifies + // only if we're not using dynamic partitioning - see HIVE-5011 outputLocation = jobInfo.getLocation(); } else if (dynHash == null && jobInfo.getPartitionValues().size() == 0) { // For non-partitioned tables, we send them to the temp dir Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1512031&r1=1512030&r2=1512031&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Thu Aug 8 21:07:40 2013 @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.LocalFileSys import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; @@ -84,6 +85,10 @@ public abstract class HCatMapReduceTest private static FileSystem fs; + protected Boolean isTableExternal() { + return false; + } + protected String inputFormat() { return RCFileInputFormat.class.getName(); } @@ -137,7 +142,11 @@ public abstract class HCatMapReduceTest Table tbl = new Table(); tbl.setDbName(databaseName); tbl.setTableName(tableName); - tbl.setTableType("MANAGED_TABLE"); + if (isTableExternal()){ + tbl.setTableType(TableType.EXTERNAL_TABLE.toString()); + } else { + tbl.setTableType(TableType.MANAGED_TABLE.toString()); + } StorageDescriptor sd = new StorageDescriptor(); sd.setCols(getTableColumns()); @@ -150,6 +159,9 @@ public abstract class HCatMapReduceTest sd.getSerdeInfo().setName(tbl.getTableName()); sd.getSerdeInfo().setParameters(new HashMap<String, String>()); sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, "1"); + if (isTableExternal()){ + sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE"); + } sd.getSerdeInfo().setSerializationLib(serdeClass()); sd.setInputFormat(inputFormat()); sd.setOutputFormat(outputFormat()); Added: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java?rev=1512031&view=auto ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java (added) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java Thu Aug 8 21:07:40 2013 @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.mapreduce; + +public class TestHCatExternalDynamicPartitioned extends TestHCatDynamicPartitioned { + + @Override + protected Boolean isTableExternal() { + return true; + } + +} Added: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java?rev=1512031&view=auto ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java (added) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalHCatNonPartitioned.java Thu Aug 8 21:07:40 2013 @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.mapreduce; + +public class TestHCatExternalHCatNonPartitioned extends TestHCatNonPartitioned { + + @Override + protected Boolean isTableExternal() { + return true; + } + +} Added: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java?rev=1512031&view=auto ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java (added) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatExternalPartitioned.java Thu Aug 8 21:07:40 2013 @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hcatalog.mapreduce; + +public class TestHCatExternalPartitioned extends TestHCatPartitioned { + + @Override + protected Boolean isTableExternal() { + return true; + } + +}