Repository: crunch Updated Branches: refs/heads/master 8121bdf5a -> 6c0ae4131
CRUNCH-664 Fixes HBase configuration properties being overwritten Signed-off-by: Josh Wills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6c0ae413 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6c0ae413 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6c0ae413 Branch: refs/heads/master Commit: 6c0ae41318a89a4aed9d343f79849fbadc329e56 Parents: 8121bdf Author: Nathan Schile <[email protected]> Authored: Mon Feb 5 09:08:46 2018 -0600 Committer: Josh Wills <[email protected]> Committed: Mon Feb 5 21:40:50 2018 -0800 ---------------------------------------------------------------------- .../org/apache/crunch/io/hbase/HBaseTarget.java | 2 +- .../org/apache/crunch/io/hbase/HFileTarget.java | 2 +- .../apache/crunch/io/hbase/HBaseTargetTest.java | 41 ++++++++++++++++++++ .../apache/crunch/io/hbase/HFileTargetTest.java | 41 ++++++++++++++++++++ .../src/test/resources/test-hbase-conf.xml | 8 ++++ 5 files changed, 92 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java index f287d5e..f4f134d 100644 --- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java +++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java @@ -100,7 +100,7 @@ public class HBaseTarget implements MapReduceTarget { @Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { final Configuration conf = job.getConfiguration(); - HBaseConfiguration.addHbaseResources(conf); + HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf)); conf.setStrings("io.serializations", conf.get("io.serializations"), MutationSerialization.class.getName()); Class<?> typeClass = ptype.getTypeClass(); // Either Put or Delete http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java index 41d56ff..8593a76 100644 --- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java +++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java @@ -56,7 +56,7 @@ public class HFileTarget extends FileTargetImpl { @Override public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) { Configuration conf = job.getConfiguration(); - HBaseConfiguration.addHbaseResources(conf); + HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf)); conf.setStrings("io.serializations", conf.get("io.serializations"), KeyValueSerialization.class.getName()); super.configureForMapReduce(job, ptype, outputPath, name); http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java new file mode 100644 index 0000000..8faa27d --- /dev/null +++ b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.io.hbase; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.Job; +import org.junit.Test; + +public class HBaseTargetTest { + + @Test + public void testConfigureForMapReduce() throws IOException { + Job job = Job.getInstance(); + // Add the test config file. We can't just call job.getConfiguration().set() because + // setting a configuration with .set will always to precedence. + job.getConfiguration().addResource("test-hbase-conf.xml"); + + HBaseTarget target = new HBaseTarget("testTable"); + target.configureForMapReduce(job, HBaseTypes.keyValues(), new Path("/"), "name"); + + assertEquals("12345", job.getConfiguration().get("hbase.client.scanner.timeout.period")); + } +} http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java new file mode 100644 index 0000000..fbe1a72 --- /dev/null +++ b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.io.hbase; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.Job; +import org.junit.Test; + +public class HFileTargetTest { + + @Test + public void testConfigureForMapReduce() throws IOException { + Job job = Job.getInstance(); + // Add the test config file. We can't just call job.getConfiguration().set() because + // setting a configuration with .set will always to precedence. + job.getConfiguration().addResource("test-hbase-conf.xml"); + + HFileTarget target = new HFileTarget("/"); + target.configureForMapReduce(job, HBaseTypes.keyValues(), new Path("/"), "name"); + + assertEquals("12345", job.getConfiguration().get("hbase.client.scanner.timeout.period")); + } +} http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/test/resources/test-hbase-conf.xml ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/test/resources/test-hbase-conf.xml b/crunch-hbase/src/test/resources/test-hbase-conf.xml new file mode 100644 index 0000000..a6e6dc0 --- /dev/null +++ b/crunch-hbase/src/test/resources/test-hbase-conf.xml @@ -0,0 +1,8 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <property> + <name>hbase.client.scanner.timeout.period</name> + <value>12345</value> + </property> +</configuration>
