[jira] [Commented] (HIVE-17498) Does hive have mr-nativetask support refer to MAPREDUCE-2841
[
https://issues.apache.org/jira/browse/HIVE-17498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16429741#comment-16429741
]
Igloo commented on HIVE-17498:
--
!image-2018-04-08-20-00-58-680.png!
> Does hive have mr-nativetask support refer to MAPREDUCE-2841
>
>
> Key: HIVE-17498
> URL: https://issues.apache.org/jira/browse/HIVE-17498
> Project: Hive
> Issue Type: Bug
> Components: Hive
>Reporter: Feng Yuan
>Priority: Major
>
> I try to implement a HivePlatform extends
> org.apache.hadoop.mapred.nativetask.Platform.
> {code}
> /**
> * Licensed to the Apache Software Foundation (ASF) under one
> * or more contributor license agreements. See the NOTICE file
> * distributed with this work for additional information
> * regarding copyright ownership. The ASF licenses this file
> * to you under the Apache License, Version 2.0 (the
> * "License"); you may not use this file except in compliance
> * with the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.hadoop.mapred.nativetask;
> import org.apache.hadoop.hive.ql.io.HiveKey;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
> import org.apache.log4j.Logger;
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> public class HivePlatform extends Platform {
> private static final Logger LOG = Logger.getLogger(HivePlatform.class);
> public HivePlatform() {
> }
> @Override
> public void init() throws IOException {
> registerKey("org.apache.hadoop.hive.ql.io.HiveKey",
> HiveKeySerializer.class);
> LOG.info("Hive platform inited");
> }
> @Override
> public String name() {
> return "Hive";
> }
> @Override
> public boolean support(String keyClassName, INativeSerializer serializer,
> JobConf job) {
> if (keyClassNames.contains(keyClassName) && serializer instanceof
> INativeComparable) {
> String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "."
> + keyClassName;
> job.set(nativeComparator,
> "HivePlatform.HivePlatform::HiveKeyComparator");
> if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
> job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN,
> "HivePlatform=libnativetask.so");
> }
> return true;
> } else {
> return false;
> }
> }
> @Override
> public boolean define(Class comparatorClass) {
> return false;
> }
> public static class HiveKeySerializer implements INativeComparable,
> INativeSerializer {
> public HiveKeySerializer() throws ClassNotFoundException,
> SecurityException, NoSuchMethodException {
> }
> @Override
> public int getLength(HiveKey w) throws IOException {
> return 4 + w.getLength();
> }
> @Override
> public void serialize(HiveKey w, DataOutput out) throws IOException {
> w.write(out);
> }
> @Override
> public void deserialize(DataInput in, int length, HiveKey w ) throws
> IOException {
> w.readFields(in);
> }
> }
> }
> {code}
> and throws exceptions:
> {code}
> Error: java.io.IOException: Initialization of all the collectors failed.
> Error in last collector was :Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at
> org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at
> org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at
> org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at
> java.security.AccessController.doPrivileged(Native Method) at
> javax.security.auth.Subject.doAs(Subject.java:422) at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by:
> java.io.IOException: Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165)
> at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402)
> ... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is
> NULL, or comparator is not set
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76)
> [0x7ffdcbba6436]
>
[jira] [Commented] (HIVE-17498) Does hive have mr-nativetask support refer to MAPREDUCE-2841
[
https://issues.apache.org/jira/browse/HIVE-17498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16429740#comment-16429740
]
Igloo commented on HIVE-17498:
--
{quote}diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
index 243668117b..0b120e1ac9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc
@@ -29,6 +29,9 @@ KeyValueType JavaClassToKeyValueType(const std::string &
clazz) {
if (clazz == "org.apache.hadoop.io.BytesWritable") {
return BytesType;
}
+ if (clazz == "org.apache.hadoop.hive.ql.io.HiveKey") {
+ return BytesType;
+ }
if (clazz == "org.apache.hadoop.io.ByteWritable") {
return ByteType;
}
{quote}
the patch works for me~
> Does hive have mr-nativetask support refer to MAPREDUCE-2841
>
>
> Key: HIVE-17498
> URL: https://issues.apache.org/jira/browse/HIVE-17498
> Project: Hive
> Issue Type: Bug
> Components: Hive
>Reporter: Feng Yuan
>Priority: Major
>
> I try to implement a HivePlatform extends
> org.apache.hadoop.mapred.nativetask.Platform.
> {code}
> /**
> * Licensed to the Apache Software Foundation (ASF) under one
> * or more contributor license agreements. See the NOTICE file
> * distributed with this work for additional information
> * regarding copyright ownership. The ASF licenses this file
> * to you under the Apache License, Version 2.0 (the
> * "License"); you may not use this file except in compliance
> * with the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.hadoop.mapred.nativetask;
> import org.apache.hadoop.hive.ql.io.HiveKey;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
> import org.apache.log4j.Logger;
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> public class HivePlatform extends Platform {
> private static final Logger LOG = Logger.getLogger(HivePlatform.class);
> public HivePlatform() {
> }
> @Override
> public void init() throws IOException {
> registerKey("org.apache.hadoop.hive.ql.io.HiveKey",
> HiveKeySerializer.class);
> LOG.info("Hive platform inited");
> }
> @Override
> public String name() {
> return "Hive";
> }
> @Override
> public boolean support(String keyClassName, INativeSerializer serializer,
> JobConf job) {
> if (keyClassNames.contains(keyClassName) && serializer instanceof
> INativeComparable) {
> String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "."
> + keyClassName;
> job.set(nativeComparator,
> "HivePlatform.HivePlatform::HiveKeyComparator");
> if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
> job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN,
> "HivePlatform=libnativetask.so");
> }
> return true;
> } else {
> return false;
> }
> }
> @Override
> public boolean define(Class comparatorClass) {
> return false;
> }
> public static class HiveKeySerializer implements INativeComparable,
> INativeSerializer {
> public HiveKeySerializer() throws ClassNotFoundException,
> SecurityException, NoSuchMethodException {
> }
> @Override
> public int getLength(HiveKey w) throws IOException {
> return 4 + w.getLength();
> }
> @Override
> public void serialize(HiveKey w, DataOutput out) throws IOException {
> w.write(out);
> }
> @Override
> public void deserialize(DataInput in, int length, HiveKey w ) throws
> IOException {
> w.readFields(in);
> }
> }
> }
> {code}
> and throws exceptions:
> {code}
> Error: java.io.IOException: Initialization of all the collectors failed.
> Error in last collector was :Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at
> org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at
> org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at
>
[jira] [Commented] (HIVE-17498) Does hive have mr-nativetask support refer to MAPREDUCE-2841
[
https://issues.apache.org/jira/browse/HIVE-17498?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16429684#comment-16429684
]
Igloo commented on HIVE-17498:
--
[~Feng Yuan] I encountered the exactly same problem, how did you fix it at
last?
> Does hive have mr-nativetask support refer to MAPREDUCE-2841
>
>
> Key: HIVE-17498
> URL: https://issues.apache.org/jira/browse/HIVE-17498
> Project: Hive
> Issue Type: Bug
> Components: Hive
>Reporter: Feng Yuan
>Priority: Major
>
> I try to implement a HivePlatform extends
> org.apache.hadoop.mapred.nativetask.Platform.
> {code}
> /**
> * Licensed to the Apache Software Foundation (ASF) under one
> * or more contributor license agreements. See the NOTICE file
> * distributed with this work for additional information
> * regarding copyright ownership. The ASF licenses this file
> * to you under the Apache License, Version 2.0 (the
> * "License"); you may not use this file except in compliance
> * with the License. You may obtain a copy of the License at
> *
> * http://www.apache.org/licenses/LICENSE-2.0
> *
> * Unless required by applicable law or agreed to in writing, software
> * distributed under the License is distributed on an "AS IS" BASIS,
> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> * See the License for the specific language governing permissions and
> * limitations under the License.
> */
> package org.apache.hadoop.mapred.nativetask;
> import org.apache.hadoop.hive.ql.io.HiveKey;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
> import org.apache.log4j.Logger;
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> public class HivePlatform extends Platform {
> private static final Logger LOG = Logger.getLogger(HivePlatform.class);
> public HivePlatform() {
> }
> @Override
> public void init() throws IOException {
> registerKey("org.apache.hadoop.hive.ql.io.HiveKey",
> HiveKeySerializer.class);
> LOG.info("Hive platform inited");
> }
> @Override
> public String name() {
> return "Hive";
> }
> @Override
> public boolean support(String keyClassName, INativeSerializer serializer,
> JobConf job) {
> if (keyClassNames.contains(keyClassName) && serializer instanceof
> INativeComparable) {
> String nativeComparator = Constants.NATIVE_MAPOUT_KEY_COMPARATOR + "."
> + keyClassName;
> job.set(nativeComparator,
> "HivePlatform.HivePlatform::HiveKeyComparator");
> if (job.get(Constants.NATIVE_CLASS_LIBRARY_BUILDIN) == null) {
> job.set(Constants.NATIVE_CLASS_LIBRARY_BUILDIN,
> "HivePlatform=libnativetask.so");
> }
> return true;
> } else {
> return false;
> }
> }
> @Override
> public boolean define(Class comparatorClass) {
> return false;
> }
> public static class HiveKeySerializer implements INativeComparable,
> INativeSerializer {
> public HiveKeySerializer() throws ClassNotFoundException,
> SecurityException, NoSuchMethodException {
> }
> @Override
> public int getLength(HiveKey w) throws IOException {
> return 4 + w.getLength();
> }
> @Override
> public void serialize(HiveKey w, DataOutput out) throws IOException {
> w.write(out);
> }
> @Override
> public void deserialize(DataInput in, int length, HiveKey w ) throws
> IOException {
> w.readFields(in);
> }
> }
> }
> {code}
> and throws exceptions:
> {code}
> Error: java.io.IOException: Initialization of all the collectors failed.
> Error in last collector was :Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:415) at
> org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:442) at
> org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at
> org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at
> java.security.AccessController.doPrivileged(Native Method) at
> javax.security.auth.Subject.doAs(Subject.java:422) at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1700)
> at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by:
> java.io.IOException: Native output collector cannot be loaded; at
> org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator.init(NativeMapOutputCollectorDelegator.java:165)
> at org.apache.hadoop.mapred.MapTask.createSortingCollector(MapTask.java:402)
> ... 7 more Caused by: java.io.IOException: /PartitionBucket.h:56:pool is
> NULL, or comparator is not set
> /usr/local/hadoop-2.7.3-yarn/lib/native/libnativetask.so.1.0.0(_ZN10NativeTask15HadoopExceptionC2ERKSs+0x76)
> [0x7ffdcbba6436]
>
