[01/44] hive git commit: HIVE-14680 : retain consistent splits /during/ (as opposed to across) LLAP failures on top of HIVE-14589 (Sergey Shelukhin, reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/hive-14535 70299dc48 -> 6d9144835
HIVE-14680 : retain consistent splits /during/ (as opposed to across) LLAP
failures on top of HIVE-14589 (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83ef6f92
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83ef6f92
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83ef6f92
Branch: refs/heads/hive-14535
Commit: 83ef6f9272d71e1918ffc89635709b4f81e8aba9
Parents: 4340d46
Author: Sergey Shelukhin
Authored: Mon Sep 19 16:11:16 2016 -0700
Committer: Sergey Shelukhin
Committed: Mon Sep 19 16:11:16 2016 -0700
--
.../hive/llap/registry/ServiceInstanceSet.java | 7 +-
.../registry/impl/InactiveServiceInstance.java | 77 ++
.../registry/impl/LlapFixedRegistryImpl.java| 2 +-
.../impl/LlapZookeeperRegistryImpl.java | 34 -
.../daemon/services/impl/LlapWebServices.java | 2 +-
.../tez/HostAffinitySplitLocationProvider.java | 80 +++---
.../apache/hadoop/hive/ql/exec/tez/Utils.java | 8 +-
.../TestHostAffinitySplitLocationProvider.java | 150 +++
.../apache/hadoop/hive/serde2/SerDeUtils.java | 11 ++
9 files changed, 306 insertions(+), 65 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/83ef6f92/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
--
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
index 13b668d..1e8c895 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
@@ -14,7 +14,6 @@
package org.apache.hadoop.hive.llap.registry;
import java.util.Collection;
-import java.util.List;
import java.util.Set;
public interface ServiceInstanceSet {
@@ -32,9 +31,11 @@ public interface ServiceInstanceSet {
/**
* Gets a list containing all the instances. This list has the same
iteration order across
* different processes, assuming the list of registry entries is the same.
- * @return
+ * @param consistentIndexes if true, also try to maintain the same exact
index for each node
+ * across calls, by inserting inactive instances to
replace the
+ * removed ones.
*/
- public Collection getAllInstancesOrdered();
+ public Collection getAllInstancesOrdered(boolean
consistentIndexes);
/**
* Get an instance by worker identity.
http://git-wip-us.apache.org/repos/asf/hive/blob/83ef6f92/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
--
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
new file mode 100644
index 000..79b7d51
--- /dev/null
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.registry.impl;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.llap.registry.ServiceInstance;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public class InactiveServiceInstance implements ServiceInstance {
+ private final String name;
+ public InactiveServiceInstance(String name) {
+this.name = name;
+ }
+
+ @Override
+ public String getWorkerIdentity() {
+return name;
+ }
+
+ @Override
+ public boolean isAlive() {
+return false;
+ }
+
+ @Override
+ public String getHost() {
+return null;
+ }
+
+ @Override
+ public int getRpcPort() {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getManagementPort() {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getShufflePort() {
+throw new UnsupportedOperationException();
+
hive git commit: HIVE-14680 : retain consistent splits /during/ (as opposed to across) LLAP failures on top of HIVE-14589 (Sergey Shelukhin, reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/master 4340d4619 -> 83ef6f927
HIVE-14680 : retain consistent splits /during/ (as opposed to across) LLAP
failures on top of HIVE-14589 (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83ef6f92
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83ef6f92
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83ef6f92
Branch: refs/heads/master
Commit: 83ef6f9272d71e1918ffc89635709b4f81e8aba9
Parents: 4340d46
Author: Sergey Shelukhin
Authored: Mon Sep 19 16:11:16 2016 -0700
Committer: Sergey Shelukhin
Committed: Mon Sep 19 16:11:16 2016 -0700
--
.../hive/llap/registry/ServiceInstanceSet.java | 7 +-
.../registry/impl/InactiveServiceInstance.java | 77 ++
.../registry/impl/LlapFixedRegistryImpl.java| 2 +-
.../impl/LlapZookeeperRegistryImpl.java | 34 -
.../daemon/services/impl/LlapWebServices.java | 2 +-
.../tez/HostAffinitySplitLocationProvider.java | 80 +++---
.../apache/hadoop/hive/ql/exec/tez/Utils.java | 8 +-
.../TestHostAffinitySplitLocationProvider.java | 150 +++
.../apache/hadoop/hive/serde2/SerDeUtils.java | 11 ++
9 files changed, 306 insertions(+), 65 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/83ef6f92/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
--
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
index 13b668d..1e8c895 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstanceSet.java
@@ -14,7 +14,6 @@
package org.apache.hadoop.hive.llap.registry;
import java.util.Collection;
-import java.util.List;
import java.util.Set;
public interface ServiceInstanceSet {
@@ -32,9 +31,11 @@ public interface ServiceInstanceSet {
/**
* Gets a list containing all the instances. This list has the same
iteration order across
* different processes, assuming the list of registry entries is the same.
- * @return
+ * @param consistentIndexes if true, also try to maintain the same exact
index for each node
+ * across calls, by inserting inactive instances to
replace the
+ * removed ones.
*/
- public Collection getAllInstancesOrdered();
+ public Collection getAllInstancesOrdered(boolean
consistentIndexes);
/**
* Get an instance by worker identity.
http://git-wip-us.apache.org/repos/asf/hive/blob/83ef6f92/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
--
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
new file mode 100644
index 000..79b7d51
--- /dev/null
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.registry.impl;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.llap.registry.ServiceInstance;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+public class InactiveServiceInstance implements ServiceInstance {
+ private final String name;
+ public InactiveServiceInstance(String name) {
+this.name = name;
+ }
+
+ @Override
+ public String getWorkerIdentity() {
+return name;
+ }
+
+ @Override
+ public boolean isAlive() {
+return false;
+ }
+
+ @Override
+ public String getHost() {
+return null;
+ }
+
+ @Override
+ public int getRpcPort() {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getManagementPort() {
+throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getShufflePort() {
+throw new UnsupportedOperationException();
+ }
+
+
