hive git commit: HIVE-13392 disable speculative execution for ACID Compactor (Eugene Koifman, reviewed by Wei Zheng, Alan Gates)

2016-06-13 Thread ekoifman 
Repository: hive
Updated Branches:
  refs/heads/branch-1 39decb0bf -> 293e22e0e


HIVE-13392 disable speculative execution for ACID Compactor (Eugene Koifman, 
reviewed by Wei Zheng, Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/293e22e0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/293e22e0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/293e22e0

Branch: refs/heads/branch-1
Commit: 293e22e0eed47ec3f7e0ce4d981366c59b65455c
Parents: 39decb0
Author: Eugene Koifman 
Authored: Mon Jun 13 11:41:30 2016 -0700
Committer: Eugene Koifman 
Committed: Mon Jun 13 11:41:30 2016 -0700

--
 .../hive/common/ValidCompactorTxnList.java  | 111 +++
 .../hive/metastore/txn/CompactionInfo.java  |   1 +
 .../hadoop/hive/metastore/txn/TxnUtils.java |   1 +
 .../metastore/txn/ValidCompactorTxnList.java| 111 ---
 .../txn/TestValidCompactorTxnList.java  |   1 +
 .../hive/ql/txn/compactor/CompactorMR.java  |   8 +-
 .../apache/hadoop/hive/ql/io/TestAcidUtils.java |   2 +-
 7 files changed, 121 insertions(+), 114 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/293e22e0/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
--
diff --git 
a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java 
b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
new file mode 100644
index 000..ad79e2c
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hive.common.ValidReadTxnList;
+
+import java.util.Arrays;
+
+/**
+ * And implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} 
for use by the compactor.
+ * For the purposes of {@link #isTxnRangeValid} this class will view a 
transaction as valid if it
+ * is committed or aborted.  Additionally it will return none if there are any 
open transactions
+ * below the max transaction given, since we don't want to compact above open 
transactions.  For
+ * {@link #isTxnValid} it will still view a transaction as valid only if it is 
committed.  These
+ * produce the logic we need to assure that the compactor only sees records 
less than the lowest
+ * open transaction when choosing which files to compact, but that it still 
ignores aborted
+ * records when compacting.
+ */
+public class ValidCompactorTxnList extends ValidReadTxnList {
+  //TODO: refactor this - minOpenTxn is not needed if we set
+  // highWatermark = Math.min(highWaterMark, minOpenTxn) (assuming there are 
open txns)
+
+  // The minimum open transaction id
+  private long minOpenTxn;
+
+  public ValidCompactorTxnList() {
+super();
+minOpenTxn = -1;
+  }
+
+  /**
+   *
+   * @param exceptions list of all open and aborted transactions
+   * @param minOpen lowest open transaction
+   * @param highWatermark highest committed transaction
+   */
+  public ValidCompactorTxnList(long[] exceptions, long minOpen, long 
highWatermark) {
+super(exceptions, highWatermark);
+minOpenTxn = minOpen;
+  }
+
+  public ValidCompactorTxnList(String value) {
+super(value);
+  }
+
+  @Override
+  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
+if (highWatermark < minTxnId) {
+  return RangeResponse.NONE;
+} else if (minOpenTxn < 0) {
+  return highWatermark >= maxTxnId ? RangeResponse.ALL : 
RangeResponse.NONE;
+} else {
+  return minOpenTxn > maxTxnId ? RangeResponse.ALL : RangeResponse.NONE;
+}
+  }
+
+  @Override
+  public String writeToString() {
+StringBuilder buf = new StringBuilder();
+buf.append(highWatermark);
+buf.append(':');
+buf.append(minOpenTxn);
+if

hive git commit: HIVE-13392 disable speculative execution for ACID Compactor (Eugene Koifman, reviewed by Wei Zheng, Alan Gates)

2016-06-13 Thread ekoifman 
Repository: hive
Updated Branches:
  refs/heads/master 2f285aea0 -> 00e177614


HIVE-13392 disable speculative execution for ACID Compactor (Eugene Koifman, 
reviewed by Wei Zheng, Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/00e17761
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/00e17761
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/00e17761

Branch: refs/heads/master
Commit: 00e1776147b2e4d4025167210ff215694170c7d8
Parents: 2f285ae
Author: Eugene Koifman 
Authored: Mon Jun 13 11:37:56 2016 -0700
Committer: Eugene Koifman 
Committed: Mon Jun 13 11:38:26 2016 -0700

--
 .../hive/common/ValidCompactorTxnList.java  | 111 +++
 .../hive/metastore/txn/CompactionInfo.java  |   1 +
 .../hadoop/hive/metastore/txn/TxnUtils.java |   1 +
 .../metastore/txn/ValidCompactorTxnList.java| 111 ---
 .../txn/TestValidCompactorTxnList.java  |   1 +
 .../hive/ql/txn/compactor/CompactorMR.java  |   8 +-
 .../apache/hadoop/hive/ql/io/TestAcidUtils.java |   2 +-
 7 files changed, 121 insertions(+), 114 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/00e17761/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
--
diff --git 
a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java 
b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
new file mode 100644
index 000..ad79e2c
--- /dev/null
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hive.common.ValidReadTxnList;
+
+import java.util.Arrays;
+
+/**
+ * And implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} 
for use by the compactor.
+ * For the purposes of {@link #isTxnRangeValid} this class will view a 
transaction as valid if it
+ * is committed or aborted.  Additionally it will return none if there are any 
open transactions
+ * below the max transaction given, since we don't want to compact above open 
transactions.  For
+ * {@link #isTxnValid} it will still view a transaction as valid only if it is 
committed.  These
+ * produce the logic we need to assure that the compactor only sees records 
less than the lowest
+ * open transaction when choosing which files to compact, but that it still 
ignores aborted
+ * records when compacting.
+ */
+public class ValidCompactorTxnList extends ValidReadTxnList {
+  //TODO: refactor this - minOpenTxn is not needed if we set
+  // highWatermark = Math.min(highWaterMark, minOpenTxn) (assuming there are 
open txns)
+
+  // The minimum open transaction id
+  private long minOpenTxn;
+
+  public ValidCompactorTxnList() {
+super();
+minOpenTxn = -1;
+  }
+
+  /**
+   *
+   * @param exceptions list of all open and aborted transactions
+   * @param minOpen lowest open transaction
+   * @param highWatermark highest committed transaction
+   */
+  public ValidCompactorTxnList(long[] exceptions, long minOpen, long 
highWatermark) {
+super(exceptions, highWatermark);
+minOpenTxn = minOpen;
+  }
+
+  public ValidCompactorTxnList(String value) {
+super(value);
+  }
+
+  @Override
+  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
+if (highWatermark < minTxnId) {
+  return RangeResponse.NONE;
+} else if (minOpenTxn < 0) {
+  return highWatermark >= maxTxnId ? RangeResponse.ALL : 
RangeResponse.NONE;
+} else {
+  return minOpenTxn > maxTxnId ? RangeResponse.ALL : RangeResponse.NONE;
+}
+  }
+
+  @Override
+  public String writeToString() {
+StringBuilder buf = new StringBuilder();
+buf.append(highWatermark);
+buf.append(':');
+buf.append(minOpenTxn);
+if (exceptions.length