Repository: hive
Updated Branches:
  refs/heads/master f9768af0e -> cbb9233a3


http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
 
b/storage-api/src/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
new file mode 100644
index 0000000..107ea90
--- /dev/null
+++ 
b/storage-api/src/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * An implementation of {@link ValidWriteIdList} for use by readers.
+ * This class will view a write id as valid only if it maps to committed 
transaction.
+ * Write ids of both open and aborted transactions will be seen as invalid.
+ */
+public class ValidReaderWriteIdList implements ValidWriteIdList {
+
+  private String tableName; // Full table name of format <db_name>.<table_name>
+  protected long[] exceptions;
+  protected BitSet abortedBits; // BitSet for flagging aborted write ids. Bit 
is true if aborted, false if open
+  //default value means there are no open write ids in the snapshot
+  private long minOpenWriteId = Long.MAX_VALUE;
+  protected long highWatermark;
+
+  public ValidReaderWriteIdList() {
+    this(null, new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE);
+  }
+
+  /**
+   * Used if there are no open write ids in the snapshot.
+   */
+  public ValidReaderWriteIdList(String tableName, long[] exceptions, BitSet 
abortedBits, long highWatermark) {
+    this(tableName, exceptions, abortedBits, highWatermark, Long.MAX_VALUE);
+  }
+  public ValidReaderWriteIdList(String tableName,
+                                long[] exceptions, BitSet abortedBits, long 
highWatermark, long minOpenWriteId) {
+    this.tableName = tableName;
+    if (exceptions.length > 0) {
+      this.minOpenWriteId = minOpenWriteId;
+    }
+    this.exceptions = exceptions;
+    this.abortedBits = abortedBits;
+    this.highWatermark = highWatermark;
+  }
+
+  public ValidReaderWriteIdList(String value) {
+    readFromString(value);
+  }
+
+  @Override
+  public boolean isWriteIdValid(long writeId) {
+    if (writeId > highWatermark) {
+      return false;
+    }
+    return Arrays.binarySearch(exceptions, writeId) < 0;
+  }
+
+  /**
+   * We cannot use a base file if its range contains an open write id.
+   * @param writeId from base_xxxx
+   */
+  @Override
+  public boolean isValidBase(long writeId) {
+    return (writeId < minOpenWriteId) && (writeId <= highWatermark);
+  }
+  @Override
+  public RangeResponse isWriteIdRangeValid(long minWriteId, long maxWriteId) {
+    // check the easy cases first
+    if (minWriteId > highWatermark) {
+      return RangeResponse.NONE;
+    } else if (exceptions.length > 0 && exceptions[0] > maxWriteId) {
+      return RangeResponse.ALL;
+    }
+
+    // since the exceptions and the range in question overlap, count the
+    // exceptions in the range
+    long count = Math.max(0, maxWriteId - highWatermark);
+    for(long txn: exceptions) {
+      if (minWriteId <= txn && txn <= maxWriteId) {
+        count += 1;
+      }
+    }
+
+    if (count == 0) {
+      return RangeResponse.ALL;
+    } else if (count == (maxWriteId - minWriteId + 1)) {
+      return RangeResponse.NONE;
+    } else {
+      return RangeResponse.SOME;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return writeToString();
+  }
+
+  // Format is 
<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>
+  @Override
+  public String writeToString() {
+    StringBuilder buf = new StringBuilder();
+    if (tableName == null) {
+      buf.append("null");
+    } else {
+      buf.append(tableName);
+    }
+    buf.append(':');
+    buf.append(highWatermark);
+    buf.append(':');
+    buf.append(minOpenWriteId);
+    if (exceptions.length == 0) {
+      buf.append(':');  // separator for open write ids
+      buf.append(':');  // separator for aborted write ids
+    } else {
+      StringBuilder open = new StringBuilder();
+      StringBuilder abort = new StringBuilder();
+      for (int i = 0; i < exceptions.length; i++) {
+        if (abortedBits.get(i)) {
+          if (abort.length() > 0) {
+            abort.append(',');
+          }
+          abort.append(exceptions[i]);
+        } else {
+          if (open.length() > 0) {
+            open.append(',');
+          }
+          open.append(exceptions[i]);
+        }
+      }
+      buf.append(':');
+      buf.append(open);
+      buf.append(':');
+      buf.append(abort);
+    }
+    return buf.toString();
+  }
+
+  @Override
+  public void readFromString(String src) {
+    if (src == null || src.length() == 0) {
+      highWatermark = Long.MAX_VALUE;
+      exceptions = new long[0];
+      abortedBits = new BitSet();
+    } else {
+      String[] values = src.split(":");
+      tableName = values[0];
+      if (tableName.equalsIgnoreCase("null")) {
+        tableName = null;
+      }
+      highWatermark = Long.parseLong(values[1]);
+      minOpenWriteId = Long.parseLong(values[2]);
+      String[] openWriteIds = new String[0];
+      String[] abortedWriteIds = new String[0];
+      if (values.length < 4) {
+        openWriteIds = new String[0];
+        abortedWriteIds = new String[0];
+      } else if (values.length == 4) {
+        if (!values[3].isEmpty()) {
+          openWriteIds = values[3].split(",");
+        }
+      } else {
+        if (!values[3].isEmpty()) {
+          openWriteIds = values[3].split(",");
+        }
+        if (!values[4].isEmpty()) {
+          abortedWriteIds = values[4].split(",");
+        }
+      }
+      exceptions = new long[openWriteIds.length + abortedWriteIds.length];
+      int i = 0;
+      for (String open : openWriteIds) {
+        exceptions[i++] = Long.parseLong(open);
+      }
+      for (String abort : abortedWriteIds) {
+        exceptions[i++] = Long.parseLong(abort);
+      }
+      Arrays.sort(exceptions);
+      abortedBits = new BitSet(exceptions.length);
+      for (String abort : abortedWriteIds) {
+        int index = Arrays.binarySearch(exceptions, Long.parseLong(abort));
+        abortedBits.set(index);
+      }
+    }
+  }
+
+  @Override
+  public String getTableName() {
+    return tableName;
+  }
+
+  @Override
+  public long getHighWatermark() {
+    return highWatermark;
+  }
+
+  @Override
+  public long[] getInvalidWriteIds() {
+    return exceptions;
+  }
+
+  @Override
+  public Long getMinOpenWriteId() {
+    return minOpenWriteId == Long.MAX_VALUE ? null : minOpenWriteId;
+  }
+
+  @Override
+  public boolean isWriteIdAborted(long writeId) {
+    int index = Arrays.binarySearch(exceptions, writeId);
+    return index >= 0 && abortedBits.get(index);
+  }
+
+  @Override
+  public RangeResponse isWriteIdRangeAborted(long minWriteId, long maxWriteId) 
{
+    // check the easy cases first
+    if (highWatermark < minWriteId) {
+      return RangeResponse.NONE;
+    }
+
+    int count = 0;  // number of aborted txns found in exceptions
+
+    // traverse the aborted txns list, starting at first aborted txn index
+    for (int i = abortedBits.nextSetBit(0); i >= 0; i = 
abortedBits.nextSetBit(i + 1)) {
+      long abortedTxnId = exceptions[i];
+      if (abortedTxnId > maxWriteId) {  // we've already gone beyond the 
specified range
+        break;
+      }
+      if (abortedTxnId >= minWriteId && abortedTxnId <= maxWriteId) {
+        count++;
+      }
+    }
+
+    if (count == 0) {
+      return RangeResponse.NONE;
+    } else if (count == (maxWriteId - minWriteId + 1)) {
+      return RangeResponse.ALL;
+    } else {
+      return RangeResponse.SOME;
+    }
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java 
b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
index 3ffb2d8..d4c3b09 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
@@ -47,13 +47,6 @@ public interface ValidTxnList {
   public boolean isTxnValid(long txnid);
 
   /**
-   * Returns {@code true} if such base file can be used to materialize the 
snapshot represented by
-   * this {@code ValidTxnList}.
-   * @param txnid highest txn in a given base_xxxx file
-   */
-  public boolean isValidBase(long txnid);
-
-  /**
    * Find out if a range of transaction ids are valid.  Note that valid may 
have different meanings
    * for different implementations, as some will only want to see committed 
transactions and some
    * both committed and aborted.

http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java 
b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
new file mode 100644
index 0000000..9867a81
--- /dev/null
+++ 
b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An implementation to store and manage list of ValidWriteIds for each tables 
read by current
+ * transaction.
+ */
+public class ValidTxnWriteIdList {
+  /**
+   * Key used to store valid write id list for all the operated tables in a
+   * {@link org.apache.hadoop.conf.Configuration} object.
+   */
+  public static final String VALID_TABLES_WRITEIDS_KEY = 
"hive.txn.tables.valid.writeids";
+
+  // Transaction for which the list of tables valid write Ids are populated
+  private Long txnId;
+
+  // Map of valid write ids list for all the tables read by the current txn
+  // Key is full table name string of format <db_name>.<table_name>
+  private Map<String, ValidWriteIdList> tablesValidWriteIdList = new 
HashMap<>();
+  public ValidTxnWriteIdList(Long txnId) {
+    this.txnId = txnId;
+  }
+
+  public ValidTxnWriteIdList(String value) {
+    readFromString(value);
+  }
+
+  @Override
+  public String toString() {
+    return writeToString();
+  }
+
+  public void addTableValidWriteIdList(ValidWriteIdList validWriteIds) {
+    tablesValidWriteIdList.put(validWriteIds.getTableName(), validWriteIds);
+  }
+
+  // Input fullTableName is of format <db_name>.<table_name>
+  public ValidWriteIdList getTableValidWriteIdList(String fullTableName) {
+    if (tablesValidWriteIdList.containsKey(fullTableName)) {
+      return tablesValidWriteIdList.get(fullTableName);
+    }
+    return null;
+  }
+
+  // Each ValidWriteIdList is separated with "$" and each one maps to one table
+  // Format 
<txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name>...
+  private void readFromString(String src) {
+    if ((src == null) || (src.length() == 0)) {
+      return;
+    }
+    String[] tblWriteIdStrList = src.split("\\$");
+    assert(tblWriteIdStrList.length >= 1);
+
+    // First $ separated substring would be txnId and the rest are 
ValidReaderWriteIdList
+    this.txnId = Long.parseLong(tblWriteIdStrList[0]);
+    for (int index = 1; index < tblWriteIdStrList.length; index++) {
+      String tableStr = tblWriteIdStrList[index];
+      ValidWriteIdList validWriteIdList = new ValidReaderWriteIdList(tableStr);
+      addTableValidWriteIdList(validWriteIdList);
+    }
+  }
+
+  // Each ValidWriteIdList is separated with "$" and each one maps to one table
+  // Format 
<txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name>...
+  private String writeToString() {
+    // First $ separated substring will be txnId and the rest are 
ValidReaderWriteIdList
+    StringBuilder buf = new StringBuilder(txnId.toString());
+    int index = 0;
+    for (HashMap.Entry<String, ValidWriteIdList> entry : 
tablesValidWriteIdList.entrySet()) {
+      if (index < tablesValidWriteIdList.size()) {
+        buf.append('$');
+      }
+      buf.append(entry.getValue().writeToString());
+
+      // Separator for multiple tables' ValidWriteIdList. Also, skip it for 
last entry.
+      index++;
+    }
+    return buf.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/java/org/apache/hadoop/hive/common/ValidWriteIdList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/common/ValidWriteIdList.java 
b/storage-api/src/java/org/apache/hadoop/hive/common/ValidWriteIdList.java
new file mode 100644
index 0000000..be83134
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidWriteIdList.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+/**
+ * Models the list of transactions that should be included in a snapshot.
+ * It is modelled as a high water mark, which is the largest transaction id 
that
+ * has been committed and a list of transactions that are not included.
+ */
+public interface ValidWriteIdList {
+
+  /**
+   * Key used to store valid write id list in a
+   * {@link org.apache.hadoop.conf.Configuration} object.
+   */
+  String VALID_WRITEIDS_KEY = "hive.txn.valid.writeids";
+
+  /**
+   * The response to a range query.  NONE means no values in this range match,
+   * SOME mean that some do, and ALL means that every value does.
+   */
+  enum RangeResponse {NONE, SOME, ALL};
+
+  /**
+   * Indicates whether a given write ID is valid. Note that valid may have 
different meanings
+   * for different implementations, as some will only want to see committed 
transactions and some
+   * both committed and aborted.
+   * @param writeId write ID of the table
+   * @return true if valid, false otherwise
+   */
+  boolean isWriteIdValid(long writeId);
+
+  /**
+   * Returns {@code true} if such base file can be used to materialize the 
snapshot represented by
+   * this {@code ValidWriteIdList}.
+   * @param writeId highest write ID in a given base_xxxx file
+   */
+  boolean isValidBase(long writeId);
+
+  /**
+   * Find out if a range of write ids are valid.  Note that valid may have 
different meanings
+   * for different implementations, as some will only want to see committed 
transactions and some
+   * both committed and aborted.
+   * @param minWriteId minimum write ID to look for, inclusive
+   * @param maxWriteId maximum write ID to look for, inclusive
+   * @return Indicate whether none, some, or all of these transactions are 
valid.
+   */
+  RangeResponse isWriteIdRangeValid(long minWriteId, long maxWriteId);
+
+  /**
+   * Write this ValidWriteIdList into a string. This should produce a string 
that
+   * can be used by {@link #readFromString(String)} to populate a 
ValidWriteIdList.
+   */
+  String writeToString();
+
+  /**
+   * Populate this ValidWriteIdList from the string.  It is assumed that the 
string
+   * was created via {@link #writeToString()} and the exceptions list is 
sorted.
+   * @param src source string.
+   */
+  void readFromString(String src);
+
+  /**
+   * Get the table for which the ValidWriteIdList is formed
+   * @return table name (<db_name>.<table_name>) associated with 
ValidWriteIdList.
+   */
+  String getTableName();
+
+  /**
+   * Get the largest write id used.
+   * @return largest write id used
+   */
+  long getHighWatermark();
+
+  /**
+   * Get the list of write ids under the high water mark that are not valid.  
Note that invalid
+   * may have different meanings for different implementations, as some will 
only want to see open
+   * transactions and some both open and aborted.
+   * @return a list of invalid write ids
+   */
+  long[] getInvalidWriteIds();
+
+  /**
+   * Indicates whether a given write maps to aborted transaction.
+   * @param writeId write id to be validated
+   * @return true if aborted, false otherwise
+   */
+  boolean isWriteIdAborted(long writeId);
+
+  /**
+   * Find out if a range of write ids are aborted.
+   * @param minWriteId minimum write Id to look for, inclusive
+   * @param maxWriteId maximum write Id  to look for, inclusive
+   * @return Indicate whether none, some, or all of these write ids are 
aborted.
+   */
+  RangeResponse isWriteIdRangeAborted(long minWriteId, long maxWriteId);
+
+  /**
+   * Returns smallest Open write Id in this set, {@code null} if there is none.
+   */
+  Long getMinOpenWriteId();
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
 
b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
deleted file mode 100644
index 867b652..0000000
--- 
a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.BitSet;
-
-public class TestValidCompactorTxnList {
-
-  @Test
-  public void minTxnHigh() {
-    BitSet bitSet = new BitSet(2);
-    bitSet.set(0, 2);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 4}, bitSet, 2);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void maxTxnLow() {
-    BitSet bitSet = new BitSet(2);
-    bitSet.set(0, 2);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{13, 14}, bitSet, 
12);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-  }
-
-  @Test
-  public void minTxnHighNoExceptions() {
-    ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 
5);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void maxTxnLowNoExceptions() {
-    ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 
15);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-  }
-
-  @Test
-  public void exceptionsAllBelow() {
-    BitSet bitSet = new BitSet(2);
-    bitSet.set(0, 2);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 6}, bitSet, 3);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void exceptionsInMidst() {
-    BitSet bitSet = new BitSet(1);
-    bitSet.set(0, 1);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{8}, bitSet, 7);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-  @Test
-  public void exceptionsAbveHighWaterMark() {
-    BitSet bitSet = new BitSet(4);
-    bitSet.set(0, 4);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{8, 11, 17, 29}, 
bitSet, 15);
-    Assert.assertArrayEquals("", new long[]{8, 11}, 
txns.getInvalidTransactions());
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-    rsp = txns.isTxnRangeValid(12, 16);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void writeToString() {
-    BitSet bitSet = new BitSet(4);
-    bitSet.set(0, 4);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{7, 9, 10, 
Long.MAX_VALUE}, bitSet, 8);
-    Assert.assertEquals("8:" + Long.MAX_VALUE + "::7", txns.writeToString());
-    txns = new ValidCompactorTxnList();
-    Assert.assertEquals(Long.toString(Long.MAX_VALUE) + ":" + Long.MAX_VALUE + 
"::", txns.writeToString());
-    txns = new ValidCompactorTxnList(new long[0], new BitSet(), 23);
-    Assert.assertEquals("23:" + Long.MAX_VALUE + "::", txns.writeToString());
-  }
-
-  @Test
-  public void readFromString() {
-    ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + 
Long.MAX_VALUE + "::7,9,10");
-    Assert.assertEquals(37L, txns.getHighWatermark());
-    Assert.assertNull(txns.getMinOpenTxn());
-    Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, 
txns.getInvalidTransactions());
-    txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":");
-    Assert.assertEquals(21L, txns.getHighWatermark());
-    Assert.assertNull(txns.getMinOpenTxn());
-    Assert.assertEquals(0, txns.getInvalidTransactions().length);
-  }
-
-  @Test
-  public void testAbortedTxn() throws Exception {
-    ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3");
-    Assert.assertEquals(5L, txnList.getHighWatermark());
-    Assert.assertEquals(4, txnList.getMinOpenTxn().longValue());
-    Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, 
txnList.getInvalidTransactions());
-  }
-
-  @Test
-  public void testAbortedRange() throws Exception {
-    ValidCompactorTxnList txnList = new ValidCompactorTxnList("11:4::5,6,7,8");
-    ValidTxnList.RangeResponse rsp = txnList.isTxnRangeAborted(1L, 3L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-    rsp = txnList.isTxnRangeAborted(9L, 10L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-    rsp = txnList.isTxnRangeAborted(6L, 7L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-    rsp = txnList.isTxnRangeAborted(4L, 6L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp);
-    rsp = txnList.isTxnRangeAborted(6L, 13L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorWriteIdList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorWriteIdList.java
 
b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorWriteIdList.java
new file mode 100644
index 0000000..f378b7b
--- /dev/null
+++ 
b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorWriteIdList.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.BitSet;
+
+/**
+ * Tests for {@link ValidCompactorWriteIdList}.
+ */
+public class TestValidCompactorWriteIdList {
+  private final String tableName = "t1";
+
+  @Test
+  public void minTxnHigh() {
+    BitSet bitSet = new BitSet(2);
+    bitSet.set(0, 2);
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[]{3, 4}, bitSet, 2);
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void maxTxnLow() {
+    BitSet bitSet = new BitSet(2);
+    bitSet.set(0, 2);
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[]{13, 14}, bitSet, 12);
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.ALL, rsp);
+  }
+
+  @Test
+  public void minTxnHighNoExceptions() {
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[0], new BitSet(), 5);
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void maxTxnLowNoExceptions() {
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[0], new BitSet(), 15);
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.ALL, rsp);
+  }
+
+  @Test
+  public void exceptionsAllBelow() {
+    BitSet bitSet = new BitSet(2);
+    bitSet.set(0, 2);
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[]{3, 6}, bitSet, 3);
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void exceptionsInMidst() {
+    BitSet bitSet = new BitSet(1);
+    bitSet.set(0, 1);
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[]{8}, bitSet, 7);
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+  }
+  @Test
+  public void exceptionsAbveHighWaterMark() {
+    BitSet bitSet = new BitSet(4);
+    bitSet.set(0, 4);
+    ValidWriteIdList writeIds = new ValidCompactorWriteIdList(tableName, new 
long[]{8, 11, 17, 29}, bitSet, 15);
+    Assert.assertArrayEquals("", new long[]{8, 11}, 
writeIds.getInvalidWriteIds());
+    ValidWriteIdList.RangeResponse rsp = writeIds.isWriteIdRangeValid(7, 9);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.ALL, rsp);
+    rsp = writeIds.isWriteIdRangeValid(12, 16);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void writeToString() {
+    BitSet bitSet = new BitSet(4);
+    bitSet.set(0, 4);
+    ValidWriteIdList writeIds
+            = new ValidCompactorWriteIdList(tableName, new long[]{7, 9, 10, 
Long.MAX_VALUE}, bitSet, 8);
+    Assert.assertEquals(tableName + ":8:" + Long.MAX_VALUE + "::7", 
writeIds.writeToString());
+    writeIds = new ValidCompactorWriteIdList();
+    Assert.assertEquals("null:" + Long.toString(Long.MAX_VALUE) + ":" + 
Long.MAX_VALUE + "::",
+            writeIds.writeToString());
+    writeIds = new ValidCompactorWriteIdList(tableName, new long[0], new 
BitSet(), 23);
+    Assert.assertEquals(tableName + ":23:" + Long.MAX_VALUE + "::", 
writeIds.writeToString());
+  }
+
+  @Test
+  public void readFromString() {
+    ValidCompactorWriteIdList writeIds
+            = new ValidCompactorWriteIdList(tableName + ":37:" + 
Long.MAX_VALUE + "::7,9,10");
+    Assert.assertEquals(tableName, writeIds.getTableName());
+    Assert.assertEquals(37L, writeIds.getHighWatermark());
+    Assert.assertNull(writeIds.getMinOpenWriteId());
+    Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, 
writeIds.getInvalidWriteIds());
+    writeIds = new ValidCompactorWriteIdList(tableName + ":21:" + 
Long.MAX_VALUE + ":");
+    Assert.assertEquals(21L, writeIds.getHighWatermark());
+    Assert.assertNull(writeIds.getMinOpenWriteId());
+    Assert.assertEquals(0, writeIds.getInvalidWriteIds().length);
+  }
+
+  @Test
+  public void testAbortedTxn() throws Exception {
+    ValidCompactorWriteIdList writeIdList = new 
ValidCompactorWriteIdList(tableName + ":5:4::1,2,3");
+    Assert.assertEquals(5L, writeIdList.getHighWatermark());
+    Assert.assertEquals(4, writeIdList.getMinOpenWriteId().longValue());
+    Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, 
writeIdList.getInvalidWriteIds());
+  }
+
+  @Test
+  public void testAbortedRange() throws Exception {
+    ValidCompactorWriteIdList writeIdList = new 
ValidCompactorWriteIdList(tableName + ":11:4::5,6,7,8");
+    ValidWriteIdList.RangeResponse rsp = writeIdList.isWriteIdRangeAborted(1L, 
3L);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+    rsp = writeIdList.isWriteIdRangeAborted(9L, 10L);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.NONE, rsp);
+    rsp = writeIdList.isWriteIdRangeAborted(6L, 7L);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.ALL, rsp);
+    rsp = writeIdList.isWriteIdRangeAborted(4L, 6L);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.SOME, rsp);
+    rsp = writeIdList.isWriteIdRangeAborted(6L, 13L);
+    Assert.assertEquals(ValidWriteIdList.RangeResponse.SOME, rsp);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReaderWriteIdList.java
----------------------------------------------------------------------
diff --git 
a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReaderWriteIdList.java
 
b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReaderWriteIdList.java
new file mode 100644
index 0000000..18f6813
--- /dev/null
+++ 
b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReaderWriteIdList.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.BitSet;
+
+/**
+ * Tests for {@link ValidReaderWriteIdList}.
+ */
+public class TestValidReaderWriteIdList {
+  private final String tableName = "t1";
+
+  @Test
+  public void noExceptions() throws Exception {
+    ValidWriteIdList writeIdList = new ValidReaderWriteIdList(tableName, new 
long[0], new BitSet(), 1, Long.MAX_VALUE);
+    String str = writeIdList.writeToString();
+    TestCase.assertEquals(tableName + ":1:" + Long.MAX_VALUE + "::", str);
+    ValidWriteIdList newList = new ValidReaderWriteIdList();
+    newList.readFromString(str);
+    TestCase.assertTrue(newList.isWriteIdValid(1));
+    TestCase.assertFalse(newList.isWriteIdValid(2));
+  }
+
+  @Test
+  public void exceptions() throws Exception {
+    ValidWriteIdList writeIdList = new ValidReaderWriteIdList(tableName, new 
long[]{2L, 4L}, new BitSet(), 5, 4L);
+    String str = writeIdList.writeToString();
+    TestCase.assertEquals(tableName + ":5:4:2,4:", str);
+    ValidWriteIdList newList = new ValidReaderWriteIdList();
+    newList.readFromString(str);
+    TestCase.assertTrue(newList.isWriteIdValid(1));
+    TestCase.assertFalse(newList.isWriteIdValid(2));
+    TestCase.assertTrue(newList.isWriteIdValid(3));
+    TestCase.assertFalse(newList.isWriteIdValid(4));
+    TestCase.assertTrue(newList.isWriteIdValid(5));
+    TestCase.assertFalse(newList.isWriteIdValid(6));
+  }
+
+  @Test
+  public void longEnoughToCompress() throws Exception {
+    long[] exceptions = new long[1000];
+    for (int i = 0; i < 1000; i++) {
+      exceptions[i] = i + 100;
+    }
+    ValidWriteIdList writeIdList = new ValidReaderWriteIdList(tableName, 
exceptions, new BitSet(), 2000, 900);
+    String str = writeIdList.writeToString();
+    ValidWriteIdList newList = new ValidReaderWriteIdList();
+    newList.readFromString(str);
+    for (int i = 0; i < 100; i++) {
+      TestCase.assertTrue(newList.isWriteIdValid(i));
+    }
+    for (int i = 100; i < 1100; i++) {
+      TestCase.assertFalse(newList.isWriteIdValid(i));
+    }
+    for (int i = 1100; i < 2001; i++) {
+      TestCase.assertTrue(newList.isWriteIdValid(i));
+    }
+    TestCase.assertFalse(newList.isWriteIdValid(2001));
+  }
+
+  @Test
+  public void readWriteConfig() throws Exception {
+    long[] exceptions = new long[1000];
+    for (int i = 0; i < 1000; i++) {
+      exceptions[i] = i + 100;
+    }
+    ValidWriteIdList writeIdList = new ValidReaderWriteIdList(tableName, 
exceptions, new BitSet(), 2000, 900);
+    String str = writeIdList.writeToString();
+    Configuration conf = new Configuration();
+    conf.set(ValidWriteIdList.VALID_WRITEIDS_KEY, str);
+    File tmpFile = File.createTempFile("TestValidTxnImpl", "readWriteConfig");
+    DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile));
+    conf.write(out);
+    out.close();
+    DataInputStream in = new DataInputStream(new FileInputStream(tmpFile));
+    Configuration newConf = new Configuration();
+    newConf.readFields(in);
+    TestCase.assertEquals(str, 
newConf.get(ValidWriteIdList.VALID_WRITEIDS_KEY));
+  }
+
+  @Test
+  public void testAbortedTxn() throws Exception {
+    long[] exceptions = {2L, 4L, 6L, 8L, 10L};
+    BitSet bitSet = new BitSet(exceptions.length);
+    bitSet.set(0);  // mark txn "2L" aborted
+    bitSet.set(3);  // mark txn "8L" aborted
+    ValidWriteIdList writeIdList = new ValidReaderWriteIdList(tableName, 
exceptions, bitSet, 11, 4L);
+    String str = writeIdList.writeToString();
+    TestCase.assertEquals(tableName + ":11:4:4,6,10:2,8", str);
+    TestCase.assertTrue(writeIdList.isWriteIdAborted(2L));
+    TestCase.assertFalse(writeIdList.isWriteIdAborted(4L));
+    TestCase.assertFalse(writeIdList.isWriteIdAborted(6L));
+    TestCase.assertTrue(writeIdList.isWriteIdAborted(8L));
+    TestCase.assertFalse(writeIdList.isWriteIdAborted(10L));
+  }
+}

Reply via email to