DRILL-5351: Minimize bounds checking in var len vectors for Parquet reader close #781
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/152c87aa Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/152c87aa Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/152c87aa Branch: refs/heads/master Commit: 152c87aa6cad84c8752b4a87967c7826cc90dbaa Parents: 0ded1d0 Author: Parth Chandra <pchan...@maprtech.com> Authored: Fri Feb 10 17:40:25 2017 -0800 Committer: Jinfeng Ni <j...@apache.org> Committed: Fri Mar 31 15:32:13 2017 -0700 ---------------------------------------------------------------------- .../codegen/templates/NullableValueVectors.java | 24 +++++-- .../templates/VariableLengthVectors.java | 72 +++++++++++++------- 2 files changed, 65 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/152c87aa/exec/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java index b242728..fcbe79a 100644 --- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java @@ -509,7 +509,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major != "VarLen"> throw new UnsupportedOperationException(); <#else> - fillEmpties(index); + if (index > lastSet + 1) { + fillEmpties(index); + } bits.getMutator().setSafe(index, 1); values.getMutator().setSafe(index, value, start, length); @@ -522,7 +524,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major != "VarLen"> throw new UnsupportedOperationException(); <#else> - fillEmpties(index); + if (index > lastSet + 1) { + fillEmpties(index); + } bits.getMutator().setSafe(index, 1); values.getMutator().setSafe(index, value, start, length); @@ -587,7 +591,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type public void setSafe(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) { <#if type.major == "VarLen"> - fillEmpties(index); + if (index > lastSet + 1) { + fillEmpties(index); + } </#if> bits.getMutator().setSafe(index, isSet); @@ -600,7 +606,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type public void setSafe(int index, Nullable${minor.class}Holder value) { <#if type.major == "VarLen"> - fillEmpties(index); + if (index > lastSet + 1) { + fillEmpties(index); + } </#if> bits.getMutator().setSafe(index, value.isSet); values.getMutator().setSafe(index, value); @@ -611,7 +619,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type public void setSafe(int index, ${minor.class}Holder value) { <#if type.major == "VarLen"> - fillEmpties(index); + if (index > lastSet + 1) { + fillEmpties(index); + } </#if> bits.getMutator().setSafe(index, 1); values.getMutator().setSafe(index, value); @@ -622,7 +632,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { <#if type.major == "VarLen"> - fillEmpties(index); + if (index > lastSet + 1) { + fillEmpties(index); + } </#if> bits.getMutator().setSafe(index, 1); values.getMutator().setSafe(index, value); http://git-wip-us.apache.org/repos/asf/drill/blob/152c87aa/exec/vector/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java index bb1d4fb..e970271 100644 --- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -507,11 +507,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V assert index >= 0; final int currentOffset = offsetVector.getAccessor().get(index); - while (data.capacity() < currentOffset + bytes.length) { - reAlloc(); - } offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length); - data.setBytes(currentOffset, bytes, 0, bytes.length); + try { + data.setBytes(currentOffset, bytes, 0, bytes.length); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < currentOffset + bytes.length) { + reAlloc(); + } + data.setBytes(currentOffset, bytes, 0, bytes.length); + } } /** @@ -533,12 +537,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V assert index >= 0; int currentOffset = offsetVector.getAccessor().get(index); - - while (data.capacity() < currentOffset + length) { - reAlloc(); - } offsetVector.getMutator().setSafe(index + 1, currentOffset + length); - data.setBytes(currentOffset, bytes, start, length); + try { + data.setBytes(currentOffset, bytes, start, length); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < currentOffset + length) { + reAlloc(); + } + data.setBytes(currentOffset, bytes, start, length); + } } public void setSafe(int index, byte[] bytes, int start, int length) { @@ -546,11 +553,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V final int currentOffset = offsetVector.getAccessor().get(index); - while (data.capacity() < currentOffset + length) { - reAlloc(); - } offsetVector.getMutator().setSafe(index + 1, currentOffset + length); - data.setBytes(currentOffset, bytes, start, length); + try { + data.setBytes(currentOffset, bytes, start, length); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < currentOffset + length) { + reAlloc(); + } + data.setBytes(currentOffset, bytes, start, length); + } } @Override @@ -567,12 +578,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V final int len = end - start; final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - while(data.capacity() < outputStart + len) { - reAlloc(); + offsetVector.getMutator().setSafe( index+1, outputStart + len); + try{ + buffer.getBytes(start, data, outputStart, len); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < outputStart + len) { + reAlloc(); + } + buffer.getBytes(start, data, outputStart, len); } - offsetVector.getMutator().setSafe( index+1, outputStart + len); - buffer.getBytes(start, data, outputStart, len); } public void setSafe(int index, Nullable${minor.class}Holder holder){ @@ -584,11 +599,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - while(data.capacity() < outputStart + len) { - reAlloc(); + try { + holder.buffer.getBytes(start, data, outputStart, len); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < outputStart + len) { + reAlloc(); + } + holder.buffer.getBytes(start, data, outputStart, len); } - - holder.buffer.getBytes(start, data, outputStart, len); offsetVector.getMutator().setSafe( index+1, outputStart + len); } @@ -598,11 +616,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V final int len = end - start; final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - while(data.capacity() < outputStart + len) { - reAlloc(); - } - holder.buffer.getBytes(start, data, outputStart, len); + try { + holder.buffer.getBytes(start, data, outputStart, len); + } catch (IndexOutOfBoundsException e) { + while(data.capacity() < outputStart + len) { + reAlloc(); + } + holder.buffer.getBytes(start, data, outputStart, len); + } offsetVector.getMutator().setSafe( index+1, outputStart + len); }