[
https://issues.apache.org/jira/browse/ARROW-855?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15975954#comment-15975954
]
Julien Le Dem edited comment on ARROW-855 at 4/25/17 10:09 PM:
---------------------------------------------------------------
1.abstract class
{code}
abstract class ArrowPillar[V <: ValueVector](val field: FieldDescription, val
allocator: BufferAllocator)
extends CountablePillar {
val vector: V = ArrowPillar.getNewVector(mField.getName, allocator,
mField.getType()).asInstanceOf[V]
initialSize match {
case Some(v) => vector.isInstanceOf[FixedWidthVector] match {
case true =>
vector.asInstanceOf[FixedWidthVector].allocateNew(v)
vector.asInstanceOf[V]
case false => vector.allocateNew()
}
case None => vector.allocateNew()
}
......
def mutator():ValueVector.Mutator = vector.getMutator
def accessor():ValueVector.Accessor = vector.getAccessor
override def pureDel(index: Int): Unit = {
throw new MemoException("Don't support this operation now.")
}
override def close(): Unit = vector.close()
override def clear(): Unit = vector.clear()
}
{code}
2.one implementation
{code}
class ArrowNullableStringPillar(override val field: FieldDescription, override
val allocator: BufferAllocator)
extends ArrowPillar[NullableVarCharVector](field, allocator) {
override def pureSet(index: Int, value: Option[Any]): Unit = {
value match {
case None =>
mutator.asInstanceOf[NullableVarCharVector#Mutator].setNull(index)
case Some(v) =>
val value = v.asInstanceOf[String].getBytes
val length = v.asInstanceOf[String].getBytes.length
mutator.asInstanceOf[NullableVarCharVector#Mutator].setSafe(index,
value, 0, length)
}
}
override def pureGet(index: Int): Option[Any] = {
if (accessor.isNull(index)) None
else
Some(accessor.asInstanceOf[NullableVarCharVector#Accessor].get(index))
}
}
{code}
was (Author: fushengxu):
1.abstract class
abstract class ArrowPillar[V <: ValueVector](val field: FieldDescription, val
allocator: BufferAllocator)
extends CountablePillar {
val vector: V = ArrowPillar.getNewVector(mField.getName, allocator,
mField.getType()).asInstanceOf[V]
initialSize match {
case Some(v) => vector.isInstanceOf[FixedWidthVector] match {
case true =>
vector.asInstanceOf[FixedWidthVector].allocateNew(v)
vector.asInstanceOf[V]
case false => vector.allocateNew()
}
case None => vector.allocateNew()
}
......
def mutator():ValueVector.Mutator = vector.getMutator
def accessor():ValueVector.Accessor = vector.getAccessor
override def pureDel(index: Int): Unit = {
throw new MemoException("Don't support this operation now.")
}
override def close(): Unit = vector.close()
override def clear(): Unit = vector.clear()
}
2.one implementation
class ArrowNullableStringPillar(override val field: FieldDescription, override
val allocator: BufferAllocator)
extends ArrowPillar[NullableVarCharVector](field, allocator) {
override def pureSet(index: Int, value: Option[Any]): Unit = {
value match {
case None =>
mutator.asInstanceOf[NullableVarCharVector#Mutator].setNull(index)
case Some(v) =>
val value = v.asInstanceOf[String].getBytes
val length = v.asInstanceOf[String].getBytes.length
mutator.asInstanceOf[NullableVarCharVector#Mutator].setSafe(index,
value, 0, length)
}
}
override def pureGet(index: Int): Option[Any] = {
if (accessor.isNull(index)) None
else
Some(accessor.asInstanceOf[NullableVarCharVector#Accessor].get(index))
}
}
> Arrow Memory Leak
> -----------------
>
> Key: ARROW-855
> URL: https://issues.apache.org/jira/browse/ARROW-855
> Project: Apache Arrow
> Issue Type: Bug
> Components: Java - Memory, Java - Vectors
> Affects Versions: 0.1.0
> Environment: CentOS release 6.7+Indellij IDEA
> Reporter: xufusheng
> Priority: Critical
> Labels: test
>
> we create a memory table by arrow and the source data come from HBase.
> Create a memory table and then drop the table,there will be a memory leak.
> Hundreds of times,There will be OutOfMemoryError.
> anyone encounter similar problems?
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
