[GitHub] incubator-hawq pull request #1360: HAWQ-1607. This commit implements applyin...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1360#discussion_r186348737
  
--- Diff: src/backend/cdb/cdbparquetrowgroup.c ---
@@ -206,54 +207,54 @@ ParquetRowGroupReader_ScanNextTuple(
ParquetRowGroupReader   *rowGroupReader,
int 
*hawqAttrToParquetColNum,
bool*projs,
+   RuntimeFilterState  *rfState,
TupleTableSlot  *slot)
 {
Assert(slot);
-
-   if (rowGroupReader->rowRead >= rowGroupReader->rowCount)
+   while (rowGroupReader->rowRead < rowGroupReader->rowCount)
{
-   ParquetRowGroupReader_FinishedScanRowGroup(rowGroupReader);
-   return false;
-   }
-
-   /*
-* get the next item (tuple) from the row group
-*/
-   rowGroupReader->rowRead++;
 
-   int natts = slot->tts_tupleDescriptor->natts;
-   Assert(natts <= tupDesc->natts);
-
-   Datum *values = slot_get_values(slot);
-   bool *nulls = slot_get_isnull(slot);
+   /*
+* get the next item (tuple) from the row group
+*/
+   rowGroupReader->rowRead++;
 
-   int colReaderIndex = 0;
-   for(int i = 0; i < natts; i++)
-   {
-   if(projs[i] == false)
-   {
-   nulls[i] = true;
-   continue;
-   }
+   int natts = slot->tts_tupleDescriptor->natts;
+   Assert(natts <= tupDesc->natts);
 
-   ParquetColumnReader *nextReader =
-   >columnReaders[colReaderIndex];
-   int hawqTypeID = tupDesc->attrs[i]->atttypid;
+   Datum *values = slot_get_values(slot);
+   bool *nulls = slot_get_isnull(slot);
 
-   if(hawqAttrToParquetColNum[i] == 1)
+   int colReaderIndex = 0;
+   int16 proj[128];
--- End diff --

It's better to use natts instead of 128.


---

[GitHub] incubator-hawq pull request #1360: HAWQ-1607. This commit implements applyin...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1360#discussion_r186347065
  
--- Diff: src/include/nodes/execnodes.h ---
@@ -1522,6 +1540,9 @@ typedef struct ScanState
/* The type of the table that is being scanned */
TableType tableType;
 
+   /* Runtime filter */
+   struct RuntimeFilterState runtimeFilter;
--- End diff --

Since ScanState need to be copied some times, it's better to use a point of 
RuntimeFilterState in the struct and allocate memory dynamically.


---

[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1358#discussion_r184842816
  
--- Diff: contrib/vexecutor/nodeVMotion.c ---
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "nodeVMotion.h"
+#include "tuplebatch.h"
+#include "execVQual.h"
+/*=
+ * FUNCTIONS PROTOTYPES
+ */
+static TupleTableSlot *execVMotionSender(MotionState * node);
+static TupleTableSlot *execVMotionUnsortedReceiver(MotionState * node);
+
+TupleTableSlot *ExecVMotion(MotionState * node);
+static void doSendTupleBatch(Motion * motion, MotionState * node, 
TupleTableSlot *outerTupleSlot);
+static SendReturnCode
+SendTupleBatch(MotionLayerState *mlStates, ChunkTransportState 
*transportStates,
+   int16 motNodeID, TupleBatch tuplebatch, int16 targetRoute);
+/*=
+ */
+
+TupleTableSlot *
+ExecVMotionVirtualLayer(MotionState *node)
+{
+if(node->mstype == MOTIONSTATE_SEND)
+return ExecVMotion(node);
+else if(node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot* slot = node->ps.ps_ResultTupleSlot;
+while(1)
+{
+bool succ = VirtualNodeProc(slot);
+if(!succ)
+{
+slot = ExecVMotion(node);
+if(TupIsNull(slot))
+break;
+else
+continue;
+}
+
+break;
+}
+return slot;
+}
+}
+
+/* 
+ * ExecVMotion
+ * 
+ */
+TupleTableSlot *
+ExecVMotion(MotionState * node)
+{
+Motion*motion = (Motion *) node->ps.plan;
+
+/*
+ * at the top here we basically decide: -- SENDER vs. RECEIVER and --
+ * SORTED vs. UNSORTED
+ */
+if (node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot *tuple;
+
+if (node->ps.state->active_recv_id >= 0)
+{
+if (node->ps.state->active_recv_id != motion->motionID)
+{
+elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from 
%d to %d",
+ node->ps.state->active_recv_id, motion->motionID);
+node->ps.state->active_recv_id = motion->motionID;
+}
+} else
+node->ps.state->active_recv_id = motion->motionID;
+
+/* Running in diagnostic mode ? */
+if (Gp_interconnect_type == INTERCONNECT_TYPE_NIL)
+{
+node->ps.state->active_recv_id = -1;
+return NULL;
+}
+
+Assert(!motion->sendSorted);
+
+tuple = execVMotionUnsortedReceiver(node);
+
+if (tuple == NULL)
+node->ps.state->active_recv_id = -1;
+else
+{
+Gpmon_M_Incr(GpmonPktFromMotionState(node), 
GPMON_QEXEC_M_ROWSIN);
+Gpmon_M_Incr_Rows_Out(GpmonPktFromMotionState(node));
+setMotionStatsForGpmon(node);
+}
+#ifdef MEASURE_MOTION_TIME
+gettimeofday(, NULL);
+
+   node->motionTime.tv_sec += stopTime.tv_sec - startTime.tv_sec;
+   node->motionTime.tv_usec += stopTime.tv_usec - 
startTime.tv_usec;
+
+   while (node->motionTime.tv_usec < 0)
+   {
+   node->motionTime.tv_usec += 100;
+   node->motionTime.tv_sec--;
+   }
+
+ 

[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1358#discussion_r184842754
  
--- Diff: contrib/vexecutor/nodeVMotion.c ---
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "nodeVMotion.h"
+#include "tuplebatch.h"
+#include "execVQual.h"
+/*=
+ * FUNCTIONS PROTOTYPES
+ */
+static TupleTableSlot *execVMotionSender(MotionState * node);
+static TupleTableSlot *execVMotionUnsortedReceiver(MotionState * node);
+
+TupleTableSlot *ExecVMotion(MotionState * node);
+static void doSendTupleBatch(Motion * motion, MotionState * node, 
TupleTableSlot *outerTupleSlot);
+static SendReturnCode
+SendTupleBatch(MotionLayerState *mlStates, ChunkTransportState 
*transportStates,
+   int16 motNodeID, TupleBatch tuplebatch, int16 targetRoute);
+/*=
+ */
+
+TupleTableSlot *
+ExecVMotionVirtualLayer(MotionState *node)
+{
+if(node->mstype == MOTIONSTATE_SEND)
+return ExecVMotion(node);
+else if(node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot* slot = node->ps.ps_ResultTupleSlot;
+while(1)
+{
+bool succ = VirtualNodeProc(slot);
+if(!succ)
+{
+slot = ExecVMotion(node);
+if(TupIsNull(slot))
+break;
+else
+continue;
+}
+
+break;
+}
+return slot;
+}
+}
+
+/* 
+ * ExecVMotion
+ * 
+ */
+TupleTableSlot *
+ExecVMotion(MotionState * node)
+{
+Motion*motion = (Motion *) node->ps.plan;
+
+/*
+ * at the top here we basically decide: -- SENDER vs. RECEIVER and --
+ * SORTED vs. UNSORTED
+ */
+if (node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot *tuple;
+
+if (node->ps.state->active_recv_id >= 0)
+{
+if (node->ps.state->active_recv_id != motion->motionID)
+{
+elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from 
%d to %d",
+ node->ps.state->active_recv_id, motion->motionID);
+node->ps.state->active_recv_id = motion->motionID;
+}
+} else
+node->ps.state->active_recv_id = motion->motionID;
+
+/* Running in diagnostic mode ? */
+if (Gp_interconnect_type == INTERCONNECT_TYPE_NIL)
+{
+node->ps.state->active_recv_id = -1;
+return NULL;
+}
+
+Assert(!motion->sendSorted);
+
+tuple = execVMotionUnsortedReceiver(node);
+
+if (tuple == NULL)
+node->ps.state->active_recv_id = -1;
+else
+{
+Gpmon_M_Incr(GpmonPktFromMotionState(node), 
GPMON_QEXEC_M_ROWSIN);
--- End diff --

Is gpmon info different between tuple and tuplebatch in motion operation ?


---

[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1358#discussion_r184842905
  
--- Diff: contrib/vexecutor/tuplebatch.c ---
@@ -150,19 +150,26 @@ tbSerialization(TupleBatch tb )
 return ret;
 }
 
-TupleBatch tbDeserialization(unsigned char *buffer)
+void tbDeserialization(unsigned char *buffer,TupleBatch* pTB )
--- End diff --

Is tuple batch deserialization always successful ? If not, please use bool 
for return value type.


---

[GitHub] incubator-hawq pull request #1356: HAWQ-1611. refactor the vtype in order to...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1356#discussion_r184327414
  
--- Diff: contrib/vexecutor/tuplebatch.c ---
@@ -93,91 +85,119 @@ tbSerializationSize(TupleBatch tb)
 //get skip tag size
 len += sizeof( bool ) * tb->nrows;
 
+int vtypeSz = VTYPESIZE(tb->nrows);
 //get all un-null columns data size
 for(int i = 0;i < tb->ncols; i++ )
 {
 if(tb->datagroup[i])
 {
 len += sizeof(int);
-len += vtypeSize(tb->datagroup[i]);
+len += vtypeSz;
 }
 }
 return len;
 }
 
-unsigned char *
+MemTuple
 tbSerialization(TupleBatch tb )
 {
+MemTuple ret;
 size_t len = 0;
 size_t tmplen = 0;
 //calculate total size for TupleBatch
 size_t size = tbSerializationSize(tb);
+size = (size + 0x8) & (~0x7);
 
-unsigned char *buffer = palloc(size);
+ret = palloc0(size);
+unsigned char *buffer = ret->PRIVATE_mt_bits;
 
 //copy TupleBatch header
 memcpy(buffer,,sizeof(size_t));
+buffer += sizeof(size_t);
 
 tmplen = offsetof(TupleBatchData ,skip);
-memcpy(buffer+len,tb,tmplen);
-len += tmplen;
+memcpy(buffer,tb,tmplen);
+buffer +=tmplen;
 
 tmplen = sizeof(bool) * tb->nrows;
-memcpy(buffer+len,tb->skip,tmplen);
-len += tmplen;
+memcpy(buffer,tb->skip,tmplen);
+buffer += tmplen;
 
 
 for(int i = 0;i < tb->ncols; i++ )
 {
 if(tb->datagroup[i])
 {
-memcpy(buffer+len,,sizeof(int));
-len += sizeof(int);
+memcpy(buffer,,sizeof(int));
+buffer += sizeof(int);
+
+unsigned char* ptr = buffer;
+memcpy(ptr,tb->datagroup[i],offsetof(vtype,isnull));
+ptr+= offsetof(vtype,isnull);
+
+tmplen = VDATUMSZ(tb->nrows);
+memcpy(ptr,tb->datagroup[i]->values, tmplen);
+ptr += tmplen;
 
-tmplen = 
GetVFunc(tb->datagroup[i]->elemtype)->serialization(tb->datagroup[i],buffer + 
len);
-len += tmplen;
+tmplen = ISNULLSZ(tb->nrows);
+memcpy(ptr,tb->datagroup[i]->isnull,tmplen);
+buffer += VTYPESIZE(tb->nrows);
 }
 }
 
-return buffer;
+memtuple_set_size(ret,NULL,size);
+return ret;
 }
 
 TupleBatch tbDeserialization(unsigned char *buffer)
 {
 size_t buflen;
-memcpy(,buffer,sizeof(size_t));
+size_t len = 0;
+size_t tmplen = 0;
+tmplen = sizeof(size_t);
+memcpy(,buffer,tmplen);
+len += tmplen;
 
 if(buflen < sizeof(TupleBatchData))
 return NULL;
 
-size_t len = 0;
-size_t tmplen = 0;
 TupleBatch tb = palloc0(sizeof(TupleBatchData));
 
 //deserial tb main data
 tmplen = offsetof(TupleBatchData,skip);
-memcpy(tb,buffer+len,tmplen);
+memcpy(tb,buffer + len,tmplen);
 len += tmplen;
 
 //deserial member value -- skip
 if(tb->nrows != 0)
 {
-tb->skip = palloc(sizeof(bool) * tb->nrows);
+tmplen = sizeof(bool) * tb->nrows;
+tb->skip = palloc(tmplen);
 memcpy(tb->skip,buffer+len,tmplen);
 len += tmplen;
 }
 
 //deserial member value -- datagroup
 if(tb->ncols != 0)
 {
-tb->datagroup = palloc0(sizeof(vheader*) * tb->ncols);
 int colid;
-while (len < buflen)
+tmplen = sizeof(vtype*) * tb->ncols;
+tb->datagroup = palloc0(tmplen);
+while (((len + 0x8) & (~0x7)) < buflen)
--- End diff --

What's the meaning of these magic number ? It's better to have a comment or 
use a literal const.


---

[GitHub] incubator-hawq pull request #1354: HAWQ-1606. Implement Deciding to Create B...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1354#discussion_r181689211
  
--- Diff: src/backend/utils/hash/bloomfilter.c ---
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "utils/bloomfilter.h"
+#include "utils/elog.h"
+#include "utils/palloc.h"
+#include "lib/stringinfo.h"
+#include 
+#include 
+
+uint32_t HASH_SEEDS[8] __attribute__((aligned(32))) = { 0x14EBCDFFU,
+0x2A1C1A99U, 0x85CB78FBU, 0x6E8F82DDU, 0xF8464DFFU, 0x1028FEADU,
+0x74F04A4DU, 0x1832DB75U };
+
+static uint32_t getBucketIdx(uint32_t hash, uint32_t mask)
+{
+/* use Knuth's multiplicative hash */
+return ((uint64_t) (hash) * 2654435769ul >> 32) & mask;
+}
+
+/*
+ * Returns the smallest power of two that is bigger than v.
+ */
+int64_t UpperPowerTwo(int64_t v) {
+--v;
+v |= v >> 1;
+v |= v >> 2;
+v |= v >> 4;
+v |= v >> 8;
+v |= v >> 16;
+v |= v >> 32;
+++v;
+return v;
+}
+
+/*
+ * Initialize a Bloom filter structure with the memory size of Bloom 
filter.
+ */
+BloomFilter InitBloomFilter(int memory_size)
+{
+BloomFilter bf;
+uint32_t nBuckets = max(1, 
memory_size/(sizeof(BucketWord)*NUM_BUCKET_WORDS));
+size_t size = nBuckets*NUM_BUCKET_WORDS*sizeof(BucketWord);
+bf = palloc0(offsetof(BloomFilterData, data)  + size);
+bf->nInserted = bf->nTested = bf->nMatched = 0;
+bf->nBuckets = nBuckets;
+bf->data_mask = bf->nBuckets - 1;
+bf->data_size = size;
+bf->isCreated = true;
+elog(LOG, "Create a Bloom filter with number of buckets:%d, size:%d",
--- End diff --

Since the query plan could have multiple hash join nodes, this log will be 
printed more than once in one query.  Suggest setting log level lower to LOG or 
use a GUC to switch it. The same with DestroyBloomFilter.


---

[GitHub] incubator-hawq pull request #1354: HAWQ-1606. Implement Deciding to Create B...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1354#discussion_r181689893
  
--- Diff: src/backend/utils/hash/bloomfilter.c ---
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "utils/bloomfilter.h"
+#include "utils/elog.h"
+#include "utils/palloc.h"
+#include "lib/stringinfo.h"
+#include 
+#include 
+
+uint32_t HASH_SEEDS[8] __attribute__((aligned(32))) = { 0x14EBCDFFU,
+0x2A1C1A99U, 0x85CB78FBU, 0x6E8F82DDU, 0xF8464DFFU, 0x1028FEADU,
+0x74F04A4DU, 0x1832DB75U };
+
+static uint32_t getBucketIdx(uint32_t hash, uint32_t mask)
+{
+/* use Knuth's multiplicative hash */
+return ((uint64_t) (hash) * 2654435769ul >> 32) & mask;
+}
+
+/*
+ * Returns the smallest power of two that is bigger than v.
+ */
+int64_t UpperPowerTwo(int64_t v) {
+--v;
+v |= v >> 1;
+v |= v >> 2;
+v |= v >> 4;
+v |= v >> 8;
+v |= v >> 16;
+v |= v >> 32;
+++v;
+return v;
+}
+
+/*
+ * Initialize a Bloom filter structure with the memory size of Bloom 
filter.
+ */
+BloomFilter InitBloomFilter(int memory_size)
+{
+BloomFilter bf;
+uint32_t nBuckets = max(1, 
memory_size/(sizeof(BucketWord)*NUM_BUCKET_WORDS));
+size_t size = nBuckets*NUM_BUCKET_WORDS*sizeof(BucketWord);
+bf = palloc0(offsetof(BloomFilterData, data)  + size);
+bf->nInserted = bf->nTested = bf->nMatched = 0;
+bf->nBuckets = nBuckets;
+bf->data_mask = bf->nBuckets - 1;
+bf->data_size = size;
+bf->isCreated = true;
+elog(LOG, "Create a Bloom filter with number of buckets:%d, size:%d",
+  bf->nBuckets, size);
+return bf;
+}
+
+/*
+ * Insert a value into Bloom filter.
+ */
+void InsertBloomFilter(BloomFilter bf, uint32_t value)
+{
+uint32_t bucket_idx = getBucketIdx(value, bf->data_mask);
+uint32_t new_bucket[8];
+for (int i = 0; i < NUM_BUCKET_WORDS; ++i) {
+/*
+ * Multiply-shift hashing proposed by Dietzfelbinger et al.
+ * hash a value universally into 5 bits using the random odd seed.
+ */
+new_bucket[i] = (HASH_SEEDS[i] * value) >> (32 - 
LOG_BUCKET_WORD_BITS);
+new_bucket[i] = 1U << new_bucket[i];
+bf->data[bucket_idx][i] |= new_bucket[i];
+}
+}
+
+/*
+ * Check whether a value is in this Bloom filter or not.
+ */
+bool FindBloomFilter(BloomFilter bf, uint32_t value)
+{
+uint32_t bucket_idx = getBucketIdx(value, bf->data_mask);
+for (int i = 0; i < NUM_BUCKET_WORDS; ++i)
+{
+BucketWord hval = (HASH_SEEDS[i] * value) >> (32 - 
LOG_BUCKET_WORD_BITS);
+hval = 1U << hval;
+if (!(bf->data[bucket_idx][i] & hval))
+{
+return false;
+}
+}
+return true;
+}
+
+void PrintBloomFilter(BloomFilter bf)
+{
+StringInfo bfinfo = makeStringInfo();
+appendStringInfo(bfinfo, "# Print Bloom filter #\n");
+appendStringInfo(bfinfo, "data_mask: %x, ", bf->data_mask);
+appendStringInfo(bfinfo, "number of buckets: %d\n s", bf->nBuckets);
+appendStringInfo(bfinfo, "number of inserted: %d\n s", bf->nInserted);
+appendStringInfo(bfinfo, "number of tested: %d\n s", bf->nTested);
+appendStringInfo(bfinfo, "number of matched: %d\n s", bf->nMatched);
+appendStringInfo(bfinfo, "size: %d\n s", bf->data_size);
+appendStringInfo(bfinfo, "# END Print Bloom filter #\n");
+if(bfinfo->data != NULL)
+{
+pfree(bfinfo->data);
--- End diff --

Print the Bloom filter info before pfree ?


---

[GitHub] incubator-hawq pull request #1354: HAWQ-1606. Implement Deciding to Create B...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1354#discussion_r181687215
  
--- Diff: src/backend/resourcemanager/utils/simplestring.c ---
@@ -200,6 +200,45 @@ int  SimpleStringToStorageSizeMB(SimpStringPtr str, 
uint32_t *value)
return FUNC_RETURN_OK;
 }
 
+int  SimpleStringToBytes(SimpStringPtr str, uint32_t *value)
+{
+   int tail= strlen(str->Str) - 1;
+   int scanres = -1;
+   int32_t val;
+   charbuff[256];
+
+   if ( tail < 2 || tail > sizeof(buff)-1 )
+   return UTIL_SIMPSTRING_WRONG_FORMAT;
+
+   strncpy(buff, str->Str, tail-1);
+   buff[tail-1] = '\0';
+
+   scanres = sscanf(buff, "%d", );
+   if ( scanres != 1 )
+   return UTIL_SIMPSTRING_WRONG_FORMAT;
+
+   if ( (str->Str[tail]   == 'b' || str->Str[tail] == 'B' ) &&
+(str->Str[tail-1] == 'k' || str->Str[tail-1] == 'K') ) {
+   *value = val * 1024;
+   }
+   else if ( (str->Str[tail]   == 'b' || str->Str[tail] == 'B' ) &&
+(str->Str[tail-1] == 'm' || str->Str[tail-1] == 'M') ) {
+   *value = val * 1024 *1024;
+   }
+   else if ( (str->Str[tail]   == 'b' || str->Str[tail] == 'B' ) &&
+ (str->Str[tail-1] == 'g' || str->Str[tail-1] == 'G') 
) {
+   *value = val * 1024 * 1024 *1024;
+   }
+   else if ( (str->Str[tail]   == 'b' || str->Str[tail] == 'B' ) &&
+ (str->Str[tail-1] == 't' || str->Str[tail-1] == 'T') 
) {
+   *value = val * 1024 * 1024 *1024 * 1024;
--- End diff --

value is int32, it will overflow here.


---

[GitHub] incubator-hawq pull request #1351: HAWQ-1603.add new hook API

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1351#discussion_r180627154
  
--- Diff: contrib/vexecutor/execVQual.c ---
@@ -123,3 +123,431 @@ VirtualNodeProc(ScanState* state,TupleTableSlot 
*slot){
 ExecStoreVirtualTuple(slot);
 return true;
 }
+
+/*
+ * get values from vectorized tuple slot
+ * copy from src/backend/executor/execQual.c
+ */
+static Datum
+VExecEvalScalarVar(ExprState *exprstate, ExprContext *econtext,
+   bool *isNull, ExprDoneCond *isDone)
+{
+   Var*variable = (Var *) exprstate->expr;
+   TupleTableSlot *slot;
+   AttrNumber  attnum;
+   TupleBatch tb;
+
+   if (isDone)
+   *isDone = ExprSingleResult;
+
+   Assert(econtext->ecxt_scantuple != NULL || econtext->ecxt_innertuple != 
NULL || econtext->ecxt_outertuple != NULL);
+   /*
+* Get the input slot and attribute number we want
+*
+* The asserts check that references to system attributes only appear at
+* the level of a relation scan; at higher levels, system attributes 
must
+* be treated as ordinary variables (since we no longer have access to 
the
+* original tuple).
+*/
+   attnum = variable->varattno;
+
+   switch (variable->varno)
+   {
+   case INNER: /* get the tuple from 
the inner node */
+   slot = econtext->ecxt_innertuple;
+   Assert(attnum > 0);
+   break;
+
+   case OUTER: /* get the tuple from 
the outer node */
+   slot = econtext->ecxt_outertuple;
+   Assert(attnum > 0);
+   break;
+
+   default:/* get the tuple from 
the relation being
+* scanned */
+   slot = econtext->ecxt_scantuple;
+   break;
+   }
+
+   /* isNull is a single value, it can not be used when data is vectorized 
*/
+   *isNull = false;
+
--- End diff --

It's better to check or assert to make sure slot is not null here.


---

[GitHub] incubator-hawq pull request #1351: HAWQ-1603.add new hook API

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1351#discussion_r180626694
  
--- Diff: contrib/vexecutor/execVQual.c ---
@@ -123,3 +123,431 @@ VirtualNodeProc(ScanState* state,TupleTableSlot 
*slot){
 ExecStoreVirtualTuple(slot);
 return true;
 }
+
+/*
+ * get values from vectorized tuple slot
+ * copy from src/backend/executor/execQual.c
+ */
+static Datum
+VExecEvalScalarVar(ExprState *exprstate, ExprContext *econtext,
+   bool *isNull, ExprDoneCond *isDone)
+{
+   Var*variable = (Var *) exprstate->expr;
+   TupleTableSlot *slot;
+   AttrNumber  attnum;
+   TupleBatch tb;
+
+   if (isDone)
+   *isDone = ExprSingleResult;
+
+   Assert(econtext->ecxt_scantuple != NULL || econtext->ecxt_innertuple != 
NULL || econtext->ecxt_outertuple != NULL);
+   /*
+* Get the input slot and attribute number we want
+*
+* The asserts check that references to system attributes only appear at
+* the level of a relation scan; at higher levels, system attributes 
must
+* be treated as ordinary variables (since we no longer have access to 
the
+* original tuple).
+*/
+   attnum = variable->varattno;
+
+   switch (variable->varno)
+   {
+   case INNER: /* get the tuple from 
the inner node */
+   slot = econtext->ecxt_innertuple;
+   Assert(attnum > 0);
+   break;
+
+   case OUTER: /* get the tuple from 
the outer node */
+   slot = econtext->ecxt_outertuple;
+   Assert(attnum > 0);
+   break;
+
+   default:/* get the tuple from 
the relation being
+* scanned */
+   slot = econtext->ecxt_scantuple;
+   break;
+   }
+
+   /* isNull is a single value, it can not be used when data is vectorized 
*/
+   *isNull = false;
+
+   /* Fetch the value from the slot */
+   tb = (TupleBatch )slot->PRIVATE_tb;
+
+   Assert(NULL != tb);
+
+   return PointerGetDatum(tb->datagroup[attnum]);
+}
+
+
+/*
+ * get values from vectorized tuple slot
+ * copy from src/backend/executor/execQual.c
+ */
+static Datum
+VExecEvalVar(ExprState *exprstate, ExprContext *econtext,
+   bool *isNull, ExprDoneCond *isDone)
+{
+   Var*variable = (Var *) exprstate->expr;
+   TupleTableSlot *slot;
+   AttrNumber  attnum;
+
+   if (isDone)
+   *isDone = ExprSingleResult;
+
+   Assert(econtext->ecxt_scantuple != NULL || econtext->ecxt_innertuple != 
NULL || econtext->ecxt_outertuple != NULL);
+   /*
+* Get the input slot and attribute number we want
+*
+* The asserts check that references to system attributes only appear at
+* the level of a relation scan; at higher levels, system attributes 
must
+* be treated as ordinary variables (since we no longer have access to 
the
+* original tuple).
+*/
+   attnum = variable->varattno;
+
+   switch (variable->varno)
+   {
+   case INNER: /* get the tuple from 
the inner node */
+   slot = econtext->ecxt_innertuple;
+   Assert(attnum > 0);
+   break;
+
+   case OUTER: /* get the tuple from 
the outer node */
+   slot = econtext->ecxt_outertuple;
+   Assert(attnum > 0);
+   break;
+
+   default:/* get the tuple from 
the relation being
+* scanned */
+   slot = econtext->ecxt_scantuple;
+   break;
+   }
+
+   if (attnum != InvalidAttrNumber)
+   {
+   TupleBatch tb;
+   /*
+* Scalar variable case.
+*
+* If it's a user attribute, check validity (bogus system 
attnums will
+* be caught inside slot_getattr).  What we have to check for 
here
+* is the possibility of an attribute having been changed in 
type
+* since the plan tree was created.  Ideally the plan would get
+* invalidated and not re-used, but until that day arrives, we 
need
+* defenses.  Fortunately it'

[GitHub] incubator-hawq pull request #1351: HAWQ-1603.add new hook API

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1351#discussion_r180626789
  
--- Diff: contrib/vexecutor/vexecutor.c ---
@@ -163,24 +165,35 @@ static PlanState* VExecInitNode(PlanState 
*node,EState *eState,int eflags,Memory
case T_AppendOnlyScan:
case T_ParquetScan:
case T_TableScanState:
-   START_MEMORY_ACCOUNT(curMemoryAccount);
+   //START_MEMORY_ACCOUNT(curMemoryAccount);
--- End diff --

Any reason that comment this code line ?


---

[GitHub] incubator-hawq pull request #1351: HAWQ-1603.add new hook API

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1351#discussion_r180625706
  
--- Diff: contrib/vexecutor/execVQual.c ---
@@ -123,3 +123,431 @@ VirtualNodeProc(ScanState* state,TupleTableSlot 
*slot){
 ExecStoreVirtualTuple(slot);
 return true;
 }
+
+/*
+ * get values from vectorized tuple slot
+ * copy from src/backend/executor/execQual.c
+ */
+static Datum
+VExecEvalScalarVar(ExprState *exprstate, ExprContext *econtext,
+   bool *isNull, ExprDoneCond *isDone)
+{
+   Var*variable = (Var *) exprstate->expr;
+   TupleTableSlot *slot;
+   AttrNumber  attnum;
+   TupleBatch tb;
+
+   if (isDone)
+   *isDone = ExprSingleResult;
+
+   Assert(econtext->ecxt_scantuple != NULL || econtext->ecxt_innertuple != 
NULL || econtext->ecxt_outertuple != NULL);
+   /*
+* Get the input slot and attribute number we want
+*
+* The asserts check that references to system attributes only appear at
+* the level of a relation scan; at higher levels, system attributes 
must
+* be treated as ordinary variables (since we no longer have access to 
the
+* original tuple).
+*/
+   attnum = variable->varattno;
+
+   switch (variable->varno)
+   {
+   case INNER: /* get the tuple from 
the inner node */
+   slot = econtext->ecxt_innertuple;
+   Assert(attnum > 0);
+   break;
+
+   case OUTER: /* get the tuple from 
the outer node */
+   slot = econtext->ecxt_outertuple;
+   Assert(attnum > 0);
+   break;
+
+   default:/* get the tuple from 
the relation being
+* scanned */
+   slot = econtext->ecxt_scantuple;
+   break;
+   }
+
+   /* isNull is a single value, it can not be used when data is vectorized 
*/
+   *isNull = false;
+
+   /* Fetch the value from the slot */
+   tb = (TupleBatch )slot->PRIVATE_tb;
+
+   Assert(NULL != tb);
+
+   return PointerGetDatum(tb->datagroup[attnum]);
--- End diff --

attnum's value starts with 1, it should be attnum -1 as the index of 
tb->datagroup ?


---

[GitHub] incubator-hawq issue #1352: HAWQ-1604. Add A New GUC hawq_hashjoin_bloomfilt...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1352
  
LGTM.


---

[GitHub] incubator-hawq pull request #1350: HAWQ-1600. Parquet table data vectorized ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1350#discussion_r178502483
  
--- Diff: contrib/vexecutor/vcheck.h ---
@@ -39,6 +39,7 @@ typedef struct VectorizedState
 {
bool vectorized;
PlanState *parent;
+bool* proj;
--- End diff --

Because this variable is only related to ao table, or has different content 
based on different table format. It's better to rename it to a more readable 
name, aoprojs or just use (void * opaque).


---

[GitHub] incubator-hawq pull request #1350: HAWQ-1600. Parquet table data vectorized ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1350#discussion_r178484642
  
--- Diff: contrib/vexecutor/execVScan.c ---
@@ -38,8 +40,8 @@ getVScanMethod(int tableType)
 },
 //PARQUETSCAN
 {
-, , 
,
-, , 

+, , 
,
+NULL,NULL,NULL
--- End diff --

So we need to understand what's the different between MarkRestrNotAllowed 
and NULL, is it possible to cause crash if setting NULL ?


---

[GitHub] incubator-hawq pull request #1350: HAWQ-1600. Parquet table data vectorized ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1350#discussion_r178459027
  
--- Diff: contrib/vexecutor/execVScan.c ---
@@ -148,7 +181,10 @@ ExecVScan(ScanState *node, ExecScanAccessMtd accessMtd)
  * Form a projection tuple, store it in the result tuple 
slot
  * and return it.
  */
-return ExecProject(projInfo, NULL);
+((TupleBatch)projInfo->pi_slot->PRIVATE_tb)->nrows = 
((TupleBatch)slot->PRIVATE_tb)->nrows;
+memcpy(((TupleBatch)projInfo->pi_slot->PRIVATE_tb)->skip,
--- End diff --

If possible, avoid memory copy during tuple batch processing.


---

[GitHub] incubator-hawq pull request #1350: HAWQ-1600. Parquet table data vectorized ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1350#discussion_r178459158
  
--- Diff: contrib/vexecutor/execVQual.c ---
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "execVQual.h"
+
+/*
+ * ExecVariableList
+ * Evaluates a simple-Variable-list projection.
+ *
+ * Results are stored into the passed values and isnull arrays.
+ */
+static void
+ExecVecVariableList(ProjectionInfo *projInfo,
+ Datum *values)
+{
+ExprContext *econtext = projInfo->pi_exprContext;
+int   *varSlotOffsets = projInfo->pi_varSlotOffsets;
+int   *varNumbers = projInfo->pi_varNumbers;
+TupleBatch  tb = (TupleBatch) values;
+inti;
+tb->ncols = list_length(projInfo->pi_targetlist);
+
+/*
+ * Assign to result by direct extraction of fields from source slots 
... a
+ * mite ugly, but fast ...
+ */
+for (i = list_length(projInfo->pi_targetlist) - 1; i >= 0; i--)
+{
+char  *slotptr = ((char *) econtext) + varSlotOffsets[i];
+TupleTableSlot *varSlot = *((TupleTableSlot **) slotptr);
+intvarNumber = varNumbers[i] - 1;
+tb->datagroup[i] = 
((TupleBatch)varSlot->PRIVATE_tb)->datagroup[varNumber];
+}
+}
+
+TupleTableSlot *
+ExecVProject(ProjectionInfo *projInfo, ExprDoneCond *isDone)
+{
+TupleTableSlot *slot;
+Assert(projInfo != NULL);
+
+/*
+ * get the projection info we want
+ */
+slot = projInfo->pi_slot;
+
+/*
+ * Clear any former contents of the result slot.  This makes it safe 
for
+ * us to use the slot's Datum/isnull arrays as workspace. (Also, we can
+ * return the slot as-is if we decide no rows can be projected.)
+ */
+ExecClearTuple(slot);
+
+/*
+ * form a new result tuple (if possible); if successful, mark the 
result
+ * slot as containing a valid virtual tuple
+ */
+if (projInfo->pi_isVarList)
+{
+/* simple Var list: this always succeeds with one result row */
+if (isDone)
+*isDone = ExprSingleResult;
+
+ExecVecVariableList(projInfo,slot->PRIVATE_tb);
+ExecStoreVirtualTuple(slot);
+}
+else
+{
+elog(FATAL,"does not support expression in projection stmt");
--- End diff --

Any unsupport operation in vectorized execution, it should be check before 
execution and fallback to original executor. Must make sure it could get right 
query results. 


---

[GitHub] incubator-hawq pull request #1350: HAWQ-1600. Parquet table data vectorized ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1350#discussion_r178458812
  
--- Diff: contrib/vexecutor/execVScan.c ---
@@ -38,8 +40,8 @@ getVScanMethod(int tableType)
 },
 //PARQUETSCAN
 {
-, , 
,
-, , 

+, , 
,
+NULL,NULL,NULL
--- End diff --

Why these function pointers are set NULLs ?


---

[GitHub] incubator-hawq pull request #1349: HAWQ-1598. Vectorized Scan Node Framework...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1349#discussion_r177941951
  
--- Diff: contrib/vexecutor/execVScan.c ---
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "execVScan.h"
+#include "miscadmin.h"
+
+static TupleTableSlot*
+ExecVScan(ScanState *node, ExecScanAccessMtd accessMtd);
+
+static const ScanMethod *
+getVScanMethod(int tableType)
+{
+static const ScanMethod scanMethods[] =
+{
+//HEAPSCAN
+{
+},
+//APPENDONLYSCAN
+{
+, 
, ,
--- End diff --

Are these scan functions placeholders?


---

[GitHub] incubator-hawq pull request #1349: HAWQ-1598. Vectorized Scan Node Framework...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1349#discussion_r177942413
  
--- Diff: contrib/vexecutor/vexecutor.c ---
@@ -66,12 +75,47 @@ _PG_fini(void)
vmthd.ExecEndNode_Hook = NULL;
 }
 
-static PlanState* VExecInitNode(PlanState *node,EState *eState,int eflags)
+static void backportTupleDescriptor(PlanState* ps,TupleDesc td)
--- End diff --

What's the purpose of this function ? Would you give the comment ?


---

[GitHub] incubator-hawq pull request #1349: HAWQ-1598. Vectorized Scan Node Framework...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1349#discussion_r177941676
  
--- Diff: contrib/vexecutor/vexecutor.c ---
@@ -96,20 +140,70 @@ static PlanState* VExecInitNode(PlanState *node,EState 
*eState,int eflags)
((VectorizedState*)(subState->vectorized))->parent = node;
}
 
-   //***TODO:process the vectorized execution operators here
+   if(Gp_role != GP_ROLE_DISPATCH)
+   {
+   switch (nodeTag(node) )
+   {
+   case T_AppendOnlyScan:
+   case T_ParquetScan:
+   case T_TableScanState:
+   START_MEMORY_ACCOUNT(curMemoryAccount);
+   {
+   TupleDesc td = ((TableScanState 
*)node)->ss.ss_ScanTupleSlot->tts_tupleDescriptor;
+   ((TableScanState 
*)node)->ss.ss_ScanTupleSlot->PRIVATE_tb = 
PointerGetDatum(tbGenerate(td->natts,BATCHSIZE));
+   
node->ps_ResultTupleSlot->PRIVATE_tb = 
PointerGetDatum(tbGenerate(td->natts,BATCHSIZE));
+   /* if V->N */
+   
backportTupleDescriptor(node,node->ps_ResultTupleSlot->tts_tupleDescriptor);
+   }
+   END_MEMORY_ACCOUNT();
+   break;
+   default:
+   ((VectorizedState 
*)node->vectorized)->vectorized = false;
+   break;
+   }
+   }
 
-   elog(DEBUG3, "PG VEXEC INIT NODE");
return node;
 }
 static TupleTableSlot* VExecProcNode(PlanState *node)
 {
-   return NULL;
+TupleTableSlot* result = NULL;
+switch(nodeTag(node))
+{
+case T_ParquetScanState:
+case T_AppendOnlyScanState:
+case T_TableScanState:
+result = ExecTableVScan((TableScanState*)node);
+break;
+default:
+break;
+}
+return result;
 }
 
 static bool VExecEndNode(PlanState *node)
 {
+if(Gp_role == GP_ROLE_DISPATCH)
+   return false;
+
elog(DEBUG3, "PG VEXEC END NODE");
-   return false;
+   bool ret = false;
+   switch (nodeTag(node))
+   {
+   case T_AppendOnlyScanState:
+   case T_ParquetScanState:
+tbDestroy((TupleBatch *) 
(>ps_ResultTupleSlot->PRIVATE_tb));
+tbDestroy((TupleBatch *) (&((TableScanState *) 
node)->ss.ss_ScanTupleSlot->PRIVATE_tb));
+   ret = true;
+   break;
+   case T_TableScanState:
--- End diff --

The logic of case T_TableScanState in VExecEndNode is not the same as 
VExecInitNode, what's the difference ?


---

[GitHub] incubator-hawq pull request #1349: HAWQ-1598. Vectorized Scan Node Framework...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1349#discussion_r177940619
  
--- Diff: contrib/vexecutor/vexecutor.c ---
@@ -21,15 +21,23 @@
 #include "vexecutor.h"
 #include "utils/guc.h"
 #include "vcheck.h"
+#include "miscadmin.h"
+#include "tuplebatch.h"
+#include "executor/nodeTableScan.h"
+#include "catalog/catquery.h"
+#include "cdb/cdbvars.h"
+#include "execVScan.h"
 
 PG_MODULE_MAGIC;
-
+#define BATCHSIZE 1024
--- End diff --

It's better to make BATCHSIZE as GUC so that we could tune this value 
easily.


---

[GitHub] incubator-hawq issue #1348: HAWQ-1593. Vectorized execution condition check ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1348
  
LGTM.


---

[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1348#discussion_r175950445
  
--- Diff: contrib/vexecutor/vcheck.c ---
@@ -54,7 +61,274 @@ typedef struct VecFuncHashEntry
 vFuncMap *vFunc;
 } VecFuncHashEntry;
 
+typedef struct VecTypeHashEntry
+{
+   Oid src;
+   Oid dest;
+}VecTypeHashEntry;
+
+/* Map between the vectorized types and non-vectorized types */
+static HTAB *hashMapN2V = NULL;
+
+/*
+ * We check the expressions tree recursively becuase the args can be a sub 
expression,
+ * we must check the return type of sub expression to fit the parent 
expressions.
+ * so the retType in Vectorized is a temporary values, after we check on 
expression,
+ * we set the retType of this expression, and transfer this value to his 
parent.
+ */
+typedef struct VectorizedContext
+{
+   plan_tree_base_prefix base; /* Required prefix for 
plan_tree_walker/mutator */
+   Oid retType;
+   bool replace;
+}VectorizedContext;
+
+/*
+ * Check all the expressions if they can be vectorized
+ * NOTE: if an expressions is vectorized, we return false...,because we 
should check
+ * all the expressions in the Plan node, if we return true, then the 
walker will be
+ * over...
+ */
+static bool
+CheckVectorizedExpression(Node *node, VectorizedContext *ctx)
+{
+if(NULL == node)
+return false;
+
+if(is_plan_node(node))
+return false;
+
+//check the type of Var if it can be vectorized
+if(IsA(node, Var))
+{
+Var *var = (Var*)node;
+Oid vtype = GetVtype(var->vartype);
+if(InvalidOid == vtype)
+   return true;
+ctx->retType = vtype;
+if(ctx->replace)
+   var->vartype = vtype;
+return false;
+}
+
+//Const treat as can be vectorzied, its return type is non-vectorized 
type
+//because we support the function like this: vtype op(vtype, const);
+if(IsA(node, Const))
+{
+   Const *c = (Const*)node;
+   ctx->retType = c->consttype;
+   return false;
+}
+
+//OpExpr:args, return types should can be vectorized,
+//and there must exists an vectorized function to implement the 
operator
+if(IsA(node, OpExpr))
+{
+OpExpr *op = (OpExpr*)node;
+Node *argnode = NULL;
+Oid ltype, rtype, rettype;
+Form_pg_operator voper;
+HeapTuple tuple;
+
+//OpExpr mostly have two args, check the first one
+argnode = linitial(op->args);
+if(CheckVectorizedExpression(argnode, ctx))
+   return true;
+
+ltype = ctx->retType;
+
+//check the second one
+argnode = lsecond(op->args);
+if(CheckVectorizedExpression(argnode, ctx))
+   return true;
+
+rtype = ctx->retType;
+
+//check the return type
+rettype = GetVtype(op->opresulttype);
+if(InvalidOid == rettype)
+   return true;
+
+
+//get the vectorized operator functions
+//NOTE:we have no ParseState now, Give the NULL value is OK but 
not good...
+tuple = oper(NULL, list_make1(makeString(get_opname(op->opno))),
+   ltype, rtype, false, -1);
+if(NULL == tuple)
+   return true;
+
+voper = (Form_pg_operator)GETSTRUCT(tuple);
+if(voper->oprresult != rettype)
+   return true;
+
+if(ctx->replace)
+{
+   op->opresulttype = rettype;
+   op->opfuncid = voper->oprcode;
+}
+
+ctx->retType = rettype;
+return false;
+}
--- End diff --

Maybe need to check more nodes, such as FuncExpr.


---

[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1348#discussion_r175950023
  
--- Diff: contrib/vexecutor/vcheck.c ---
@@ -54,7 +61,274 @@ typedef struct VecFuncHashEntry
 vFuncMap *vFunc;
 } VecFuncHashEntry;
 
+typedef struct VecTypeHashEntry
+{
+   Oid src;
+   Oid dest;
+}VecTypeHashEntry;
+
+/* Map between the vectorized types and non-vectorized types */
+static HTAB *hashMapN2V = NULL;
+
+/*
+ * We check the expressions tree recursively becuase the args can be a sub 
expression,
+ * we must check the return type of sub expression to fit the parent 
expressions.
+ * so the retType in Vectorized is a temporary values, after we check on 
expression,
+ * we set the retType of this expression, and transfer this value to his 
parent.
+ */
+typedef struct VectorizedContext
+{
+   plan_tree_base_prefix base; /* Required prefix for 
plan_tree_walker/mutator */
+   Oid retType;
+   bool replace;
+}VectorizedContext;
+
+/*
+ * Check all the expressions if they can be vectorized
+ * NOTE: if an expressions is vectorized, we return false...,because we 
should check
+ * all the expressions in the Plan node, if we return true, then the 
walker will be
+ * over...
+ */
+static bool
+CheckVectorizedExpression(Node *node, VectorizedContext *ctx)
+{
+if(NULL == node)
+return false;
+
+if(is_plan_node(node))
+return false;
+
+//check the type of Var if it can be vectorized
+if(IsA(node, Var))
+{
+Var *var = (Var*)node;
+Oid vtype = GetVtype(var->vartype);
+if(InvalidOid == vtype)
+   return true;
+ctx->retType = vtype;
+if(ctx->replace)
+   var->vartype = vtype;
+return false;
+}
+
+//Const treat as can be vectorzied, its return type is non-vectorized 
type
+//because we support the function like this: vtype op(vtype, const);
+if(IsA(node, Const))
+{
+   Const *c = (Const*)node;
+   ctx->retType = c->consttype;
+   return false;
+}
+
+//OpExpr:args, return types should can be vectorized,
+//and there must exists an vectorized function to implement the 
operator
+if(IsA(node, OpExpr))
+{
+OpExpr *op = (OpExpr*)node;
+Node *argnode = NULL;
+Oid ltype, rtype, rettype;
+Form_pg_operator voper;
+HeapTuple tuple;
+
+//OpExpr mostly have two args, check the first one
+argnode = linitial(op->args);
+if(CheckVectorizedExpression(argnode, ctx))
+   return true;
+
+ltype = ctx->retType;
+
+//check the second one
+argnode = lsecond(op->args);
+if(CheckVectorizedExpression(argnode, ctx))
+   return true;
+
+rtype = ctx->retType;
+
+//check the return type
+rettype = GetVtype(op->opresulttype);
+if(InvalidOid == rettype)
+   return true;
+
+
+//get the vectorized operator functions
+//NOTE:we have no ParseState now, Give the NULL value is OK but 
not good...
+tuple = oper(NULL, list_make1(makeString(get_opname(op->opno))),
+   ltype, rtype, false, -1);
+if(NULL == tuple)
+   return true;
+
+voper = (Form_pg_operator)GETSTRUCT(tuple);
+if(voper->oprresult != rettype)
+   return true;
+
+if(ctx->replace)
+{
+   op->opresulttype = rettype;
+   op->opfuncid = voper->oprcode;
+}
+
+ctx->retType = rettype;
+return false;
+}
+
+//now, other nodes treat as can not be vectorized
+return plan_tree_walker(node, CheckVectorizedExpression, ctx);;
+}
+
+/*
+ * check an plan node, all the expressions in it should be checked
+ * set the flag if an plan node can be vectorized
+ */
+static bool
+CheckPlanNodeWalker(PlannerInfo *root, Plan *plan)
+{
+   VectorizedContext ctx;
+
+if(plan->vectorized)
+   return true;
+
+ctx.replace =false;
+
+ctx.retType = InvalidOid;
+plan->vectorized = !plan_tree_walker((Node*)plan,
+
CheckVectorizedExpression,
+);
+
+
+return false;
+}
+
+/*
+ * check the p

[GitHub] incubator-hawq issue #1347: HAWQ-1591 Common tuple batch structure for VecEx...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1347
  
LGTM


---

[GitHub] incubator-hawq pull request #1347: HAWQ-1591 Common tuple batch structure fo...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1347#discussion_r175084597
  
--- Diff: contrib/vexecutor/tuplebatch.c ---
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "postgres.h"
+#include "tuplebatch.h"
+
+static size_t vtypeSize(vheader *vh);
+static size_t tbSerializationSize(TupleBatch tb);
+
+#define TBGETVALUEPTR(tb,type,colid,rowid) (&(((type 
*)tb->datagroup[colid])->data[rowid]))
+#define TBGETISNULLPTR(tb,colid,rowid) 
(&((tb->datagroup[colid])->isnull[rowid]))
+
+TupleBatch tbGenerate(int colnum,int batchsize)
+{
+Assert(colnum > 0 && batchsize > 0);
+TupleBatch tb = palloc0(sizeof(TupleBatchData));
+if(!tb)
+{
+elog(FATAL,"TupleBatch Allocation failed");
+return NULL;
+}
+
+tb->ncols = colnum;
+tb->batchsize = batchsize;
+
+tb->skip = palloc0(sizeof(bool) * tb->batchsize);
+tb->datagroup = palloc0(sizeof(struct vtypeheader*) * tb->ncols);
+
--- End diff --

It's better to init all the struct variable members (nrow, iter) in 
tbGenerate or call tbReset before return.


---

[GitHub] incubator-hawq pull request #1347: HAWQ-1591 Common tuple batch structure fo...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1347#discussion_r175018979
  
--- Diff: contrib/vexecutor/tuplebatch.c ---
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "postgres.h"
+#include "tuplebatch.h"
+
+static size_t vtypeSize(vheader *vh);
+static size_t tbSerializationSize(TupleBatch tb);
+
+#define TBGETVALUEPTR(tb,type,colid,rowid) (&(((type 
*)tb->datagroup[colid])->data[rowid]))
+#define TBGETISNULLPTR(tb,colid,rowid) 
(&((tb->datagroup[colid])->isnull[rowid]))
+
+TupleBatch tbGenerate(int colnum,int batchsize)
+{
+Assert(colnum > 0 && batchsize > 0);
+TupleBatch tb = palloc0(sizeof(TupleBatchData));
+if(!tb)
+{
+elog(FATAL,"TupleBatch Allocation failed");
+return NULL;
+}
+
+tb->ncols = colnum;
+tb->batchsize = batchsize;
+
+tb->skip = palloc0(sizeof(bool) * tb->batchsize);
+tb->datagroup = palloc0(sizeof(struct vtypeheader*) * tb->ncols);
+
+return tb;
+}
+
+void tbDestory(TupleBatch* tb){
+free((*tb)->skip);
+for(int i = 0 ;i < (*tb)->ncols; ++i)
+{
+if((*tb)->datagroup[i])
+tbfreeColumn((*tb)->datagroup,i);
+}
+
+free((*tb)->datagroup);
+
+free((*tb));
+*tb = NULL;
+}
+
+void tbReset(TupleBatch tb)
+{
+tb->iter = 0;
+tb->nrows = 0;
+memset(tb->skip,0, sizeof(bool) * tb->batchsize);
+}
+
+void tbCreateColumn(TupleBatch tb,int colid,Oid type)
+{
+if(tb->ncols <= colid)
+return;
+int bs = tb->batchsize;
+
+GetVFunc(type)->vtbuild((bs));
+}
+
+void tbfreeColumn(vheader** vh,int colid)
+{
+GetVFunc(vh[colid]->elemtype)->vtfree([colid]);
+}
+
+static size_t vtypeSize(vheader *vh)
+{
+return GetVFunc(vh->elemtype)->vtsize(vh);
+}
+
+static size_t
+tbSerializationSize(TupleBatch tb)
+{
+//buffer size stick in the head of the buffer
+size_t len = sizeof(size_t);
+
+//get TupleBatch structure size
+len += offsetof(TupleBatchData ,skip);
+
+//get skip tag size
+len += sizeof( bool ) * tb->nrows;
+
+//get all un-null columns data size
+for(int i = 0;i < tb->ncols; i++ )
+{
+if(tb->datagroup[i])
+{
+len += sizeof(int);
+len += vtypeSize(tb->datagroup[i]);
+}
+}
+return len;
+}
+
+unsigned char *
+tbSerialization(TupleBatch tb )
+{
+size_t len = 0;
+size_t tmplen = 0;
+//calculate total size for TupleBatch
+size_t size = tbSerializationSize(tb);
+
+unsigned char *buffer = palloc(size);
+
+//copy TupleBatch header
+memcpy(buffer,,sizeof(size_t));
+
+tmplen = offsetof(TupleBatchData ,skip);
+memcpy(buffer+len,tb,tmplen);
+len += tmplen;
+
+tmplen = sizeof(bool) * tb->nrows;
+memcpy(buffer+len,tb->skip,tmplen);
+len += tmplen;
+
+
+for(int i = 0;i < tb->ncols; i++ )
+{
+if(tb->datagroup[i])
+{
+memcpy(buffer+len,,sizeof(int));
+len += sizeof(int);
+
+tmplen = 
GetVFunc(tb->datagroup[i]->elemtype)->serialization(tb->datagroup[i],buffer + 
len);
+len += tmplen;
+}
+}
+
+return buffer;
+}
+
+TupleBatch tbDeserialization(unsigned char *buffer)
+{
+size_t buflen;
+memcpy(,buffer,sizeof(size_t));
+
+if(buflen < sizeof(TupleBatchData))
+return NULL;
 

[GitHub] incubator-hawq pull request #1347: HAWQ-1591 Common tuple batch structure fo...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1347#discussion_r175019315
  
--- Diff: contrib/vexecutor/tuplebatch.h ---
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef __TUPLEBATCH_H__
+#define __TUPLEBATCH_H__
+
+#include "vexecutor.h"
+#include "vcheck.h"
+
+
+typedef struct TupleBatchData
+{
+int _tb_len;
+int batchsize;  //indicate maximum number of batch
+int ncols;  //the number of target table column
+int nrows;  //the number of tuples scaned out
+int iter;   //used for data
+bool*   skip;   //used for qualification
+vheader** datagroup;
+}TupleBatchData,*TupleBatch;
+
+/* TupleBatch ctor */
--- End diff --

1. Would you give a user case of tuplebatch struct or the function call 
logic of its functions ?
2. How the tuple batch is integrated with current TupleTableSlot ?


---

[GitHub] incubator-hawq pull request #1347: HAWQ-1591 Common tuple batch structure fo...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1347#discussion_r175015247
  
--- Diff: contrib/vexecutor/tuplebatch.c ---
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "postgres.h"
+#include "tuplebatch.h"
+
+static size_t vtypeSize(vheader *vh);
+static size_t tbSerializationSize(TupleBatch tb);
+
+#define TBGETVALUEPTR(tb,type,colid,rowid) (&(((type 
*)tb->datagroup[colid])->data[rowid]))
+#define TBGETISNULLPTR(tb,colid,rowid) 
(&((tb->datagroup[colid])->isnull[rowid]))
+
+TupleBatch tbGenerate(int colnum,int batchsize)
+{
+Assert(colnum > 0 && batchsize > 0);
+TupleBatch tb = palloc0(sizeof(TupleBatchData));
+if(!tb)
+{
+elog(FATAL,"TupleBatch Allocation failed");
+return NULL;
+}
+
+tb->ncols = colnum;
+tb->batchsize = batchsize;
+
+tb->skip = palloc0(sizeof(bool) * tb->batchsize);
+tb->datagroup = palloc0(sizeof(struct vtypeheader*) * tb->ncols);
+
+return tb;
+}
+
+void tbDestory(TupleBatch* tb){
--- End diff --

tbDestory -> tbDestroy ?


---

[GitHub] incubator-hawq issue #1345: HAWQ-1592. vectorized data types initialization ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1345
  
LGTM. +1


---

[GitHub] incubator-hawq issue #1346: HAWQ-1594. Memory leak in standby master (gpsync...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1346
  
LGTM


---

[GitHub] incubator-hawq issue #1345: HAWQ-1592. vectorized data types initialization ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1345
  
Great to have this PR. Would you add some instructions about how to add a 
new vectorized type and related functions to this framework ?


---

[GitHub] incubator-hawq issue #1341: HAWQ-1583 Add vectorized executor extension and ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1341
  
All new files should add ASF license header.


---

[GitHub] incubator-hawq pull request #1341: HAWQ-1583 Add vectorized executor extensi...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1341#discussion_r167461629
  
--- Diff: src/backend/executor/execProcnode.c ---
@@ -797,7 +801,7 @@ ExecSliceDependencyNode(PlanState *node)
ExecSliceDependencyNode(outerPlanState(node));
ExecSliceDependencyNode(innerPlanState(node));
 }
-
+
--- End diff --

This diff should be removed.


---

[GitHub] incubator-hawq pull request #1341: HAWQ-1583 Add vectorized executor extensi...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1341#discussion_r167461583
  
--- Diff: src/include/executor/executor.h ---
@@ -41,6 +41,17 @@
 
 #include "cdb/cdbdef.h" /* CdbVisitOpt */
 
+#define ISVECTORIZED(note) vectorized_executor_enable
--- End diff --

note -> node ?


---

[GitHub] incubator-hawq issue #1341: HAWQ-1583 Add vectorized executor extension and ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1341
  
It's better to add a README for describing what this module is and how it 
works.


---

[GitHub] incubator-hawq issue #1336: HAWQ-1514. TDE feature makes libhdfs3 require op...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1336
  
LGTM


---

[GitHub] incubator-hawq issue #1288: HAWQ-1518. Add a UDF for showing whether the dat...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1288
  
LGTM


---

[GitHub] incubator-hawq issue #1286: HAWQ-1525. Segmentation fault occurs if reindex ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1286
  
LGTM


---

[GitHub] incubator-hawq issue #1280: HAWQ-1510. Add TDE-related functionality into ha...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1280
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1276: HAWQ-1511. Add TDE-related properties into hdfs-...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1276
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1274: HAWQ-1509. Support TDE read function.

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1274
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #1274: HAWQ-1509. Support TDE read function.

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1274#discussion_r131581898
  
--- Diff: depends/libhdfs3/src/client/CryptoCodec.cpp ---
@@ -119,33 +119,38 @@ namespace Hdfs {
return -1;
}
 
-   //calculate new IV when appending a existed file
+   // Calculate iv and counter in order to init cipher context 
with cipher method. Default value is 0.
+   resetStreamOffset(crypto_method, stream_offset);
+
+   LOG(DEBUG3, "CryptoCodec init success, length of the decrypted 
key is : %llu, crypto method is : %d", AlgorithmBlockSize, crypto_method);
+   is_init = true;
+   return 1;
+
+   }
+
+   int CryptoCodec::resetStreamOffset(CryptoMethod crypto_method, int64_t 
stream_offset) {
--- End diff --

The function prototype defines that it could return 1, 0, -1, but there is 
no value 0 return. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1269: HAWQ-1506. Support multi-append a file within en...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1269
  
LGTM. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #1270: HAWQ-1507. Fix unittest-check fail on SuS...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/1270

HAWQ-1507. Fix unittest-check fail on SuSE



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1507

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/1270.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1270


commit 3ee258ed17291cf2ce33694e954df956194528c1
Author: Ivan <wengyanq...@gmail.com>
Date:   2017-07-27T07:28:49Z

HAWQ-1507. Fix unittest-check fail on SuSE




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1265: HAWQ-1500. HAWQ-1501. HAWQ-1502. Support TDE wri...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1265
  
Great job. LGTM.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1227: HAWQ-1448. Fixed postmaster process hung at recv...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1227
  
Agree with Radar's comment. LGTM.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #1265: HAWQ-1500. HAWQ-1501. HAWQ-1502. Support ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1265#discussion_r126638288
  
--- Diff: depends/libhdfs3/src/client/HttpClient.cpp ---
@@ -0,0 +1,337 @@
+/
+ * 2014 -
+ * open source under Apache License Version 2.0
+ /
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "HttpClient.h"
+#include "Logger.h"
+
+using namespace Hdfs::Internal;
+
+namespace Hdfs {
+
+#define CURL_SETOPT(handle, option, optarg, fmt, ...) \
+res = curl_easy_setopt(handle, option, optarg); \
+if (res != CURLE_OK) { \
+THROW(HdfsIOException, fmt, ##__VA_ARGS__); \
+}
+
+#define CURL_SETOPT_ERROR1(handle, option, optarg, fmt) \
+CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res));
+
+#define CURL_SETOPT_ERROR2(handle, option, optarg, fmt) \
+CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res), \
+errorString().c_str())
+
+#define CURL_PERFORM(handle, fmt) \
+res = curl_easy_perform(handle); \
+if (res != CURLE_OK) { \
+THROW(HdfsIOException, fmt, curl_easy_strerror(res), 
errorString().c_str()); \
+}
+
+
+#define CURL_GETOPT_ERROR2(handle, option, optarg, fmt) \
+res = curl_easy_getinfo(handle, option, optarg); \
+if (res != CURLE_OK) { \
+THROW(HdfsIOException, fmt, curl_easy_strerror(res), 
errorString().c_str()); \
+}
+
+#define CURL_GET_RESPONSE(handle, code, fmt) \
+CURL_GETOPT_ERROR2(handle, CURLINFO_RESPONSE_CODE, code, fmt);
+
+HttpClient::HttpClient() : curl(NULL), list(NULL) {
+
+}
+
+/**
+ * Construct a HttpClient instance.
+ * @param url a url which is the address to send the request to the 
corresponding http server.
+ */
+HttpClient::HttpClient(const std::string ) {
+   curl = NULL;
+   list = NULL;
+   this->url = url;
+}
+
+/**
+ * Destroy a HttpClient instance.
+ */
+HttpClient::~HttpClient()
+{
+   destroy();
+}
+
+/**
+ * Receive error string from curl.
+ */
+std::string HttpClient::errorString() {
+   if (strlen(errbuf) == 0)
+   return "";
+   return errbuf;
+}
+
+/**
+ * Curl call back function to receive the reponse messages.
+ * @return return the size of reponse messages. 
+ */
+size_t HttpClient::CurlWriteMemoryCallback(void *contents, size_t size, 
size_t nmemb, void *userp)
+{
+  size_t realsize = size * nmemb;
+ if (userp == NULL || contents == NULL) {
+ return 0;
+ }
+  ((std::string *)userp)->append((const char *)contents, realsize);
+ LOG(DEBUG2, "HttpClient : Http response is : %s", ((std::string 
*)userp)->c_str());
+  return realsize;
+}
+
+/**
+ * Init curl handler and set curl options.
+ */
+void HttpClient::init() {
+   if (!initialized)
+   {
+   initialized = true;
+   if (curl_global_init(CURL_GLOBAL_ALL)) {
+   THROW(HdfsIOException, "Cannot initialize curl client 
for KMS");
+   }
+   }
+
+   curl = curl_easy_init();
+   if (!curl) {
+   THROW(HdfsIOException, "Cannot initialize curl handle for KMS");
+   }
+   
+CURL_SETOPT_ERROR1(curl, CURLOPT_ERRORBUFFER, errbuf,
+"Cannot initialize curl error buffer for KMS: %s");
+
+errbuf[0] = 0;
+
+CURL_SETOPT_ERROR2(curl, CURLOPT_NOPROGRESS, 1,
+"Cannot initialize no progress in HttpClient: %s: %s");
+
+CURL_SETOPT_ERROR2(curl, CURLOPT_VERBOSE, 0,
+"Cannot initialize no verbose in HttpClient: %s: %s&

[GitHub] incubator-hawq pull request #1265: HAWQ-1500. HAWQ-1501. HAWQ-1502. Support ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1265#discussion_r126637139
  
--- Diff: depends/libhdfs3/src/client/HttpClient.cpp ---
@@ -0,0 +1,337 @@
+/
+ * 2014 -
+ * open source under Apache License Version 2.0
+ /
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "HttpClient.h"
+#include "Logger.h"
+
+using namespace Hdfs::Internal;
+
+namespace Hdfs {
+
+#define CURL_SETOPT(handle, option, optarg, fmt, ...) \
+res = curl_easy_setopt(handle, option, optarg); \
+if (res != CURLE_OK) { \
+THROW(HdfsIOException, fmt, ##__VA_ARGS__); \
+}
+
+#define CURL_SETOPT_ERROR1(handle, option, optarg, fmt) \
+CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res));
+
+#define CURL_SETOPT_ERROR2(handle, option, optarg, fmt) \
+CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res), \
+errorString().c_str())
+
+#define CURL_PERFORM(handle, fmt) \
+res = curl_easy_perform(handle); \
+if (res != CURLE_OK) { \
+THROW(HdfsIOException, fmt, curl_easy_strerror(res), 
errorString().c_str()); \
+}
+
+
+#define CURL_GETOPT_ERROR2(handle, option, optarg, fmt) \
+res = curl_easy_getinfo(handle, option, optarg); \
+if (res != CURLE_OK) { \
+THROW(HdfsIOException, fmt, curl_easy_strerror(res), 
errorString().c_str()); \
+}
+
+#define CURL_GET_RESPONSE(handle, code, fmt) \
+CURL_GETOPT_ERROR2(handle, CURLINFO_RESPONSE_CODE, code, fmt);
+
+HttpClient::HttpClient() : curl(NULL), list(NULL) {
+
+}
+
+/**
+ * Construct a HttpClient instance.
+ * @param url a url which is the address to send the request to the 
corresponding http server.
+ */
+HttpClient::HttpClient(const std::string ) {
+   curl = NULL;
+   list = NULL;
+   this->url = url;
+}
+
+/**
+ * Destroy a HttpClient instance.
+ */
+HttpClient::~HttpClient()
+{
+   destroy();
+}
+
+/**
+ * Receive error string from curl.
+ */
+std::string HttpClient::errorString() {
+   if (strlen(errbuf) == 0)
+   return "";
+   return errbuf;
+}
+
+/**
+ * Curl call back function to receive the reponse messages.
+ * @return return the size of reponse messages. 
+ */
+size_t HttpClient::CurlWriteMemoryCallback(void *contents, size_t size, 
size_t nmemb, void *userp)
+{
+  size_t realsize = size * nmemb;
+ if (userp == NULL || contents == NULL) {
+ return 0;
+ }
+  ((std::string *)userp)->append((const char *)contents, realsize);
+ LOG(DEBUG2, "HttpClient : Http response is : %s", ((std::string 
*)userp)->c_str());
+  return realsize;
+}
+
+/**
+ * Init curl handler and set curl options.
+ */
+void HttpClient::init() {
+   if (!initialized)
+   {
+   initialized = true;
+   if (curl_global_init(CURL_GLOBAL_ALL)) {
+   THROW(HdfsIOException, "Cannot initialize curl client 
for KMS");
+   }
+   }
+
+   curl = curl_easy_init();
+   if (!curl) {
+   THROW(HdfsIOException, "Cannot initialize curl handle for KMS");
+   }
+   
+CURL_SETOPT_ERROR1(curl, CURLOPT_ERRORBUFFER, errbuf,
+"Cannot initialize curl error buffer for KMS: %s");
+
+errbuf[0] = 0;
+
+CURL_SETOPT_ERROR2(curl, CURLOPT_NOPROGRESS, 1,
+"Cannot initialize no progress in HttpClient: %s: %s");
+
+CURL_SETOPT_ERROR2(curl, CURLOPT_VERBOSE, 0,
+"Cannot initialize no verbose in HttpClient: %s: %s&

[GitHub] incubator-hawq issue #1236: HAWQ-1457. Shared memory for SegmentStatus and M...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1236
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1222: HAWQ-1438. Support resource owner beyond transac...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1222
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1207: HAWQ-326. Fix specified HAWQ_RELEASE_VERSION for...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1207
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1198: HAWQ-1417. Fixed crash when ANALYZE after COPY b...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1198
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1187: HAWQ-1408. Fixed crash when alloc not enough seg...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1187
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #1157: HAWQ-1371. Fix QE process hang in shared ...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/1157#discussion_r104548625
  
--- Diff: src/backend/executor/nodeShareInputScan.c ---
@@ -634,16 +634,16 @@ static int retry_read(int fd, char *buf, int rsize)
 
 read_retry:
sz = read(fd, buf, rsize);
-   if (sz > 0)
+   if (sz >= 0)  
return sz;
-   else if(sz == 0 || errno == EINTR)
+   else if(errno == EINTR)
--- End diff --

It needs to handle EAGAIN in nonblocking read.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #:

[wengyanqing]

Github user wengyanqing commented on the pull request:


https://github.com/apache/incubator-hawq/commit/b37f18768dceec5600f8a90f6f8b17045954629c#commitcomment-21009101
  
Hi @lisakowen ,

Would you modify the commit message following the format as 
HAWQ-(JIRA). message content ?Thanks a lot.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1135: HAWQ-1347. QD should check segment health only

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1135
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1134: HAWQ-1345. Fixed relfile path bug: catalog and h...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1134
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1128: HAWQ-1338. Fixed writer process doesn't exit nic...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1128
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1115: HAWQ-1324. Fixed crash at query cancel, signal h...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1115
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1019: HAWQ-870. Allocate target's tuple table slot in ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1019
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1040: HAWQ-1195. Fixed error "Two or more external tab...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1040
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1023: HAWQ-77. Fix source code comment for new ALTER/C...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1023
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #1037: HAWQ-1188. Fix guc type issue for Gp_inte...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/1037

HAWQ-1188. Fix guc type issue for Gp_interconnect_transmit_timeout



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1188

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/1037.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #1037


commit 7c05a23f0ef6ccb8d5d6d5f613683f8d18e29426
Author: ivan <iw...@pivotal.io>
Date:   2016-12-05T07:27:32Z

HAWQ-1188. Fix guc type issue for Gp_interconnect_transmit_timeout




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1031: HAWQ-1182. Add Macro for unused argument and var...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1031
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1015: HAWQ-1159. Skip namenode check while namenode no...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1015
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #1010: HAWQ-1153. Add global init file support for feat...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/1010
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #996: HAWQ-1117.RM crash when init db after configure w...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/996
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #995: HAWQ-1139. Fix TestCreateTableInherits bug...

[wengyanqing]

Github user wengyanqing closed the pull request at:

https://github.com/apache/incubator-hawq/pull/995


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #995: HAWQ-1139. Fix TestCreateTableInherits bug...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/995

HAWQ-1139. Fix TestCreateTableInherits bug in the feature test



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1139

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/995.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #995


commit 5ba9fa8d00a4cd13a3db127d55687ac53cb44cc9
Author: ivan <iw...@pivotal.io>
Date:   2016-11-02T07:15:09Z

HAWQ-1139. Fix TestCreateTableInherits bug in the feature test




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #989: HAWQ-1129. Install PLR into hawq home directory

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/989
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #979: HAWQ-1125. Running pl/python related feature_test...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/979
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #953: HAWQ-1092. lc_collate and lc_ctype do not work af...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/953
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #934: HAWQ-1061. Fix register with empty folder in usag...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/934
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #932: HAWQ-1024. Add rollback before all necessary exit...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/932
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #924: HAWQ-1035. Treat ddl of list and range partition ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/924
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #899: HAWQ-1017. Add feature test for goh_create...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/899

HAWQ-1017. Add feature test for goh_create_type_composite with new te…

…st framework

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1017

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/899.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #899


commit cda54558c3cc852ab2e0fc26b3a67c2cbc0173d6
Author: ivan <iw...@pivotal.io>
Date:   2016-09-13T01:15:36Z

HAWQ-1017. Add feature test for goh_create_type_composite with new test 
framework




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #897: HAWQ-1014. Add feature test for informatio...

[wengyanqing]

Github user wengyanqing closed the pull request at:

https://github.com/apache/incubator-hawq/pull/897


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #877: HAWQ-1016. Add feature test for rowtypes w...

[wengyanqing]

Github user wengyanqing closed the pull request at:

https://github.com/apache/incubator-hawq/pull/877


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #897: HAWQ-1014. Add feature test for informatio...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/897

HAWQ-1014.  Add feature test for information_schema with new test fra…

…mework

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1014

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/897.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #897


commit 291284362bfc4caec024b8aee1c9fef8dac4ce22
Author: ivan <iw...@pivotal.io>
Date:   2016-09-09T02:02:12Z

HAWQ-1014.  Add feature test for information_schema with new test framework




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #880: HAWQ-1037. Modify way to get HDFS port in TestHaw...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/880
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #871: HAWQ-909. Add feature test for goh_databas...

[wengyanqing]

Github user wengyanqing closed the pull request at:

https://github.com/apache/incubator-hawq/pull/871


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #823: HAWQ-905. add init file for temp table tes...

[wengyanqing]

Github user wengyanqing closed the pull request at:

https://github.com/apache/incubator-hawq/pull/823


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #822: HAWQ-905. Add feature test for temp table ...

[wengyanqing]

Github user wengyanqing closed the pull request at:

https://github.com/apache/incubator-hawq/pull/822


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #892: HAWQ-1007. Add the pgcrypto code into hawq

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/892
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #891: HAWQ-960. Remove the file BUILD_INSTRUCTIONS.md a...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/891
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #881: HAWQ-1032. Bucket number of new added partition i...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/881
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #877: HAWQ-1016. Add feature test for rowtypes w...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/877

HAWQ-1016. Add feature test for rowtypes with new test framework



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1016

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/877.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #877


commit 976dfe4c7f4b8936d1a00dbba71065b9e1fdad65
Author: ivan <iw...@pivotal.io>
Date:   2016-08-31T03:31:28Z

HAWQ-1016. Add feature test for rowtypes with new test framework




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #876: HAWQ-1018. Add feature test for goh_gp_dis...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/876

HAWQ-1018. Add feature test for goh_gp_dist_random with new test fram…

…ework

You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1018

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/876.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #876


commit 569f7545d289a396f9c002bb304b613a89c3cfc1
Author: ivan <iw...@pivotal.io>
Date:   2016-08-30T01:52:52Z

HAWQ-1018. Add feature test for goh_gp_dist_random with new test framework




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #875: HAWQ-1015. Add feature test for transactio...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/875

HAWQ-1015. Add feature test for transactions with new test framework



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-1015PR

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/875.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #875


commit cb80c333176e38a29eba618b70440b12f1b4ab33
Author: ivan <iw...@pivotal.io>
Date:   2016-08-29T03:01:31Z

HAWQ-1015. Add feature test for transactions with new test framework




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #871: HAWQ-909. Add feature test for goh_databas...

[wengyanqing]

GitHub user wengyanqing opened a pull request:

https://github.com/apache/incubator-hawq/pull/871

HAWQ-909. Add feature test for goh_database with new test framework



You can merge this pull request into a Git repository by running:

$ git pull https://github.com/wengyanqing/incubator-hawq HAWQ-909

Alternatively you can review and apply these changes as the patch at:

https://github.com/apache/incubator-hawq/pull/871.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

This closes #871


commit def7b3083784f17c02e6c2f6365f627204cdb4d0
Author: ivan <iw...@pivotal.io>
Date:   2016-08-29T02:59:12Z

HAWQ-909. Add feature test for goh_database with new test framework




---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #850: HAWQ-980. hawq does not handle guc value with spa...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/850
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #835: HAWQ-980. hawq does not handle guc value with spa...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/835
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #835: HAWQ-980. hawq does not handle guc value w...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/835#discussion_r73815359
  
--- Diff: src/backend/utils/misc/guc.c ---
@@ -12216,7 +12216,36 @@ ProcessGUCArray(ArrayType *array, GucSource source)
 * GPSQL needs to dispatch the database/user config to segments.
 */
if (Gp_role == GP_ROLE_DISPATCH)
-   appendStringInfo(>override_options, "-c 
%s=%s ", name, value);
+   {
--- End diff --

It's better to make a function to handle this common logic which process 
the special char. 


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #836: HAWQ-979. Resource Broker Should Reconnect Hadoop...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/836
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #833: HAWQ-978. Fixed deadlock in signal handler which ...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/833
  
+1


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq issue #828: HAWQ-969. Add getting configuration from HDFS and...

[wengyanqing]

Github user wengyanqing commented on the issue:

https://github.com/apache/incubator-hawq/pull/828
  
LGTM


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] incubator-hawq pull request #828: HAWQ-969. Add getting configuration from H...

[wengyanqing]

Github user wengyanqing commented on a diff in the pull request:

https://github.com/apache/incubator-hawq/pull/828#discussion_r73455316
  
--- Diff: src/test/feature/lib/hdfs_config.h ---
@@ -0,0 +1,175 @@
+#ifndef HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_
+#define HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_
+
+#include 
+#include 
+
+#include "psql.h"
+#include "sql_util.h"
+#include "xml_parser.h"
+
+namespace hawq {
+namespace test {
+
+/**
+ * HdfsConfig common libray. Get detailed information about HDFS
+ * including checking state of namenodes and datanodes, get parameter value
+ * @author Chunling Wang
+ */
+class HdfsConfig {
--- End diff --

Should HdfsConfig be singleton ?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---