[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...
Github user weinan003 commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1358#discussion_r18481
--- Diff: contrib/vexecutor/nodeVMotion.c ---
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "nodeVMotion.h"
+#include "tuplebatch.h"
+#include "execVQual.h"
+/*=
+ * FUNCTIONS PROTOTYPES
+ */
+static TupleTableSlot *execVMotionSender(MotionState * node);
+static TupleTableSlot *execVMotionUnsortedReceiver(MotionState * node);
+
+TupleTableSlot *ExecVMotion(MotionState * node);
+static void doSendTupleBatch(Motion * motion, MotionState * node,
TupleTableSlot *outerTupleSlot);
+static SendReturnCode
+SendTupleBatch(MotionLayerState *mlStates, ChunkTransportState
*transportStates,
+ int16 motNodeID, TupleBatch tuplebatch, int16 targetRoute);
+/*=
+ */
+
+TupleTableSlot *
+ExecVMotionVirtualLayer(MotionState *node)
+{
+if(node->mstype == MOTIONSTATE_SEND)
+return ExecVMotion(node);
+else if(node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot* slot = node->ps.ps_ResultTupleSlot;
+while(1)
+{
+bool succ = VirtualNodeProc(slot);
+if(!succ)
+{
+slot = ExecVMotion(node);
+if(TupIsNull(slot))
+break;
+else
+continue;
+}
+
+break;
+}
+return slot;
+}
+}
+
+/*
+ * ExecVMotion
+ *
+ */
+TupleTableSlot *
+ExecVMotion(MotionState * node)
+{
+Motion*motion = (Motion *) node->ps.plan;
+
+/*
+ * at the top here we basically decide: -- SENDER vs. RECEIVER and --
+ * SORTED vs. UNSORTED
+ */
+if (node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot *tuple;
+
+if (node->ps.state->active_recv_id >= 0)
+{
+if (node->ps.state->active_recv_id != motion->motionID)
+{
+elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from
%d to %d",
+ node->ps.state->active_recv_id, motion->motionID);
+node->ps.state->active_recv_id = motion->motionID;
+}
+} else
+node->ps.state->active_recv_id = motion->motionID;
+
+/* Running in diagnostic mode ? */
+if (Gp_interconnect_type == INTERCONNECT_TYPE_NIL)
+{
+node->ps.state->active_recv_id = -1;
+return NULL;
+}
+
+Assert(!motion->sendSorted);
+
+tuple = execVMotionUnsortedReceiver(node);
+
+if (tuple == NULL)
+node->ps.state->active_recv_id = -1;
+else
+{
+Gpmon_M_Incr(GpmonPktFromMotionState(node),
GPMON_QEXEC_M_ROWSIN);
--- End diff --
Gpmon in VE model records tuplebatch instead of per tuple
---
[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...
Github user weinan003 commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1358#discussion_r184843784
--- Diff: contrib/vexecutor/nodeVMotion.c ---
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "nodeVMotion.h"
+#include "tuplebatch.h"
+#include "execVQual.h"
+/*=
+ * FUNCTIONS PROTOTYPES
+ */
+static TupleTableSlot *execVMotionSender(MotionState * node);
+static TupleTableSlot *execVMotionUnsortedReceiver(MotionState * node);
+
+TupleTableSlot *ExecVMotion(MotionState * node);
+static void doSendTupleBatch(Motion * motion, MotionState * node,
TupleTableSlot *outerTupleSlot);
+static SendReturnCode
+SendTupleBatch(MotionLayerState *mlStates, ChunkTransportState
*transportStates,
+ int16 motNodeID, TupleBatch tuplebatch, int16 targetRoute);
+/*=
+ */
+
+TupleTableSlot *
+ExecVMotionVirtualLayer(MotionState *node)
+{
+if(node->mstype == MOTIONSTATE_SEND)
+return ExecVMotion(node);
+else if(node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot* slot = node->ps.ps_ResultTupleSlot;
+while(1)
+{
+bool succ = VirtualNodeProc(slot);
+if(!succ)
+{
+slot = ExecVMotion(node);
+if(TupIsNull(slot))
+break;
+else
+continue;
+}
+
+break;
+}
+return slot;
+}
+}
+
+/*
+ * ExecVMotion
+ *
+ */
+TupleTableSlot *
+ExecVMotion(MotionState * node)
+{
+Motion*motion = (Motion *) node->ps.plan;
+
+/*
+ * at the top here we basically decide: -- SENDER vs. RECEIVER and --
+ * SORTED vs. UNSORTED
+ */
+if (node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot *tuple;
+
+if (node->ps.state->active_recv_id >= 0)
+{
+if (node->ps.state->active_recv_id != motion->motionID)
+{
+elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from
%d to %d",
+ node->ps.state->active_recv_id, motion->motionID);
+node->ps.state->active_recv_id = motion->motionID;
+}
+} else
+node->ps.state->active_recv_id = motion->motionID;
+
+/* Running in diagnostic mode ? */
+if (Gp_interconnect_type == INTERCONNECT_TYPE_NIL)
+{
+node->ps.state->active_recv_id = -1;
+return NULL;
+}
+
+Assert(!motion->sendSorted);
+
+tuple = execVMotionUnsortedReceiver(node);
+
+if (tuple == NULL)
+node->ps.state->active_recv_id = -1;
+else
+{
+Gpmon_M_Incr(GpmonPktFromMotionState(node),
GPMON_QEXEC_M_ROWSIN);
+Gpmon_M_Incr_Rows_Out(GpmonPktFromMotionState(node));
+setMotionStatsForGpmon(node);
+}
+#ifdef MEASURE_MOTION_TIME
+gettimeofday(, NULL);
+
+ node->motionTime.tv_sec += stopTime.tv_sec - startTime.tv_sec;
+ node->motionTime.tv_usec += stopTime.tv_usec -
startTime.tv_usec;
+
+ while (node->motionTime.tv_usec < 0)
+ {
+ node->motionTime.tv_usec += 100;
+ node->motionTime.tv_sec--;
+ }
+
+ while (node->motionTime.tv_usec >= 100)
+ {
+ node->motionTime.tv_usec -= 100;
+
[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...
Github user wengyanqing commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1358#discussion_r184842816
--- Diff: contrib/vexecutor/nodeVMotion.c ---
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "nodeVMotion.h"
+#include "tuplebatch.h"
+#include "execVQual.h"
+/*=
+ * FUNCTIONS PROTOTYPES
+ */
+static TupleTableSlot *execVMotionSender(MotionState * node);
+static TupleTableSlot *execVMotionUnsortedReceiver(MotionState * node);
+
+TupleTableSlot *ExecVMotion(MotionState * node);
+static void doSendTupleBatch(Motion * motion, MotionState * node,
TupleTableSlot *outerTupleSlot);
+static SendReturnCode
+SendTupleBatch(MotionLayerState *mlStates, ChunkTransportState
*transportStates,
+ int16 motNodeID, TupleBatch tuplebatch, int16 targetRoute);
+/*=
+ */
+
+TupleTableSlot *
+ExecVMotionVirtualLayer(MotionState *node)
+{
+if(node->mstype == MOTIONSTATE_SEND)
+return ExecVMotion(node);
+else if(node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot* slot = node->ps.ps_ResultTupleSlot;
+while(1)
+{
+bool succ = VirtualNodeProc(slot);
+if(!succ)
+{
+slot = ExecVMotion(node);
+if(TupIsNull(slot))
+break;
+else
+continue;
+}
+
+break;
+}
+return slot;
+}
+}
+
+/*
+ * ExecVMotion
+ *
+ */
+TupleTableSlot *
+ExecVMotion(MotionState * node)
+{
+Motion*motion = (Motion *) node->ps.plan;
+
+/*
+ * at the top here we basically decide: -- SENDER vs. RECEIVER and --
+ * SORTED vs. UNSORTED
+ */
+if (node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot *tuple;
+
+if (node->ps.state->active_recv_id >= 0)
+{
+if (node->ps.state->active_recv_id != motion->motionID)
+{
+elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from
%d to %d",
+ node->ps.state->active_recv_id, motion->motionID);
+node->ps.state->active_recv_id = motion->motionID;
+}
+} else
+node->ps.state->active_recv_id = motion->motionID;
+
+/* Running in diagnostic mode ? */
+if (Gp_interconnect_type == INTERCONNECT_TYPE_NIL)
+{
+node->ps.state->active_recv_id = -1;
+return NULL;
+}
+
+Assert(!motion->sendSorted);
+
+tuple = execVMotionUnsortedReceiver(node);
+
+if (tuple == NULL)
+node->ps.state->active_recv_id = -1;
+else
+{
+Gpmon_M_Incr(GpmonPktFromMotionState(node),
GPMON_QEXEC_M_ROWSIN);
+Gpmon_M_Incr_Rows_Out(GpmonPktFromMotionState(node));
+setMotionStatsForGpmon(node);
+}
+#ifdef MEASURE_MOTION_TIME
+gettimeofday(, NULL);
+
+ node->motionTime.tv_sec += stopTime.tv_sec - startTime.tv_sec;
+ node->motionTime.tv_usec += stopTime.tv_usec -
startTime.tv_usec;
+
+ while (node->motionTime.tv_usec < 0)
+ {
+ node->motionTime.tv_usec += 100;
+ node->motionTime.tv_sec--;
+ }
+
+ while (node->motionTime.tv_usec >= 100)
+ {
+ node->motionTime.tv_usec -= 100;
+
[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...
Github user wengyanqing commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1358#discussion_r184842754
--- Diff: contrib/vexecutor/nodeVMotion.c ---
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "nodeVMotion.h"
+#include "tuplebatch.h"
+#include "execVQual.h"
+/*=
+ * FUNCTIONS PROTOTYPES
+ */
+static TupleTableSlot *execVMotionSender(MotionState * node);
+static TupleTableSlot *execVMotionUnsortedReceiver(MotionState * node);
+
+TupleTableSlot *ExecVMotion(MotionState * node);
+static void doSendTupleBatch(Motion * motion, MotionState * node,
TupleTableSlot *outerTupleSlot);
+static SendReturnCode
+SendTupleBatch(MotionLayerState *mlStates, ChunkTransportState
*transportStates,
+ int16 motNodeID, TupleBatch tuplebatch, int16 targetRoute);
+/*=
+ */
+
+TupleTableSlot *
+ExecVMotionVirtualLayer(MotionState *node)
+{
+if(node->mstype == MOTIONSTATE_SEND)
+return ExecVMotion(node);
+else if(node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot* slot = node->ps.ps_ResultTupleSlot;
+while(1)
+{
+bool succ = VirtualNodeProc(slot);
+if(!succ)
+{
+slot = ExecVMotion(node);
+if(TupIsNull(slot))
+break;
+else
+continue;
+}
+
+break;
+}
+return slot;
+}
+}
+
+/*
+ * ExecVMotion
+ *
+ */
+TupleTableSlot *
+ExecVMotion(MotionState * node)
+{
+Motion*motion = (Motion *) node->ps.plan;
+
+/*
+ * at the top here we basically decide: -- SENDER vs. RECEIVER and --
+ * SORTED vs. UNSORTED
+ */
+if (node->mstype == MOTIONSTATE_RECV)
+{
+TupleTableSlot *tuple;
+
+if (node->ps.state->active_recv_id >= 0)
+{
+if (node->ps.state->active_recv_id != motion->motionID)
+{
+elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from
%d to %d",
+ node->ps.state->active_recv_id, motion->motionID);
+node->ps.state->active_recv_id = motion->motionID;
+}
+} else
+node->ps.state->active_recv_id = motion->motionID;
+
+/* Running in diagnostic mode ? */
+if (Gp_interconnect_type == INTERCONNECT_TYPE_NIL)
+{
+node->ps.state->active_recv_id = -1;
+return NULL;
+}
+
+Assert(!motion->sendSorted);
+
+tuple = execVMotionUnsortedReceiver(node);
+
+if (tuple == NULL)
+node->ps.state->active_recv_id = -1;
+else
+{
+Gpmon_M_Incr(GpmonPktFromMotionState(node),
GPMON_QEXEC_M_ROWSIN);
--- End diff --
Is gpmon info different between tuple and tuplebatch in motion operation ?
---
[GitHub] incubator-hawq pull request #1358: HAWQ-1609. Implement Vectorized Motion No...
Github user wengyanqing commented on a diff in the pull request: https://github.com/apache/incubator-hawq/pull/1358#discussion_r184842905 --- Diff: contrib/vexecutor/tuplebatch.c --- @@ -150,19 +150,26 @@ tbSerialization(TupleBatch tb ) return ret; } -TupleBatch tbDeserialization(unsigned char *buffer) +void tbDeserialization(unsigned char *buffer,TupleBatch* pTB ) --- End diff -- Is tuple batch deserialization always successful ? If not, please use bool for return value type. ---
