This is an automated email from the ASF dual-hosted git repository. xingtanzjr pushed a commit to branch xingtanzjr/mpp_issues in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bc62d7eb24482345bc30545025b39754d87b0f8d Merge: fb7bf77ccc 8a00e48876 Author: Jinrui.Zhang <[email protected]> AuthorDate: Tue Apr 19 15:12:54 2022 +0800 Merge branch 'master' into xingtanzjr/mpp_issues .github/workflows/grafana-plugin.yml | 7 +- .../iotdb/confignode/cli/TemporaryClient.java | 22 +- .../consensus/response/PermissionInfoDataSet.java | 32 +- .../iotdb/confignode/manager/ConfigManager.java | 19 +- .../apache/iotdb/confignode/manager/Manager.java | 14 +- .../confignode/manager/PermissionManager.java | 5 + .../persistence/AuthorInfoPersistence.java | 129 +- .../confignode/service/executor/PlanExecutor.java | 4 +- .../server/ConfigNodeRPCServerProcessor.java | 36 +- .../apache/iotdb/confignode/auth/AuthorTest.java | 44 - .../server/ConfigNodeRPCServerProcessorTest.java | 375 ++ docs/Community/About.md | 79 + grafana-plugin/.gitignore | 1 + grafana-plugin/Magefile.go | 32 + grafana-plugin/README.md | 38 +- grafana-plugin/backend-compile.sh | 36 + grafana-plugin/go.mod | 23 + grafana-plugin/go.sum | 659 +++ grafana-plugin/package.json | 7 +- grafana-plugin/pkg/main.go | 40 + grafana-plugin/pkg/plugin/plugin.go | 291 ++ grafana-plugin/pom.xml | 21 + grafana-plugin/src/QueryEditor.tsx | 2 +- grafana-plugin/src/datasource.ts | 130 +- grafana-plugin/src/plugin.json | 3 + grafana-plugin/yarn.lock | 4726 ++++++++++---------- .../db/integration/IoTDBTriggerExecutionIT.java | 188 +- .../resources/conf/iotdb-engine.properties | 6 - .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 - .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 - .../db/engine/trigger/executor/TriggerEngine.java | 20 +- .../engine/trigger/executor/TriggerExecutor.java | 29 +- .../service/TriggerRegistrationService.java | 52 +- .../iotdb/db/metadata/LocalSchemaProcessor.java | 11 +- .../idtable/entry/InsertMeasurementMNode.java | 20 + .../org/apache/iotdb/db/metadata/mnode/IMNode.java | 8 + .../org/apache/iotdb/db/metadata/mnode/MNode.java | 29 + .../iotdb/db/metadata/mnode/MeasurementMNode.java | 2 - .../mnode/estimator/BasicMNodSizeEstimator.java | 6 +- .../db/metadata/schemaregion/ISchemaRegion.java | 5 +- .../db/metadata/schemaregion/SchemaRegion.java | 11 +- .../schemaregion/rocksdb/RSchemaRegion.java | 6 +- .../schemaregion/rocksdb/mnode/RMNode.java | 20 + .../iotdb/db/mpp/buffer/DataBlockManager.java | 22 +- .../org/apache/iotdb/db/mpp/buffer/SinkHandle.java | 17 +- .../apache/iotdb/db/mpp/buffer/SourceHandle.java | 16 +- .../org/apache/iotdb/db/mpp/common/QueryId.java | 2 +- .../apache/iotdb/db/mpp/execution/Coordinator.java | 2 +- .../mpp/execution/config/AuthorizerConfigTask.java | 95 + .../scheduler/AbstractFragInsStateTracker.java | 5 +- .../scheduler/InternalServiceClientFactory.java | 7 +- .../scheduler/SimpleFragInstanceDispatcher.java | 4 +- .../execution/scheduler/SimpleQueryTerminator.java | 5 +- .../db/mpp/operator/process/TransformOperator.java | 152 +- .../operator/schema/DevicesSchemaScanOperator.java | 12 +- .../db/mpp/operator/schema/SchemaScanOperator.java | 17 +- .../schema/TimeSeriesSchemaScanOperator.java | 12 +- .../apache/iotdb/db/mpp/sql/analyze/Analyzer.java | 8 +- .../db/mpp/sql/planner/LocalExecutionPlanner.java | 8 +- .../iotdb/db/mpp/sql/planner/QueryPlanBuilder.java | 9 +- .../sql/planner/plan/node/PlanGraphPrinter.java | 309 ++ .../db/mpp/sql/planner/plan/node/PlanNodeType.java | 12 +- .../node/metedata/read/DevicesSchemaScanNode.java | 47 +- .../plan/node/metedata/read/SchemaFetchNode.java | 6 - .../plan/node/metedata/read/SchemaScanNode.java | 27 +- .../plan/node/metedata/read/ShowDevicesNode.java | 67 - .../planner/plan/node/metedata/read/ShowNode.java | 36 - .../metedata/read/TimeSeriesSchemaScanNode.java | 51 +- .../plan/node/metedata/write/AuthorNode.java | 2 +- .../metadata/ShowStorageGroupStatement.java | 7 + .../java/org/apache/iotdb/db/service/DataNode.java | 6 +- .../iotdb/db/service/DataNodeManagementServer.java | 102 - .../db/service/DataNodeManagementServerMBean.java | 22 - .../apache/iotdb/db/service/InternalService.java | 4 +- .../iotdb/db/service/InternalServiceImpl.java | 112 - .../handler/DataNodeManagementServiceHandler.java | 58 - .../handler}/InternalServiceThriftHandler.java | 2 +- ...ntServiceImpl.java => InternalServiceImpl.java} | 105 +- .../apache/iotdb/db/mpp/buffer/SinkHandleTest.java | 16 +- .../iotdb/db/mpp/buffer/SourceHandleTest.java | 30 +- .../{ => schema}/SchemaScanOperatorTest.java | 20 +- .../db/mpp/sql/plan/DistributionPlannerTest.java | 1 - .../read/DeviceSchemaScanNodeSerdeTest.java | 76 + .../metadata/read/ShowDevicesNodeSerdeTest.java | 42 - .../read/TimeSeriesSchemaScanNodeSerdeTest.java | 79 + .../plan/node/process/AggregateNodeSerdeTest.java | 15 +- .../node/process/DeviceMergeNodeSerdeTest.java | 17 +- .../plan/node/process/ExchangeNodeSerdeTest.java | 19 +- .../sql/plan/node/process/FillNodeSerdeTest.java | 17 +- .../sql/plan/node/process/FilterNodeSerdeTest.java | 17 +- .../plan/node/process/FilterNullNodeSerdeTest.java | 17 +- .../node/process/GroupByLevelNodeSerdeTest.java | 17 +- .../sql/plan/node/process/LimitNodeSerdeTest.java | 17 +- .../sql/plan/node/process/OffsetNodeSerdeTest.java | 15 +- .../sql/plan/node/process/SortNodeSerdeTest.java | 18 +- .../plan/node/process/TimeJoinNodeSerdeTest.java | 19 +- .../plan/node/sink/FragmentSinkNodeSerdeTest.java | 7 +- .../iotdb/db/service/InternalServiceImplTest.java | 1 + .../src/main/thrift/confignode.thrift | 7 + thrift/src/main/thrift/management.thrift | 77 - thrift/src/main/thrift/mpp.thrift | 92 +- .../tsfile/read/common/block/TsBlockBuilder.java | 15 +- .../iotdb/tsfile/utils/ReadWriteIOUtils.java | 12 + 103 files changed, 5652 insertions(+), 3586 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/mpp/buffer/SinkHandle.java index fc3cd825fe,af4ecf089d..c569c8d04e --- a/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SinkHandle.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SinkHandle.java @@@ -160,10 -160,14 +160,10 @@@ public class SinkHandle implements ISin } private void sendEndOfDataBlockEvent() throws TException { - logger.debug( - "Send end of data block event to plan node {} of {}. {}", - remotePlanNodeId, - remoteFragmentInstanceId, - Thread.currentThread().getName()); + logger.info("[SinkHandle {}]: send end of data block event.", this.getRemotePlanNodeId()); int attempt = 0; - EndOfDataBlockEvent endOfDataBlockEvent = - new EndOfDataBlockEvent( + TEndOfDataBlockEvent endOfDataBlockEvent = + new TEndOfDataBlockEvent( remoteFragmentInstanceId, remotePlanNodeId, localFragmentInstanceId, diff --cc server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java index 01dc2f0fb1,aece366b82..d044be172e --- a/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java @@@ -316,14 -307,15 +316,14 @@@ public class SourceHandle implements IS @Override public void run() { - logger.debug( - "Get data blocks [{}, {}) from {} for plan node {} of {}.", + logger.info( + "[SourceHandle-{}]: Get data blocks [{}, {}) from {}", + localPlanNodeId, startSequenceId, endSequenceId, - remoteFragmentInstanceId, - localPlanNodeId, - localFragmentInstanceId); + remoteFragmentInstanceId); - GetDataBlockRequest req = - new GetDataBlockRequest(remoteFragmentInstanceId, startSequenceId, endSequenceId); + TGetDataBlockRequest req = + new TGetDataBlockRequest(remoteFragmentInstanceId, startSequenceId, endSequenceId); int attempt = 0; while (attempt < MAX_ATTEMPT_TIMES) { attempt += 1; diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/scheduler/SimpleFragInstanceDispatcher.java index b39ec04406,17a193d9b4..40b4df4a63 --- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/scheduler/SimpleFragInstanceDispatcher.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/scheduler/SimpleFragInstanceDispatcher.java @@@ -48,29 -46,33 +48,29 @@@ public class SimpleFragInstanceDispatch return executor.submit( () -> { TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(false); - try { - for (FragmentInstance instance : instances) { - // TODO: (jackie tien) change the port - InternalService.Iface client = - InternalServiceClientFactory.getInternalServiceClient( - new Endpoint( - instance.getHostEndpoint().getIp(), - IoTDBDescriptor.getInstance().getConfig().getInternalPort())); - // TODO: (xingtanzjr) consider how to handle the buffer here - ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024); - instance.serializeRequest(buffer); - buffer.flip(); - TConsensusGroupId groupId = - new TConsensusGroupId( - instance.getRegionReplicaSet().getConsensusGroupId().getId(), - instance.getRegionReplicaSet().getConsensusGroupId().getType().toString()); - TSendFragmentInstanceReq req = - new TSendFragmentInstanceReq( - new TFragmentInstance(buffer), groupId, instance.getType().toString()); - resp = client.sendFragmentInstance(req); - if (!resp.accepted) { - break; - } + for (FragmentInstance instance : instances) { + InternalService.Iface client = - InternalServiceClientFactory.getMppServiceClient( ++ InternalServiceClientFactory.getInternalServiceClient( + new Endpoint( + instance.getHostEndpoint().getIp(), - IoTDBDescriptor.getInstance().getConfig().getMppPort())); ++ IoTDBDescriptor.getInstance().getConfig().getInternalPort())); + // TODO: (xingtanzjr) consider how to handle the buffer here + ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024); + instance.serializeRequest(buffer); + buffer.flip(); + TConsensusGroupId groupId = + new TConsensusGroupId( + instance.getRegionReplicaSet().getConsensusGroupId().getId(), + instance.getRegionReplicaSet().getConsensusGroupId().getType().toString()); + TSendFragmentInstanceReq req = + new TSendFragmentInstanceReq( + new TFragmentInstance(buffer), groupId, instance.getType().toString()); + LOGGER.info("start to dispatch fragment instance: {}", instance.getId()); + resp = client.sendFragmentInstance(req); + LOGGER.info("dispatch complete: {}", instance.getId()); + if (!resp.accepted) { + break; } - } catch (Exception e) { - // TODO: (xingtanzjr) add more details - return new FragInstanceDispatchResult(false); } return new FragInstanceDispatchResult(resp.accepted); });
