This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 89c10eb2b1a8393c438af494873304ae392e2e43 Merge: 1cb3eed1da 0f3fb3b0b6 Author: Michael Blow <[email protected]> AuthorDate: Mon Jul 31 22:24:45 2023 -0400 Merge branch 'gerrit/trinity' into 'master' Change-Id: Ia4c6bdfa9dce9437049d59e137494b2372348754 .../http/server/NCQueryCancellationServlet.java | 3 + .../api/http/server/NCQueryServiceServlet.java | 3 +- .../app/active/ActiveEntityEventsListener.java | 10 +- .../app/active/ActiveNotificationHandler.java | 52 +++++----- .../asterix/app/message/CancelQueryRequest.java | 8 +- .../message/ExecuteStatementRequestMessage.java | 4 +- .../org/apache/asterix/app/nc/HaltCallback.java | 4 +- .../apache/asterix/app/nc/NCAppRuntimeContext.java | 16 +--- .../asterix/app/translator/QueryTranslator.java | 4 + .../rebalance/IDatasetRebalanceCallback.java | 8 +- .../rebalance/NoOpDatasetRebalanceCallback.java | 3 +- .../org/apache/asterix/utils/RebalanceUtil.java | 42 +++++--- ...tic_partitioning.conf => cc-cloud-storage.conf} | 5 +- .../src/main/resources/cc_static_partitioning.conf | 6 +- .../asterix/test/common/ResultExtractor.java | 4 +- .../asterix-app/src/test/resources/cc-analyze.conf | 2 +- .../src/test/resources/cc-cloud-storage.conf | 7 -- .../common/api/IDatasetLifecycleManager.java | 6 +- .../common/context/BaseOperationTracker.java | 8 +- .../apache/asterix/common/context/DatasetInfo.java | 31 +++++- .../common/context/DatasetLifecycleManager.java | 4 +- .../context/PrimaryIndexOperationTracker.java | 11 +-- .../ioopcallbacks/LSMIOOperationCallback.java | 6 +- .../ioopcallbacks/LSMIOOperationCallbackTest.java | 7 ++ .../apache/asterix/metadata/RMIClientFactory.java | 13 ++- .../apache/asterix/metadata/RMIServerFactory.java | 25 ++++- .../metadata/bootstrap/AsterixStateProxy.java | 15 +-- .../asterix/metadata/entities/NodeGroup.java | 3 +- .../apache/asterix/metadata/utils/DatasetUtil.java | 2 +- .../replication/api/ReplicationDestination.java | 32 +++++++ .../management/IndexReplicationManager.java | 99 +++++++++---------- .../management/ReplicationOperation.java | 106 +++++++++++++++++++++ .../replication/sync/ReplicaSynchronizer.java | 2 +- .../SecondaryIndexOperationTrackerFactory.java | 4 +- .../service/logging/LogManagerWithReplication.java | 19 +++- asterixdb/pom.xml | 4 +- .../appended-resources/supplemental-models.xml | 97 +++++++++---------- ....com_netty_netty_netty-4.1.94.Final_NOTICE.txt} | 0 ...ent.com_xerial_snappy-java_v1.1.10.1_NOTICE.txt | 16 ++++ .../apache/hyracks/api/comm/NetworkAddress.java | 7 +- .../api/network/INetworkSecurityConfig.java | 9 ++ .../org/apache/hyracks/api/util/InvokeUtil.java | 40 +++++++- .../hyracks/control/cc/work/CancelJobWork.java | 5 + .../cc/work/JobletCleanupNotificationWork.java | 7 +- .../hyracks/control/cc/work/RegisterNodeWork.java | 6 +- .../control/cc/work/WaitForJobCompletionWork.java | 5 + .../hyracks/control/nc/NodeControllerService.java | 2 +- .../hyracks/control/nc/work/AbortTasksWork.java | 5 + .../hyracks/control/nc/work/CleanupJobletWork.java | 7 +- .../hyracks/control/nc/work/StartTasksWork.java | 5 + .../hyracks/http/server/ChunkedResponse.java | 57 +++++++---- .../ipc/security/NetworkSecurityConfig.java | 7 ++ .../impls/AbstractAsynchronousScheduler.java | 52 +++++++++- .../org/apache/hyracks/util/IOThrowingAction.java | 29 ++---- hyracks-fullstack/pom.xml | 25 ++++- 55 files changed, 690 insertions(+), 269 deletions(-) diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 21ce833ac1,9f6a5cbf8e..d41b5499f3 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@@ -4796,23 -4860,11 +4796,27 @@@ public class QueryTranslator extends Ab appCtx.getReceptionist().ensureSchedulable(schedulableRequest); // ensure request not cancelled before running job ensureNotCancelled(clientRequest); - final JobId jobId = JobUtils.runJob(hcc, jobSpec, jobFlags, false); + if (atomicStatement != null) { + Dataset ds = metadataProvider.findDataset(((InsertStatement) atomicStatement).getDataverseName(), + ((InsertStatement) atomicStatement).getDatasetName()); + atomic = ds.isAtomic(); + if (atomic) { + int numParticipatingNodes = appCtx.getNodeJobTracker() + .getJobParticipatingNodes(jobSpec, LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.class) + .size(); + int numParticipatingPartitions = appCtx.getNodeJobTracker().getNumParticipatingPartitions(jobSpec, + LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.class); + List<Integer> participatingDatasetIds = new ArrayList<>(); + participatingDatasetIds.add(ds.getDatasetId()); + jobSpec.setProperty(GlobalTxManager.GlOBAL_TX_PROPERTY_NAME, new GlobalTxInfo( + participatingDatasetIds, numParticipatingNodes, numParticipatingPartitions)); + } + } + jobId = JobUtils.runJob(hcc, jobSpec, jobFlags, false); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("createAndRunJob jobId:{}, uuid:{}", jobId, + requestParameters.getRequestReference().getUuid()); + } clientRequest.setJobId(jobId); if (jId != null) { jId.setValue(jobId); diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java index 1b0b4fdcd5,601cd0260e..837d2cb17a --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java @@@ -20,7 -20,8 +20,9 @@@ package org.apache.asterix.utils import static org.apache.asterix.app.translator.QueryTranslator.abort; import static org.apache.asterix.common.config.DatasetConfig.DatasetType; + import static org.apache.asterix.common.utils.IdentifierUtil.dataset; + import static org.apache.asterix.metadata.utils.DatasetUtil.getFullyQualifiedDisplayName; +import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.ALL_FIELDS_TYPE; import static org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption; import java.rmi.RemoteException; @@@ -83,14 -82,20 +85,15 @@@ public class RebalanceUtil /** * Rebalances an existing dataset to a list of target nodes. * - * @param dataverseName, - * the dataverse name. - * @param datasetName, - * the dataset name. - * @param targetNcNames, - * the list of target nodes. - * @param metadataProvider, - * the metadata provider. - * @param hcc, - * the reusable hyracks connection. + * @param dataverseName, the dataverse name. + * @param datasetName, the dataset name. + * @param targetNcNames, the list of target nodes. + * @param metadataProvider, the metadata provider. + * @param hcc, the reusable hyracks connection. + * @return <code>false</code> if the rebalance was safely skipped * @throws Exception */ - public static void rebalance(DataverseName dataverseName, String datasetName, Set<String> targetNcNames, + public static boolean rebalance(DataverseName dataverseName, String datasetName, Set<String> targetNcNames, MetadataProvider metadataProvider, IHyracksClientConnection hcc, IDatasetRebalanceCallback datasetRebalanceCallback, boolean forceRebalance) throws Exception { Dataset sourceDataset; diff --cc asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf index 08d9dfd177,3923222c56..9884116a32 --- a/asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf +++ b/asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf @@@ -62,4 -65,4 +65,5 @@@ compiler.internal.sanitycheck=tru messaging.frame.size=4096 messaging.frame.count=512 storage.buffercache.pagesize=32KB - storage.partitioning=static + storage.partitioning=static ++storage.compression.block=snappy diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java index 3ec2d22869,9c2b6564ab..2e32fb32a9 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java @@@ -75,13 -66,10 +73,12 @@@ public class PrimaryIndexOperationTrack private boolean flushLogCreated = false; private final Map<String, FlushOperation> scheduledFlushes = new HashMap<>(); private long lastFlushTime = System.nanoTime(); + private final Map<String, FlushOperation> lastFlushOperation = new HashMap<>(); + private final IIndexCheckpointManagerProvider indexCheckpointManagerProvider; public PrimaryIndexOperationTracker(int datasetID, int partition, ILogManager logManager, DatasetInfo dsInfo, - ILSMComponentIdGenerator idGenerator) { + ILSMComponentIdGenerator idGenerator, IIndexCheckpointManagerProvider indexCheckpointManagerProvider) { - super(datasetID, dsInfo); - this.partition = partition; + super(datasetID, dsInfo, partition); this.logManager = logManager; this.numActiveOperations = new AtomicInteger(); this.idGenerator = idGenerator; diff --cc asterixdb/pom.xml index 681e9a8236,c1ee08facf..30a987e7a7 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@@ -2026,13 -1993,8 +2026,13 @@@ <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util-ajax</artifactId> - <version>9.4.48.v20220622</version> + <version>9.4.51.v20230217</version> </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.11.1</version> + </dependency> </dependencies> </dependencyManagement>
