http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj index 0cc6789..59ed8ae 100644 --- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj +++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj @@ -448,7 +448,6 @@ DatasetDecl DatasetSpecification() throws ParseException: DatasetDecl dsetDecl = null; boolean autogenerated = false; String compactionPolicy = null; - boolean temp = false; Pair<Integer, List<String>> filterField = null; Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null); } @@ -480,10 +479,7 @@ DatasetDecl DatasetSpecification() throws ParseException: ifNotExists); } - | (<INTERNAL> | <TEMPORARY> { - temp = token.image.toLowerCase().equals("temporary"); - } - )? + | ( <INTERNAL> )? <DATASET> nameComponents = QualifiedName() <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN> ( @@ -511,8 +507,7 @@ DatasetDecl DatasetSpecification() throws ParseException: InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated, - filterField == null? null : filterField.second, - temp); + filterField == null? null : filterField.second); dsetDecl = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, @@ -2729,7 +2724,7 @@ TOKEN : | <START: "start"> | <STOP: "stop"> | <SOME : "some"> - | <TEMPORARY : "temporary"> + | <TEMPORARY : "temporary"> // intentionally not used but reserved for future usage | <THEN : "then"> | <TO : "to"> | <TYPE : "type">
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java index b15579c..db599c4 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ExternalDetailsDecl.java @@ -40,9 +40,4 @@ public class ExternalDetailsDecl implements IDatasetDetailsDecl { return properties; } - @Override - public boolean isTemp() { - return false; - } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IDatasetDetailsDecl.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IDatasetDetailsDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IDatasetDetailsDecl.java index da27f17..7f48c32 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IDatasetDetailsDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IDatasetDetailsDecl.java @@ -20,6 +20,4 @@ package org.apache.asterix.lang.common.statement; public interface IDatasetDetailsDecl { - public boolean isTemp(); - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java index 41d97a8..f312ddd 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java @@ -24,16 +24,14 @@ public class InternalDetailsDecl implements IDatasetDetailsDecl { private final List<List<String>> partitioningExprs; private final List<Integer> keySourceIndicators; private final boolean autogenerated; - private final boolean temp; private final List<String> filterField; public InternalDetailsDecl(List<List<String>> partitioningExpr, List<Integer> keySourceIndicators, - boolean autogenerated, List<String> filterField, boolean temp) { + boolean autogenerated, List<String> filterField) { this.partitioningExprs = partitioningExpr; this.keySourceIndicators = keySourceIndicators; this.autogenerated = autogenerated; this.filterField = filterField; - this.temp = temp; } public List<List<String>> getPartitioningExprs() { @@ -48,11 +46,6 @@ public class InternalDetailsDecl implements IDatasetDetailsDecl { return autogenerated; } - @Override - public boolean isTemp() { - return temp; - } - public List<String> getFilterField() { return filterField; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java index 5a3a530..21a4b76 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java @@ -468,8 +468,7 @@ public class FormatPrintVisitor implements ILangVisitor<Void, Integer> { @Override public Void visit(DatasetDecl dd, Integer step) throws CompilationException { if (dd.getDatasetType() == DatasetType.INTERNAL) { - String temp = dd.getDatasetDetailsDecl().isTemp() ? "temporary" : ""; - out.print(skip(step) + "create " + temp + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName()) + out.print(skip(step) + "create " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName()) + generateIfNotExists(dd.getIfNotExists()) + "(" + dd.getQualifiedTypeName() + ")" + " primary key "); printDelimitedKeys(((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs(), ","); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index a58ce0f..7ac8c57 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -501,7 +501,6 @@ DatasetDecl DatasetSpecification() throws ParseException: DatasetDecl dsetDecl = null; boolean autogenerated = false; String compactionPolicy = null; - boolean temp = false; Pair<Integer, List<String>> filterField = null; Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null); } @@ -533,7 +532,7 @@ DatasetDecl DatasetSpecification() throws ParseException: ifNotExists); } - | (<INTERNAL> | <TEMPORARY> { temp = true; })? + | ( <INTERNAL> )? Dataset() nameComponents = QualifiedName() <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN> ( @@ -561,8 +560,7 @@ DatasetDecl DatasetSpecification() throws ParseException: InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated, - filterField == null? null : filterField.second, - temp); + filterField == null? null : filterField.second); dsetDecl = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, @@ -3265,7 +3263,7 @@ TOKEN [IGNORE_CASE]: | <SOME : "some"> | <START : "start"> | <STOP : "stop"> - | <TEMPORARY : "temporary"> + | <TEMPORARY : "temporary"> // intentionally not used but reserved for future usage | <THEN : "then"> | <TYPE : "type"> | <TO : "to"> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/GarbageCollector.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/GarbageCollector.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/GarbageCollector.java deleted file mode 100644 index a97e22a..0000000 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/GarbageCollector.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.metadata; - -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Periodically recycle temporary datasets. - * - * @author yingyib - */ -public class GarbageCollector implements Runnable { - private static final Logger LOGGER = Logger.getLogger(GarbageCollector.class.getName()); - - // TODO(mblow): make this configurable - private static final long CLEANUP_PERIOD = 1; - private static final TimeUnit CLEANUP_PERIOD_UNIT = TimeUnit.DAYS; - - static { - // Starts the garbage collector thread which - // should always be running. - Thread gcThread = new Thread(new GarbageCollector(), "Metadata GC"); - gcThread.setDaemon(true); - gcThread.start(); - } - - @Override - @SuppressWarnings({"squid:S2142", "squid:S2189"}) // rethrow/interrupt thread on InterruptedException, endless loop - public void run() { - LOGGER.info("Starting Metadata GC"); - while (true) { - try { - synchronized (this) { - CLEANUP_PERIOD_UNIT.timedWait(this, CLEANUP_PERIOD); - } - MetadataManager.INSTANCE.cleanupTempDatasets(); - } catch (InterruptedException e) { - break; - } catch (Exception e) { - LOGGER.log(Level.WARNING, "Exception cleaning temp datasets", e); - } - } - LOGGER.info("Exiting Metadata GC"); - } - - public static void ensure() { - // no need to do anything, <clinit> does the work - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/IDatasetDetails.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/IDatasetDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/IDatasetDetails.java index b9ced54..47c6432 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/IDatasetDetails.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/IDatasetDetails.java @@ -26,28 +26,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; public interface IDatasetDetails extends Serializable { - public DatasetType getDatasetType(); + DatasetType getDatasetType(); - public void writeDatasetDetailsRecordType(DataOutput out) throws HyracksDataException; - - /** - * @return if the dataset is a temporary dataset. - * Here is a summary of temporary datasets: - * 1. Different from a persistent dataset, reads and writes over a temporary dataset do not require any lock. - * Writes over a temporary dataset do not generate any write-ahead update and commit log but generate - * flush log and job commit log. - * 2. A temporary dataset can only be an internal dataset, stored in partitioned LSM-Btrees. - * 3. All secondary indexes for persistent datasets are supported for temporary datasets. - * 4. A temporary dataset will be automatically garbage collected if it is not active in the past 30 days. - * A temporary dataset could be used for the following scenarios: - * 1. A data scientist wants to run some one-time data analysis queries over a dataset that s/he pre-processed - * and the dataset is only used by her/himself in an one-query-at-a-time manner. - * 2. Articulate AQL with external systems such as Pregelix/IMRU/Spark. A user can first run an AQL - * query to populate a temporary dataset, then kick off an external runtime to read this dataset, - * dump the results of the external runtime to yet-another-temporary dataset, and finally run yet-another AQL - * over the second temporary dataset. - */ - public boolean isTemp(); - - public long getLastAccessTime(); + void writeDatasetDetailsRecordType(DataOutput out) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java index 8f88ca7..4f827f5 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java @@ -50,8 +50,6 @@ import org.apache.asterix.metadata.utils.IndexUtil; */ public class MetadataCache { - // Default life time period of a temp dataset. It is 30 days. - private final static long TEMP_DATASET_INACTIVE_TIME_THRESHOLD = 3600 * 24 * 30 * 1000L; // Key is dataverse name. protected final Map<String, Dataverse> dataverses = new HashMap<>(); // Key is dataverse name. Key of value map is dataset name. @@ -574,28 +572,6 @@ public class MetadataCache { } /** - * Clean up temp datasets that are expired. - * The garbage collection will pause other dataset operations. - */ - public void cleanupTempDatasets() { - synchronized (datasets) { - for (Map<String, Dataset> map : datasets.values()) { - Iterator<Dataset> datasetIterator = map.values().iterator(); - while (datasetIterator.hasNext()) { - Dataset dataset = datasetIterator.next(); - if (dataset.getDatasetDetails().isTemp()) { - long currentTime = System.currentTimeMillis(); - long duration = currentTime - dataset.getDatasetDetails().getLastAccessTime(); - if (duration > TEMP_DATASET_INACTIVE_TIME_THRESHOLD) { - datasetIterator.remove(); - } - } - } - } - } - } - - /** * Represents a logical operation against the metadata. */ protected class MetadataLogicalOperation { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java index 1384d26..f8adc81 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java @@ -118,7 +118,7 @@ public class MetadataManager implements IMetadataManager { @Override public void init() throws HyracksDataException { - GarbageCollector.ensure(); + // no op } @Override @@ -215,18 +215,6 @@ public class MetadataManager implements IMetadataManager { public List<Dataset> getDataverseDatasets(MetadataTransactionContext ctx, String dataverseName) throws AlgebricksException { List<Dataset> dataverseDatasets = new ArrayList<>(); - // add uncommitted temporary datasets - for (Dataset dataset : ctx.getDataverseDatasets(dataverseName)) { - if (dataset.getDatasetDetails().isTemp()) { - dataverseDatasets.add(dataset); - } - } - // add the committed temporary datasets with the cache - for (Dataset dataset : cache.getDataverseDatasets(dataverseName)) { - if (dataset.getDatasetDetails().isTemp()) { - dataverseDatasets.add(dataset); - } - } try { // Assuming that the transaction can read its own writes on the // metadata node. @@ -242,14 +230,11 @@ public class MetadataManager implements IMetadataManager { @Override public void addDataset(MetadataTransactionContext ctx, Dataset dataset) throws AlgebricksException { // add dataset into metadataNode - if (!dataset.getDatasetDetails().isTemp()) { - try { - metadataNode.addDataset(ctx.getTxnId(), dataset); - } catch (RemoteException e) { - throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); - } + try { + metadataNode.addDataset(ctx.getTxnId(), dataset); + } catch (RemoteException e) { + throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); } - // reflect the dataset into the cache ctx.addDataset(dataset); } @@ -257,16 +242,11 @@ public class MetadataManager implements IMetadataManager { @Override public void dropDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName) throws AlgebricksException { - Dataset dataset = findDataset(ctx, dataverseName, datasetName); - // If a dataset is not in the cache, then it could not be a temp dataset - if (dataset == null || !dataset.getDatasetDetails().isTemp()) { - try { - metadataNode.dropDataset(ctx.getTxnId(), dataverseName, datasetName); - } catch (RemoteException e) { - throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); - } + try { + metadataNode.dropDataset(ctx.getTxnId(), dataverseName, datasetName); + } catch (RemoteException e) { + throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); } - // Drops the dataset from cache ctx.dropDataset(dataverseName, datasetName); } @@ -315,16 +295,10 @@ public class MetadataManager implements IMetadataManager { if (dataset == null) { return datasetIndexes; } - if (dataset.getDatasetDetails().isTemp()) { - // for temp datsets - datasetIndexes = cache.getDatasetIndexes(dataverseName, datasetName); - } else { - try { - // for persistent datasets - datasetIndexes = metadataNode.getDatasetIndexes(ctx.getTxnId(), dataverseName, datasetName); - } catch (RemoteException e) { - throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); - } + try { + datasetIndexes = metadataNode.getDatasetIndexes(ctx.getTxnId(), dataverseName, datasetName); + } catch (RemoteException e) { + throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); } return datasetIndexes; } @@ -423,15 +397,10 @@ public class MetadataManager implements IMetadataManager { @Override public void addIndex(MetadataTransactionContext ctx, Index index) throws AlgebricksException { - String dataverseName = index.getDataverseName(); - String datasetName = index.getDatasetName(); - Dataset dataset = findDataset(ctx, dataverseName, datasetName); - if (dataset == null || !dataset.getDatasetDetails().isTemp()) { - try { - metadataNode.addIndex(ctx.getTxnId(), index); - } catch (RemoteException e) { - throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); - } + try { + metadataNode.addIndex(ctx.getTxnId(), index); + } catch (RemoteException e) { + throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); } ctx.addIndex(index); } @@ -450,17 +419,10 @@ public class MetadataManager implements IMetadataManager { @Override public void dropIndex(MetadataTransactionContext ctx, String dataverseName, String datasetName, String indexName) throws AlgebricksException { - Dataset dataset = findDataset(ctx, dataverseName, datasetName); - // If a dataset is not in the cache, then it could be an unloaded persistent - // dataset. - // If the dataset is a temp dataset, then we do not need to call any MedataNode - // operations. - if (dataset == null || !dataset.getDatasetDetails().isTemp()) { - try { - metadataNode.dropIndex(ctx.getTxnId(), dataverseName, datasetName, indexName); - } catch (RemoteException e) { - throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); - } + try { + metadataNode.dropIndex(ctx.getTxnId(), dataverseName, datasetName, indexName); + } catch (RemoteException e) { + throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); } ctx.dropIndex(dataverseName, datasetName, indexName); } @@ -982,11 +944,6 @@ public class MetadataManager implements IMetadataManager { ctx.addDataset(dataset); } - @Override - public void cleanupTempDatasets() { - cache.cleanupTempDatasets(); - } - public Dataset findDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName) { Dataset dataset = ctx.getDataset(dataverseName, datasetName); if (dataset == null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java index 43a927b..b2d0d3e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java @@ -661,13 +661,6 @@ public interface IMetadataManager extends IMetadataBootstrap { void updateDataset(MetadataTransactionContext ctx, Dataset dataset) throws AlgebricksException; /** - * Clean up temporary datasets that have not been active for a long time. - * - * @throws AlgebricksException - */ - void cleanupTempDatasets() throws AlgebricksException; - - /** * Add an extension entity to its extension dataset under the ongoing metadata * transaction * http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java index bff721e..4c98904 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java @@ -206,7 +206,7 @@ public class MetadataBootstrap { for (int i = 0; i < indexes.length; i++) { IDatasetDetails id = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH, indexes[i].getPartitioningExpr(), indexes[i].getPartitioningExpr(), null, - indexes[i].getPartitioningExprType(), false, null, false); + indexes[i].getPartitioningExprType(), false, null); MetadataManager.INSTANCE.addDataset(mdTxnCtx, new Dataset(indexes[i].getDataverseName(), indexes[i].getIndexedDatasetName(), indexes[i].getDataverseName(), indexes[i].getPayloadRecordType().getTypeName(), http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java index bfc6a8e..301aafb 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeResourceFactoryProvider.java @@ -69,7 +69,6 @@ public class BTreeResourceFactoryProvider implements IResourceFactoryProvider { ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType); IBinaryComparatorFactory[] cmpFactories = getCmpFactories(mdProvider, dataset, index, recordType, metaType); int[] bloomFilterFields = getBloomFilterFields(dataset, index); - boolean durable = !dataset.isTemp(); double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(); ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index); ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index); @@ -84,12 +83,12 @@ public class BTreeResourceFactoryProvider implements IResourceFactoryProvider { ? new ExternalBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, - mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, + mergePolicyFactory, mergePolicyProperties, true, bloomFilterFields, bloomFilterFalsePositiveRate, false, btreeFields) : new ExternalBTreeWithBuddyLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, - mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, + mergePolicyFactory, mergePolicyProperties, true, bloomFilterFields, bloomFilterFalsePositiveRate, false, btreeFields); case INTERNAL: AsterixVirtualBufferCacheProvider vbcProvider = @@ -97,7 +96,7 @@ public class BTreeResourceFactoryProvider implements IResourceFactoryProvider { return new LSMBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, - mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, + mergePolicyProperties, true, bloomFilterFields, bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields); default: throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 71ed913..305cdfa 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -158,7 +158,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> private IResultSerializerFactoryProvider resultSerializerFactoryProvider; private TxnId txnId; private Map<String, Integer> externalDataLocks; - private boolean isTemporaryDatasetWriteJob = true; private boolean blockingOperatorDisabled = false; public MetadataProvider(ICcApplicationContext appCtx, Dataverse defaultDataverse) { @@ -257,11 +256,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> return isWriteTransaction; } - public boolean isTemporaryDatasetWriteJob() { - // The transaction only writes temporary datasets. - return isTemporaryDatasetWriteJob; - } - public IFunctionManager getFunctionManager() { return functionManager; } @@ -569,8 +563,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> fieldPermutation[numKeys + 1] = idx; } - boolean temp = dataset.getDatasetDetails().isTemp(); - isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp; Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = getSplitProviderAndConstraints(dataset); long numElementsHint = getCardinalityPerPartitionHint(dataset); @@ -820,8 +812,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> throw new AlgebricksException( "Unknown dataset " + datasetName + " in dataverse " + dataSource.getId().getDataverseName()); } - boolean temp = dataset.getDatasetDetails().isTemp(); - isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp; int numKeys = primaryKeys.size(); int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1; int numOfAdditionalFields = additionalNonFilterFields == null ? 0 : additionalNonFilterFields.size(); @@ -933,9 +923,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> String datasetName = dataSource.getId().getDatasourceName(); Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataSource.getId().getDataverseName(), datasetName); - boolean temp = dataset.getDatasetDetails().isTemp(); - isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp; - int numKeys = keys.size(); int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1; // Move key fields to front. @@ -1040,9 +1027,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> JobSpecification spec, IndexOperation indexOp, boolean bulkload, List<LogicalVariable> prevSecondaryKeys, List<LogicalVariable> prevAdditionalFilteringKeys) throws AlgebricksException { Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataverseName, datasetName); - boolean temp = dataset.getDatasetDetails().isTemp(); - isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp; - int numKeys = primaryKeys.size() + secondaryKeys.size(); int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1; @@ -1128,9 +1112,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> JobSpecification spec, IndexOperation indexOp, boolean bulkload, List<LogicalVariable> prevSecondaryKeys, List<LogicalVariable> prevAdditionalFilteringKeys) throws AlgebricksException { Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataverseName, datasetName); - boolean temp = dataset.getDatasetDetails().isTemp(); - isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp; - String itemTypeName = dataset.getItemTypeName(); IAType itemType = MetadataManager.INSTANCE .getDatatype(mdTxnCtx, dataset.getItemTypeDataverseName(), itemTypeName).getDatatype(); @@ -1245,9 +1226,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> throw new AlgebricksException("Cannot create composite inverted index on multiple fields."); } Dataset dataset = MetadataManagerUtil.findExistingDataset(mdTxnCtx, dataverseName, datasetName); - boolean temp = dataset.getDatasetDetails().isTemp(); - isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp; - // For tokenization, sorting and loading. // One token (+ optional partitioning field) + primary keys: [token, // number of token, PK] http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java index 48a6e6a..e6c0de8 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java @@ -30,8 +30,8 @@ import java.util.stream.IntStream; import org.apache.asterix.active.IActiveEntityEventsListener; import org.apache.asterix.active.IActiveNotificationHandler; import org.apache.asterix.common.config.DatasetConfig.DatasetType; -import org.apache.asterix.common.context.DatasetLSMComponentIdGeneratorFactory; import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory; +import org.apache.asterix.common.context.DatasetLSMComponentIdGeneratorFactory; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.NoOpFrameOperationCallbackFactory; import org.apache.asterix.common.exceptions.CompilationException; @@ -76,8 +76,6 @@ import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperati import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexModificationOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerFactory; import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexSearchOperationCallbackFactory; -import org.apache.asterix.transaction.management.opcallbacks.TempDatasetPrimaryIndexModificationOperationCallbackFactory; -import org.apache.asterix.transaction.management.opcallbacks.TempDatasetSecondaryIndexModificationOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.UpsertOperationCallbackFactory; import org.apache.asterix.transaction.management.resource.DatasetLocalResourceFactory; import org.apache.asterix.transaction.management.runtime.CommitRuntimeFactory; @@ -556,17 +554,15 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { */ public ISearchOperationCallbackFactory getSearchCallbackFactory(IStorageComponentProvider storageComponentProvider, Index index, TxnId txnId, IndexOperation op, int[] primaryKeyFields) throws AlgebricksException { - if (getDatasetDetails().isTemp()) { - return NoOpOperationCallbackFactory.INSTANCE; - } else if (index.isPrimaryIndex()) { - /** + if (index.isPrimaryIndex()) { + /* * Due to the read-committed isolation level, * we may acquire very short duration lock(i.e., instant lock) for readers. */ - return (op == IndexOperation.UPSERT) - ? new LockThenSearchOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, - storageComponentProvider.getTransactionSubsystemProvider(), ResourceType.LSM_BTREE) - : new PrimaryIndexInstantSearchOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, + return (op == IndexOperation.UPSERT) ? + new LockThenSearchOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, + storageComponentProvider.getTransactionSubsystemProvider(), ResourceType.LSM_BTREE) : + new PrimaryIndexInstantSearchOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, storageComponentProvider.getTransactionSubsystemProvider(), ResourceType.LSM_BTREE); } else if (index.getKeyFieldNames().isEmpty()) { // this is the case where the index is secondary primary index and locking is required @@ -596,32 +592,22 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { public IModificationOperationCallbackFactory getModificationCallbackFactory( IStorageComponentProvider componentProvider, Index index, TxnId txnId, IndexOperation op, int[] primaryKeyFields) throws AlgebricksException { - if (getDatasetDetails().isTemp()) { - return op == IndexOperation.DELETE || op == IndexOperation.INSERT || op == IndexOperation.UPSERT - ? index.isPrimaryIndex() - ? new TempDatasetPrimaryIndexModificationOperationCallbackFactory(txnId, datasetId, - primaryKeyFields, componentProvider.getTransactionSubsystemProvider(), - Operation.get(op), index.resourceType()) - : new TempDatasetSecondaryIndexModificationOperationCallbackFactory(txnId, getDatasetId(), - primaryKeyFields, componentProvider.getTransactionSubsystemProvider(), - Operation.get(op), index.resourceType()) - : NoOpOperationCallbackFactory.INSTANCE; - } else if (index.isPrimaryIndex()) { - return op == IndexOperation.UPSERT - ? new UpsertOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, + if (index.isPrimaryIndex()) { + return op == IndexOperation.UPSERT ? + new UpsertOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, componentProvider.getTransactionSubsystemProvider(), Operation.get(op), - index.resourceType()) - : op == IndexOperation.DELETE || op == IndexOperation.INSERT - ? new PrimaryIndexModificationOperationCallbackFactory(txnId, getDatasetId(), + index.resourceType()) : + op == IndexOperation.DELETE || op == IndexOperation.INSERT ? + new PrimaryIndexModificationOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, componentProvider.getTransactionSubsystemProvider(), - Operation.get(op), index.resourceType()) - : NoOpOperationCallbackFactory.INSTANCE; + Operation.get(op), index.resourceType()) : + NoOpOperationCallbackFactory.INSTANCE; } else { - return op == IndexOperation.DELETE || op == IndexOperation.INSERT || op == IndexOperation.UPSERT - ? new SecondaryIndexModificationOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, + return op == IndexOperation.DELETE || op == IndexOperation.INSERT || op == IndexOperation.UPSERT ? + new SecondaryIndexModificationOperationCallbackFactory(txnId, getDatasetId(), primaryKeyFields, componentProvider.getTransactionSubsystemProvider(), Operation.get(op), - index.resourceType()) - : NoOpOperationCallbackFactory.INSTANCE; + index.resourceType()) : + NoOpOperationCallbackFactory.INSTANCE; } } @@ -678,18 +664,13 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { int[] primaryKeyFieldPermutation, boolean isSink) throws AlgebricksException { int[] datasetPartitions = getDatasetPartitions(metadataProvider); return new CommitRuntimeFactory(txnId, datasetId, primaryKeyFieldPermutation, - metadataProvider.isTemporaryDatasetWriteJob(), metadataProvider.isWriteTransaction(), datasetPartitions, - isSink); + metadataProvider.isWriteTransaction(), datasetPartitions, isSink); } public IFrameOperationCallbackFactory getFrameOpCallbackFactory() { return NoOpFrameOperationCallbackFactory.INSTANCE; } - public boolean isTemp() { - return getDatasetDetails().isTemp(); - } - public boolean isCorrelated() { return CorrelatedPrefixMergePolicyFactory.NAME.equals(compactionPolicyFactory); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/ExternalDatasetDetails.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/ExternalDatasetDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/ExternalDatasetDetails.java index b9b4cd9..ea514e4 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/ExternalDatasetDetails.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/ExternalDatasetDetails.java @@ -54,7 +54,6 @@ public class ExternalDatasetDetails implements IDatasetDetails { private static final Logger LOGGER = Logger.getLogger(ExternalDatasetDetails.class.getName()); private final String adapter; private final Map<String, String> properties; - private final long addToCacheTime; private Date lastRefreshTime; private TransactionState state; @@ -62,7 +61,6 @@ public class ExternalDatasetDetails implements IDatasetDetails { TransactionState state) { this.properties = properties; this.adapter = adapter; - this.addToCacheTime = System.currentTimeMillis(); this.lastRefreshTime = lastRefreshTime; this.state = state; } @@ -132,16 +130,6 @@ public class ExternalDatasetDetails implements IDatasetDetails { externalRecordBuilder.write(out, true); } - @Override - public boolean isTemp() { - return false; - } - - @Override - public long getLastAccessTime() { - return addToCacheTime; - } - public Date getTimestamp() { return lastRefreshTime; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java index 4b64f47..80c4f3d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java @@ -61,8 +61,6 @@ public class InternalDatasetDetails implements IDatasetDetails { private final List<List<String>> primaryKeys; private final List<IAType> primaryKeyTypes; private final boolean autogenerated; - private final boolean temp; - private long lastAccessTime; private final List<String> filterField; private final List<Integer> keySourceIndicators; @@ -71,7 +69,7 @@ public class InternalDatasetDetails implements IDatasetDetails { public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy, List<List<String>> partitioningKey, List<List<String>> primaryKey, List<Integer> keyFieldIndicators, - List<IAType> primaryKeyType, boolean autogenerated, List<String> filterField, boolean temp) { + List<IAType> primaryKeyType, boolean autogenerated, List<String> filterField) { this.fileStructure = fileStructure; this.partitioningStrategy = partitioningStrategy; this.partitioningKeys = partitioningKey; @@ -87,8 +85,6 @@ public class InternalDatasetDetails implements IDatasetDetails { this.primaryKeyTypes = primaryKeyType; this.autogenerated = autogenerated; this.filterField = filterField; - this.temp = temp; - this.lastAccessTime = System.currentTimeMillis(); } public List<List<String>> getPartitioningKey() { @@ -125,21 +121,10 @@ public class InternalDatasetDetails implements IDatasetDetails { @Override public DatasetType getDatasetType() { - lastAccessTime = System.currentTimeMillis(); return DatasetType.INTERNAL; } @Override - public long getLastAccessTime() { - return lastAccessTime; - } - - @Override - public boolean isTemp() { - return temp; - } - - @Override public void writeDatasetDetailsRecordType(DataOutput out) throws HyracksDataException { IARecordBuilder internalRecordBuilder = new RecordBuilder(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java index c274833..d5e179b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java @@ -207,13 +207,8 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> { } } - // Temporary dataset only lives in the compiler therefore the temp field is - // false. - // DatasetTupleTranslator always read from the metadata node, so the temp flag - // should be always false. datasetDetails = new InternalDatasetDetails(fileStructure, partitioningStrategy, partitioningKey, - partitioningKey, keyFieldSourceIndicator, partitioningKeyType, autogenerated, filterField, - false); + partitioningKey, keyFieldSourceIndicator, partitioningKeyType, autogenerated, filterField); break; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java index e4c4860..5973c06 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java @@ -88,7 +88,6 @@ import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory import org.apache.hyracks.storage.am.common.dataflow.IndexCreateOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor; -import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor; @@ -297,7 +296,7 @@ public class DatasetUtil { compactionInfo.first, compactionInfo.second); IndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), - splitsAndConstraint.first, resourceFactory, !dataset.isTemp()); + splitsAndConstraint.first, resourceFactory, true); IndexCreateOperatorDescriptor indexCreateOp = new IndexCreateOperatorDescriptor(spec, indexBuilderFactory); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp, splitsAndConstraint.second); @@ -352,9 +351,8 @@ public class DatasetUtil { // +Infinity int[] highKeyFields = null; ITransactionSubsystemProvider txnSubsystemProvider = TransactionSubsystemProvider.INSTANCE; - boolean temp = dataset.getDatasetDetails().isTemp(); - ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE - : new PrimaryIndexInstantSearchOperationCallbackFactory(txnId, dataset.getDatasetId(), + ISearchOperationCallbackFactory searchCallbackFactory = + new PrimaryIndexInstantSearchOperationCallbackFactory(txnId, dataset.getDatasetId(), dataset.getPrimaryBloomFilterFields(), txnSubsystemProvider, IRecoveryManager.ResourceType.LSM_BTREE); IndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory( http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java index 5bb0aa9..d625dd0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java @@ -202,7 +202,7 @@ public class ExternalIndexingOperations { IResourceFactory resourceFactory = dataset.getResourceFactory(metadataProvider, fileIndex, recordType, null, mergePolicyFactory, mergePolicyProperties); IIndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), - secondaryFileSplitProvider, resourceFactory, !dataset.isTemp()); + secondaryFileSplitProvider, resourceFactory, true); IIndexDataflowHelperFactory dataflowHelperFactory = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), secondaryFileSplitProvider); ExternalFilesIndexCreateOperatorDescriptor externalFilesOp = new ExternalFilesIndexCreateOperatorDescriptor( http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java index 362305e..43cc0ec 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/InvertedIndexResourceFactoryProvider.java @@ -110,7 +110,6 @@ public class InvertedIndexResourceFactoryProvider implements IResourceFactoryPro AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()); ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider(); - boolean durable = !dataset.isTemp(); double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(); ITypeTraits[] typeTraits = getInvListTypeTraits(mdProvider, dataset, recordType, metaType); IBinaryComparatorFactory[] cmpFactories = @@ -122,7 +121,7 @@ public class InvertedIndexResourceFactoryProvider implements IResourceFactoryPro return new LSMInvertedIndexLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, - durable, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, isPartitioned, invertedIndexFields, + true, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, isPartitioned, invertedIndexFields, secondaryFilterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, bloomFilterFalsePositiveRate); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java index 425f8a1..5075adc 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java @@ -142,7 +142,6 @@ public class RTreeResourceFactoryProvider implements IResourceFactoryProvider { storageComponentProvider.getMetadataPageManagerFactory(); ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider(); - boolean durable = !dataset.isTemp(); ILinearizeComparatorFactory linearizeCmpFactory = MetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length); ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType); @@ -159,12 +158,12 @@ public class RTreeResourceFactoryProvider implements IResourceFactoryProvider { return new LSMRTreeWithAntiMatterLocalResourceFactory(storageManager, typeTraits, rtreeCmpFactories, filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, - mergePolicyProperties, durable, valueProviderFactories, rTreePolicyType, linearizeCmpFactory, + mergePolicyProperties, true, valueProviderFactories, rTreePolicyType, linearizeCmpFactory, rtreeFields, isPointMBR, btreeCompFactories); } else { return new ExternalRTreeLocalResourceFactory(storageManager, typeTraits, rtreeCmpFactories, filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory, - metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, + metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, true, btreeCompFactories, valueProviderFactories, rTreePolicyType, linearizeCmpFactory, rtreeFields, new int[] { numNestedSecondaryKeyFields }, isPointMBR, mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java index 994370c..2a4a952 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java @@ -56,7 +56,6 @@ import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDese import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDiskComponentScanOperatorDescriptor; /** @@ -276,9 +275,8 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon protected IOperatorDescriptor createPrimaryIndexScanDiskComponentsOp(JobSpecification spec, MetadataProvider metadataProvider, RecordDescriptor outRecDesc, TxnId txnId) throws AlgebricksException { ITransactionSubsystemProvider txnSubsystemProvider = TransactionSubsystemProvider.INSTANCE; - boolean temp = dataset.getDatasetDetails().isTemp(); - ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE - : new PrimaryIndexInstantSearchOperationCallbackFactory(txnId, dataset.getDatasetId(), + ISearchOperationCallbackFactory searchCallbackFactory = + new PrimaryIndexInstantSearchOperationCallbackFactory(txnId, dataset.getDatasetId(), dataset.getPrimaryBloomFilterFields(), txnSubsystemProvider, IRecoveryManager.ResourceType.LSM_BTREE); IndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory( http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java index b63ea16..185cf8c 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java @@ -59,7 +59,7 @@ public abstract class SecondaryTreeIndexOperationsHelper extends SecondaryIndexO IResourceFactory resourceFactory = dataset.getResourceFactory(metadataProvider, index, itemType, metaType, mergePolicyFactory, mergePolicyProperties); IIndexBuilderFactory indexBuilderFactory = new IndexBuilderFactory(storageComponentProvider.getStorageManager(), - secondaryFileSplitProvider, resourceFactory, !dataset.isTemp()); + secondaryFileSplitProvider, resourceFactory, true); IndexCreateOperatorDescriptor secondaryIndexCreateOp = new IndexCreateOperatorDescriptor(spec, indexBuilderFactory); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java index 98cfc57..e5f5c19 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java @@ -83,9 +83,7 @@ public class SplitsAndConstraintsUtil { for (int k = 0; k < numPartitions; k++) { // format: 'storage dir name'/partition_#/dataverse/dataset_idx_index File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName, - nodePartitions[k].getPartitionId()) - + (dataset.isTemp() ? (File.separator + StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER) : "") - + File.separator + relPathFile); + nodePartitions[k].getPartitionId()) + File.separator + relPathFile); splits.add(StoragePathUtil.getFileSplitForClusterPartition(nodePartitions[k], f.getPath())); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java index 0198d74..b87ef2a 100644 --- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java +++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java @@ -49,7 +49,7 @@ public class DatasetTupleTranslatorTest { Collections.singletonList(Collections.singletonList("row_id")), Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), - Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false); + Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList()); Dataset dataset = new Dataset("test", "log", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix", http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java index 32f65e1..ab4229c 100644 --- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java +++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java @@ -60,7 +60,7 @@ public class IndexTupleTranslatorTest { Collections.singletonList(Collections.singletonList("row_id")), Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), - Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false); + Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList()); Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix", http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java deleted file mode 100644 index c609fc6..0000000 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetIndexModificationOperationCallback.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.transaction.management.opcallbacks; - -import org.apache.asterix.common.transactions.DatasetId; -import org.apache.asterix.common.transactions.ILockManager; -import org.apache.asterix.common.transactions.ITransactionContext; -import org.apache.asterix.common.transactions.ITransactionSubsystem; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; - -/** - * This class is the operation callback for temporary datasets. - * A temporary dataset does not require any lock and does not generate any write-ahead update and commit log - * but generates flush log and job commit log. - * The "before" and "found" method in this callback is empty so that no locking is requested for accessing a temporary - * dataset and no write-ahead log is written for update operations. - */ -public class TempDatasetIndexModificationOperationCallback extends AbstractIndexModificationOperationCallback { - - public TempDatasetIndexModificationOperationCallback(DatasetId datasetId, int[] primaryKeyFields, - ITransactionContext txnCtx, ILockManager lockManager, ITransactionSubsystem txnSubsystem, long resourceId, - int resourcePartition, byte resourceType, Operation indexOp) { - super(datasetId, primaryKeyFields, txnCtx, lockManager, txnSubsystem, resourceId, resourcePartition, - resourceType, indexOp); - } - - @Override - public void before(ITupleReference tuple) throws HyracksDataException { - - } - - @Override - public void found(ITupleReference before, ITupleReference after) throws HyracksDataException { - - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java deleted file mode 100644 index 735d7ea..0000000 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetPrimaryIndexModificationOperationCallbackFactory.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.transaction.management.opcallbacks; - -import org.apache.asterix.common.api.IJobEventListenerFactory; -import org.apache.asterix.common.context.ITransactionSubsystemProvider; -import org.apache.asterix.common.dataflow.DatasetLocalResource; -import org.apache.asterix.common.exceptions.ACIDException; -import org.apache.asterix.common.transactions.AbstractOperationCallbackFactory; -import org.apache.asterix.common.transactions.DatasetId; -import org.apache.asterix.common.transactions.ITransactionContext; -import org.apache.asterix.common.transactions.ITransactionSubsystem; -import org.apache.asterix.common.transactions.TxnId; -import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback.Operation; -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.api.dataflow.IOperatorNodePushable; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.job.IJobletEventListenerFactory; -import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; -import org.apache.hyracks.storage.common.IModificationOperationCallback; -import org.apache.hyracks.storage.common.IResourceLifecycleManager; -import org.apache.hyracks.storage.common.LocalResource; - -public class TempDatasetPrimaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory - implements IModificationOperationCallbackFactory { - - private static final long serialVersionUID = 1L; - private final Operation indexOp; - - public TempDatasetPrimaryIndexModificationOperationCallbackFactory(TxnId txnId, int datasetId, - int[] primaryKeyFields, ITransactionSubsystemProvider txnSubsystemProvider, Operation indexOp, - byte resourceType) { - super(txnId, datasetId, primaryKeyFields, txnSubsystemProvider, resourceType); - this.indexOp = indexOp; - } - - @Override - public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, - IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { - ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); - IResourceLifecycleManager indexLifeCycleManager = - txnSubsystem.getAsterixAppRuntimeContextProvider().getDatasetLifecycleManager(); - ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); - if (index == null) { - throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); - } - - try { - IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); - ITransactionContext txnCtx = txnSubsystem.getTransactionManager() - .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(txnId)); - DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); - IModificationOperationCallback modCallback = new TempDatasetIndexModificationOperationCallback( - new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, - resource.getId(), aResource.getPartition(), resourceType, indexOp); - txnCtx.register(resource.getId(), index, modCallback, true); - return modCallback; - } catch (ACIDException e) { - throw new HyracksDataException(e); - } - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java deleted file mode 100644 index b744606..0000000 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/opcallbacks/TempDatasetSecondaryIndexModificationOperationCallbackFactory.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.transaction.management.opcallbacks; - -import org.apache.asterix.common.api.IJobEventListenerFactory; -import org.apache.asterix.common.context.ITransactionSubsystemProvider; -import org.apache.asterix.common.dataflow.DatasetLocalResource; -import org.apache.asterix.common.exceptions.ACIDException; -import org.apache.asterix.common.transactions.AbstractOperationCallbackFactory; -import org.apache.asterix.common.transactions.DatasetId; -import org.apache.asterix.common.transactions.ITransactionContext; -import org.apache.asterix.common.transactions.ITransactionSubsystem; -import org.apache.asterix.common.transactions.TxnId; -import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback.Operation; -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.api.dataflow.IOperatorNodePushable; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.job.IJobletEventListenerFactory; -import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; -import org.apache.hyracks.storage.common.IIndex; -import org.apache.hyracks.storage.common.IModificationOperationCallback; -import org.apache.hyracks.storage.common.IResourceLifecycleManager; -import org.apache.hyracks.storage.common.LocalResource; - -public class TempDatasetSecondaryIndexModificationOperationCallbackFactory extends AbstractOperationCallbackFactory - implements IModificationOperationCallbackFactory { - - private static final long serialVersionUID = 1L; - private final Operation indexOp; - - public TempDatasetSecondaryIndexModificationOperationCallbackFactory(TxnId txnId, int datasetId, - int[] primaryKeyFields, ITransactionSubsystemProvider txnSubsystemProvider, Operation indexOp, - byte resourceType) { - super(txnId, datasetId, primaryKeyFields, txnSubsystemProvider, resourceType); - this.indexOp = indexOp; - } - - @Override - public IModificationOperationCallback createModificationOperationCallback(LocalResource resource, - IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException { - DatasetLocalResource aResource = (DatasetLocalResource) resource.getResource(); - ITransactionSubsystem txnSubsystem = txnSubsystemProvider.getTransactionSubsystem(ctx); - IResourceLifecycleManager<IIndex> indexLifeCycleManager = - txnSubsystem.getAsterixAppRuntimeContextProvider().getDatasetLifecycleManager(); - ILSMIndex index = (ILSMIndex) indexLifeCycleManager.get(resource.getPath()); - if (index == null) { - throw new HyracksDataException("Index(id:" + resource.getId() + ") is not registered."); - } - - try { - IJobletEventListenerFactory fact = ctx.getJobletContext().getJobletEventListenerFactory(); - ITransactionContext txnCtx = txnSubsystem.getTransactionManager() - .getTransactionContext(((IJobEventListenerFactory) fact).getTxnId(txnId)); - IModificationOperationCallback modCallback = new TempDatasetIndexModificationOperationCallback( - new DatasetId(datasetId), primaryKeyFields, txnCtx, txnSubsystem.getLockManager(), txnSubsystem, - resource.getId(), aResource.getPartition(), resourceType, indexOp); - txnCtx.register(resource.getId(), index, modCallback, false); - return modCallback; - } catch (ACIDException e) { - throw new HyracksDataException(e); - } - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2c04ae07/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java index 5666b48..db3647e 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java @@ -380,25 +380,14 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito private void createReplicationJob(ReplicationOperation operation, FileReference fileRef) throws HyracksDataException { - /** - * Durable resources path format: - * /partition/dataverse/idx/fileName - * Temporary resources path format: - * /partition/TEMP_DATASETS_STORAGE_FOLDER/dataverse/idx/fileName - */ - String[] fileNameTokens = fileRef.getAbsolutePath().split(File.separator); - String partitionDir = fileNameTokens[fileNameTokens.length - 4]; - //exclude temporary datasets resources - if (!partitionDir.equals(StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER)) { - filesToBeReplicated.clear(); - filesToBeReplicated.add(fileRef.getAbsolutePath()); - ReplicationJob job = new ReplicationJob(ReplicationJobType.METADATA, operation, - ReplicationExecutionType.SYNC, filesToBeReplicated); - try { - replicationManager.submitJob(job); - } catch (IOException e) { - throw new HyracksDataException(e); - } + filesToBeReplicated.clear(); + filesToBeReplicated.add(fileRef.getAbsolutePath()); + ReplicationJob job = new ReplicationJob(ReplicationJobType.METADATA, operation, ReplicationExecutionType.SYNC, + filesToBeReplicated); + try { + replicationManager.submitJob(job); + } catch (IOException e) { + throw new HyracksDataException(e); } }