abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/766
Change subject: ASTERIXDB-1378 Fix NPE on Feed Connect After Restart
......................................................................
ASTERIXDB-1378 Fix NPE on Feed Connect After Restart
This issue was caused by the way type traits are computed for each
index. since we have duplicate code to create the index dataflow
helper, we had to fix the way type traits are created in all of
these different places. We propably need to do further refactoring
to have common code for creating index dataflow helper instances.
Change-Id: If8f8696d252868a8cce0afdbaeda0dd046f99186
---
M asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
M asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
M
asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
M
asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionTest.java
M
asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M
asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
M
asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
8 files changed, 52 insertions(+), 23 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/66/766/1
diff --git
a/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
b/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
index 06d2b71..25893c9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/DatasetOperations.java
@@ -69,7 +69,7 @@
public static JobSpecification
createDropDatasetJobSpec(CompiledDatasetDropStatement datasetDropStmt,
AqlMetadataProvider metadataProvider)
- throws AlgebricksException, HyracksDataException,
RemoteException, ACIDException, AsterixException {
+ throws AlgebricksException, HyracksDataException, RemoteException,
ACIDException, AsterixException {
String dataverseName = null;
if (datasetDropStmt.getDataverseName() != null) {
@@ -228,7 +228,13 @@
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IBinaryComparatorFactory[] comparatorFactories =
DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
itemType, format.getBinaryComparatorFactoryProvider());
- ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+ // get meta item type
+ ARecordType metaItemType = null;
+ if (dataset.hasMetaPart()) {
+ metaItemType = (ARecordType)
metadata.findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
+ }
+ ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType, metaItemType);
int[] blooFilterKeyFields =
DatasetUtils.createBloomFilterKeyFields(dataset);
ITypeTraits[] filterTypeTraits =
DatasetUtils.computeFilterTypeTraits(dataset, itemType);
diff --git
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index 8d020e7..6e24f04 100644
---
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -48,9 +48,10 @@
File.separator);
protected static final String TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
-
protected static AsterixTransactionProperties txnProperties;
- private final static TestExecutor testExecutor = new TestExecutor();
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final boolean cleanupOnStart = true;
+ private static final boolean cleanupOnStop = true;
@BeforeClass
public static void setUp() throws Exception {
@@ -60,7 +61,7 @@
// remove library directory
TestLibrarian.removeLibraryDir();
testExecutor.setLibrarian(new TestLibrarian());
- ExecutionTestUtil.setUp();
+ ExecutionTestUtil.setUp(cleanupOnStart);
} catch (Throwable th) {
th.printStackTrace();
throw th;
@@ -71,7 +72,7 @@
public static void tearDown() throws Exception {
// remove library directory
TestLibrarian.removeLibraryDir();
- ExecutionTestUtil.tearDown();
+ ExecutionTestUtil.tearDown(cleanupOnStop);
}
@Parameters(name = "ExecutionTest {index}: {0}")
@@ -94,6 +95,7 @@
}
protected TestCaseContext tcCtx;
+ int i = 0;
public ExecutionTest(TestCaseContext tcCtx) {
this.tcCtx = tcCtx;
diff --git
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
index 5e76ecb..d919c92 100644
---
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
+++
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
@@ -43,7 +43,7 @@
protected static TestGroup FailedGroup;
- public static void setUp() throws Exception {
+ public static void setUp(boolean cleanup) throws Exception {
System.out.println("Starting setup");
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting setup");
@@ -53,7 +53,7 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("initializing pseudo cluster");
}
- AsterixHyracksIntegrationUtil.init(true);
+ AsterixHyracksIntegrationUtil.init(cleanup);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("initializing HDFS");
@@ -81,9 +81,9 @@
}
}
- public static void tearDown() throws Exception {
+ public static void tearDown(boolean cleanup) throws Exception {
// validateBufferCacheState(); <-- Commented out until bug is fixed -->
- AsterixHyracksIntegrationUtil.deinit(true);
+ AsterixHyracksIntegrationUtil.deinit(cleanup);
File outdir = new File(PATH_ACTUAL);
File[] files = outdir.listFiles();
if (files == null || files.length == 0) {
diff --git
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionTest.java
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionTest.java
index cbb14c5..b827a0d 100644
---
a/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionTest.java
+++
b/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionTest.java
@@ -51,7 +51,9 @@
protected static final String TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
protected static AsterixTransactionProperties txnProperties;
- private final static TestExecutor testExecutor = new TestExecutor();
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final boolean cleanupOnStart = true;
+ private static final boolean cleanupOnStop = true;
protected static TestGroup FailedGroup;
@@ -59,12 +61,12 @@
public static void setUp() throws Exception {
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
- ExecutionTestUtil.setUp();
+ ExecutionTestUtil.setUp(cleanupOnStart);
}
@AfterClass
public static void tearDown() throws Exception {
- ExecutionTestUtil.tearDown();
+ ExecutionTestUtil.tearDown(cleanupOnStop);
AsterixHyracksIntegrationUtil.removeTestStorageFiles();
}
diff --git
a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 6401234..4533c5b 100644
---
a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++
b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -85,6 +85,8 @@
}
tupleForwarder.addTuple(tb);
}
+ } catch (InterruptedException e) {
+ LOGGER.warn("Feed has been interrupted. Closing the feed");
} catch (Exception e) {
failed = true;
tupleForwarder.flush();
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index a629cdf..a04fee7 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -701,7 +701,14 @@
bloomFilterKeyFields[i] = i;
}
- typeTraits = DatasetUtils.computeTupleTypeTraits(dataset,
itemType);
+ // get meta item type
+ ARecordType metaItemType = null;
+ if (dataset.hasMetaPart()) {
+ metaItemType = (ARecordType)
findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
+ }
+
+ typeTraits = DatasetUtils.computeTupleTypeTraits(dataset,
itemType, metaItemType);
comparatorFactories =
DatasetUtils.computeKeysBinaryComparatorFactories(dataset, itemType,
context.getBinaryComparatorFactoryProvider());
@@ -1089,7 +1096,7 @@
String itemTypeName = dataset.getItemTypeName();
ARecordType itemType = (ARecordType) MetadataManager.INSTANCE
.getDatatype(mdTxnCtx, dataset.getItemTypeDataverseName(),
itemTypeName).getDatatype();
- ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+ ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType, null);
IBinaryComparatorFactory[] comparatorFactories =
DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
itemType, context.getBinaryComparatorFactoryProvider());
@@ -1179,7 +1186,14 @@
ARecordType itemType = (ARecordType) MetadataManager.INSTANCE
.getDatatype(mdTxnCtx, dataset.getItemTypeDataverseName(),
itemTypeName).getDatatype();
- ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+ // get meta item type
+ ARecordType metaItemType = null;
+ if (dataset.hasMetaPart()) {
+ metaItemType = (ARecordType)
findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
+ }
+
+ ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType, metaItemType);
IAsterixApplicationContextInfo appContext =
(IAsterixApplicationContextInfo) context.getAppContext();
IBinaryComparatorFactory[] comparatorFactories =
DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -2348,7 +2362,14 @@
ARecordType itemType = (ARecordType) MetadataManager.INSTANCE
.getDatatype(mdTxnCtx,
dataSource.getId().getDataverseName(), itemTypeName).getDatatype();
- ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+ // get meta item type
+ ARecordType metaItemType = null;
+ if (dataset.hasMetaPart()) {
+ metaItemType = (ARecordType)
findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
+ }
+
+ ITypeTraits[] typeTraits =
DatasetUtils.computeTupleTypeTraits(dataset, itemType, metaItemType);
IAsterixApplicationContextInfo appContext =
(IAsterixApplicationContextInfo) context.getAppContext();
IBinaryComparatorFactory[] comparatorFactories =
DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
diff --git
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
index 581d01c..af04592 100644
---
a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
+++
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
@@ -105,11 +105,6 @@
return bhffs;
}
- public static ITypeTraits[] computeTupleTypeTraits(Dataset dataset,
ARecordType itemType)
- throws AlgebricksException {
- return computeTupleTypeTraits(dataset, itemType, null);
- }
-
public static ITypeTraits[] computeTupleTypeTraits(Dataset dataset,
ARecordType itemType, ARecordType metaItemType)
throws AlgebricksException {
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
@@ -156,7 +151,7 @@
public static IBinaryComparatorFactory[]
computeFilterBinaryComparatorFactories(Dataset dataset,
ARecordType itemType, IBinaryComparatorFactoryProvider
comparatorFactoryProvider)
- throws AlgebricksException {
+ throws AlgebricksException {
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
return null;
}
diff --git
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
index 83c6e34..7bc2280 100644
---
a/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
+++
b/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
@@ -194,6 +194,7 @@
accessor.reset(buffer);
LSMTreeIndexAccessor lsmAccessor = (LSMTreeIndexAccessor)
indexAccessor;
int tupleCount = accessor.getTupleCount();
+ accessor.prettyPrint();
int i = 0;
try {
while (i < tupleCount) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/766
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If8f8696d252868a8cce0afdbaeda0dd046f99186
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
