http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java index 126ddf5..2f22ade 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java @@ -28,7 +28,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; -import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.index.IndexDocumentBuilder; @@ -39,58 +38,31 @@ import org.apache.vxquery.xmlparser.IParser; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; import org.apache.vxquery.xmlparser.XMLParser; -import javax.xml.bind.JAXBException; import java.io.DataInputStream; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Paths; -import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; -import java.util.Arrays; import java.util.concurrent.ConcurrentHashMap; public class IndexConstructorUtil { - static boolean isMetaFilePresent = false; - static MetaFileUtil metaFileUtil; - static ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>(); + boolean isMetaFilePresent = false; + MetaFileUtil metaFileUtil; + ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>(); - public static void evaluate(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp, - ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, + public void evaluate(String collectioFolder, String indexFolder, IPointable result, UTF8StringPointable + stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, - boolean isElementPath, String nodeId) throws SystemException, JAXBException { - String collectionFolder; - String indexFolder; - TaggedValuePointable collectionTVP = args[0]; - TaggedValuePointable indexTVP = args[1]; - - if (collectionTVP.getTag() != ValueTag.XS_STRING_TAG || indexTVP.getTag() != ValueTag.XS_STRING_TAG) { - throw new SystemException(ErrorCode.FORG0006); - } + boolean isElementPath, String nodeId) throws SystemException { - try { - // Get the list of files. - collectionTVP.getValue(stringp); - bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(), - stringp.getLength() + stringp.getStartOffset())), 0); - collectionFolder = di.readUTF(); - - // Get the index folder - indexTVP.getValue(stringp); - bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(), - stringp.getLength() + stringp.getStartOffset())), 0); - indexFolder = di.readUTF(); - - metaFileUtil = MetaFileUtil.create(indexFolder); + metaFileUtil = new MetaFileUtil(indexFolder); +// metaFileUtil = .create(indexFolder); isMetaFilePresent = metaFileUtil.isMetaFilePresent(); - metaFileUtil.setCollectionForIndex(indexFolder, collectionFolder); + metaFileUtil.setCollection(collectioFolder); - } catch (IOException e) { - throw new SystemException(ErrorCode.SYSE0001, e); - } - File collectionDirectory = new File(collectionFolder); + File collectionDirectory = new File(collectioFolder); if (!collectionDirectory.exists()) { - throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist."); + throw new RuntimeException("The collection directory (" + collectioFolder + ") does not exist."); } try { @@ -132,7 +104,7 @@ public class IndexConstructorUtil { /*This function goes recursively one file at a time. First it turns the file into an ABVS document node, then * it indexes that document node. */ - public static void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath, + public void indexXmlFiles(File collectionDirectory, IndexWriter writer, boolean isElementPath, TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, SequenceBuilder sb, ByteBufferInputStream bbis, DataInputStream di, String nodeId) throws SystemException, IOException { @@ -153,11 +125,7 @@ public class IndexConstructorUtil { xmlMetadata.setPath(file.getCanonicalPath()); xmlMetadata.setFileName(file.getName()); xmlMetadata.setLastModified(sdf.format(file.lastModified())); - try { - xmlMetadata.setMd5(metaFileUtil.generateMD5(file)); - } catch (NoSuchAlgorithmException e) { - throw new SystemException(ErrorCode.SYSE0001, e); - } + xmlMetadata.setMd5(metaFileUtil.generateMD5(file)); metadataMap.put(file.getCanonicalPath(), xmlMetadata); } @@ -168,14 +136,14 @@ public class IndexConstructorUtil { } } - public static boolean readableXmlFile(String path) { + public boolean readableXmlFile(String path) { return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz")); } /** * Separated from create index method so that it could be used as a helper function in IndexUpdater */ - public static IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep, + public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis, DataInputStream di, String nodeId) throws IOException {
http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java deleted file mode 100644 index e713b20..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexDeleteEvaluatorFactory.java +++ /dev/null @@ -1,83 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index; - -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; -import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; -import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; -import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; -import org.apache.vxquery.datamodel.values.XDMConstants; -import org.apache.vxquery.exceptions.ErrorCode; -import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; -import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; -import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater; -import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; -import org.apache.vxquery.xmlparser.TreeNodeIdProvider; - -import javax.xml.bind.JAXBException; -import java.io.DataInputStream; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; - -/** - * Delete the index of a given index directory - */ -public class IndexDeleteEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { - public IndexDeleteEvaluatorFactory(IScalarEvaluatorFactory[] args) { - super(args); - } - - @Override - protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) - throws AlgebricksException { - final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); - final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); - final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); - final ByteBufferInputStream bbis = new ByteBufferInputStream(); - final DataInputStream di = new DataInputStream(bbis); - final SequenceBuilder sb = new SequenceBuilder(); - final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage(); - final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); - final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - - return new AbstractTaggedValueArgumentScalarEvaluator(args) { - - @Override - protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { - IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider, - abvsFileNode, nodep, nodeId); - try { - updater.setup(); - updater.deleteAllIndexes(); - XDMConstants.setTrue(result); - } catch (IOException | NoSuchAlgorithmException | JAXBException e) { - throw new SystemException(ErrorCode.SYSE0001, e); - } - - } - - }; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java deleted file mode 100644 index 3fbc279..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexUpdaterEvaluatorFactory.java +++ /dev/null @@ -1,84 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index; - -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; -import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; -import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; -import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; -import org.apache.vxquery.datamodel.values.XDMConstants; -import org.apache.vxquery.exceptions.ErrorCode; -import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; -import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; -import org.apache.vxquery.runtime.functions.index.updateIndex.IndexUpdater; -import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; -import org.apache.vxquery.xmlparser.TreeNodeIdProvider; - -import javax.xml.bind.JAXBException; -import java.io.DataInputStream; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; - -/** - * Update the index of collection - */ -public class IndexUpdaterEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { - public IndexUpdaterEvaluatorFactory(IScalarEvaluatorFactory[] args) { - super(args); - } - - @Override - protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) - throws AlgebricksException { - final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); - final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable(); - final TaggedValuePointable nodep = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); - final ByteBufferInputStream bbis = new ByteBufferInputStream(); - final DataInputStream di = new DataInputStream(bbis); - final SequenceBuilder sb = new SequenceBuilder(); - final ArrayBackedValueStorage abvsFileNode = new ArrayBackedValueStorage(); - final int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); - final String nodeId = ctx.getJobletContext().getApplicationContext().getNodeId(); - final ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); - - return new AbstractTaggedValueArgumentScalarEvaluator(args) { - - @Override - protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { - IndexUpdater updater = new IndexUpdater(args, result, stringp, bbis, di, sb, abvs, nodeIdProvider, - abvsFileNode, nodep, nodeId); - try { - updater.setup(); - updater.updateIndex(); - updater.updateMetadataFile(); - updater.exit(); - XDMConstants.setTrue(result); - } catch (IOException | NoSuchAlgorithmException | JAXBException e) { - throw new SystemException(ErrorCode.SYSE0001, e); - } - } - - }; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java new file mode 100644 index 0000000..9b72a34 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/ShowIndexScalarEvaluatorFactory.java @@ -0,0 +1,65 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index; + +import java.io.IOException; + +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; +import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluator; +import org.apache.vxquery.runtime.functions.base.AbstractTaggedValueArgumentScalarEvaluatorFactory; +import org.apache.vxquery.runtime.functions.index.indexCentralizer.IndexCentralizerUtil; + +public class ShowIndexScalarEvaluatorFactory extends AbstractTaggedValueArgumentScalarEvaluatorFactory { + public ShowIndexScalarEvaluatorFactory(IScalarEvaluatorFactory[] args) { + super(args); + } + + @Override + protected IScalarEvaluator createEvaluator(IHyracksTaskContext ctx, IScalarEvaluator[] args) + throws AlgebricksException { + final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); + final SequenceBuilder sb = new SequenceBuilder(); + + return new AbstractTaggedValueArgumentScalarEvaluator(args) { + @Override + protected void evaluate(TaggedValuePointable[] args, IPointable result) throws SystemException { + try { + abvs.reset(); + sb.reset(abvs); + IndexCentralizerUtil indexCentralizerUtil = new IndexCentralizerUtil( + ctx.getIOManager().getIODevices().get(0).getPath()); + indexCentralizerUtil.readIndexDirectory(); + indexCentralizerUtil.getAllCollections(sb); + sb.finish(); + result.set(abvs); + } catch (IOException e) { + throw new SystemException(ErrorCode.SYSE0001, e); + } + + } + }; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java new file mode 100644 index 0000000..8750849 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/VXQueryIndexReader.java @@ -0,0 +1,284 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index; + +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.FSDirectory; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.index.IndexAttributes; +import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; +import org.apache.vxquery.xmlparser.SAXContentHandler; +import org.apache.vxquery.xmlparser.TreeNodeIdProvider; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +public class VXQueryIndexReader { + + private ArrayBackedValueStorage nodeAbvs = new ArrayBackedValueStorage(); + + private int indexPlace; + private int indexLength; + private String elementPath; + private String indexName; + + private ByteBufferInputStream bbis = new ByteBufferInputStream(); + private DataInputStream di = new DataInputStream(bbis); + + private IndexReader reader; + private IndexSearcher searcher; + private QueryParser parser; + private ScoreDoc[] hits; + private SAXContentHandler handler; + private Query query; + private Document doc; + private List<IndexableField> fields; + private IHyracksTaskContext ctx; + + public VXQueryIndexReader(IHyracksTaskContext context, String indexPath, String elementPath) { + this.ctx = context; + this.indexName = indexPath; + this.elementPath = elementPath; + } + + public boolean step(IPointable result) throws AlgebricksException { + /*each step will create a tuple for a single xml file + * This is done using the parse function + * checkoverflow is used throughout. This is because memory might not be + * able to hold all of the results at once, so we return 1 million at + * a time and check when we need to get more + */ + if (indexPlace < indexLength) { + nodeAbvs.reset(); + try { + //TODO: now we get back the entire document + doc = searcher.doc(hits[indexPlace].doc); + fields = doc.getFields(); + parse(nodeAbvs); + } catch (IOException e) { + throw new AlgebricksException(e); + } + indexPlace += 1; + result.set(nodeAbvs.getByteArray(), nodeAbvs.getStartOffset(), nodeAbvs.getLength()); + return true; + } + return false; + } + + public void init() throws SystemException { + + int partition = ctx.getTaskAttemptId().getTaskId().getPartition(); + ITreeNodeIdProvider nodeIdProvider = new TreeNodeIdProvider((short) partition); + handler = new SAXContentHandler(false, nodeIdProvider, true); + + nodeAbvs.reset(); + indexPlace = 0; + + try { + indexPlace = 0; + + //Create the index reader. + reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexName))); + } catch (IOException e) { + throw new SystemException(ErrorCode.SYSE0001, e); + } + + searcher = new IndexSearcher(reader); + Analyzer analyzer = new CaseSensitiveAnalyzer(); + + parser = new CaseSensitiveQueryParser("item", analyzer); + + String queryString = elementPath.replaceAll("/", "."); + queryString = "item:" + queryString + "*"; + + int lastslash = elementPath.lastIndexOf("/"); + elementPath = elementPath.substring(0, lastslash) + ":" + elementPath.substring(lastslash + 1); + elementPath = elementPath.replaceAll("/", ".") + ".element"; + + TopDocs results = null; + try { + query = parser.parse(queryString); + + //TODO: Right now it only returns 1000000 results + results = searcher.search(query, 1000000); + + } catch (Exception e) { + throw new SystemException(null); + } + + hits = results.scoreDocs; + System.out.println("found: " + results.totalHits); + indexPlace = 0; + indexLength = hits.length; + + } + + public void parse(ArrayBackedValueStorage abvsFileNode) throws IOException { + try { + handler.startDocument(); + + for (int i = 0; i < fields.size(); i++) { + String fieldValue = fields.get(i).stringValue(); + if (fieldValue.equals(elementPath)) { + buildElement(abvsFileNode, i); + } + } + + handler.endDocument(); + handler.writeDocument(abvsFileNode); + } catch (Exception e) { + throw new IOException(e); + } + } + + private int buildElement(ArrayBackedValueStorage abvsFileNode, int fieldNum) throws SAXException { + int whereIFinish = fieldNum; + IndexableField field = fields.get(fieldNum); + String contents = field.stringValue(); + String uri = ""; + + int firstColon = contents.indexOf(':'); + int lastDot = contents.lastIndexOf('.'); + String type = contents.substring(lastDot + 1); + String lastBit = contents.substring(firstColon + 1, lastDot); + + if (type.equals("textnode")) { + char[] charContents = lastBit.toCharArray(); + handler.characters(charContents, 0, charContents.length); + + } + if (type.equals("element")) { + List<String> names = new ArrayList<String>(); + List<String> values = new ArrayList<String>(); + List<String> uris = new ArrayList<String>(); + List<String> localNames = new ArrayList<String>(); + List<String> types = new ArrayList<String>(); + List<String> qNames = new ArrayList<String>(); + whereIFinish = findAttributeChildren(whereIFinish, names, values, uris, localNames, types, qNames); + Attributes atts = new IndexAttributes(names, values, uris, localNames, types, qNames); + + handler.startElement(uri, lastBit, lastBit, atts); + + boolean noMoreChildren = false; + + while (whereIFinish + 1 < fields.size() && !noMoreChildren) { + if (isChild(fields.get(whereIFinish + 1), field)) { + whereIFinish = buildElement(abvsFileNode, whereIFinish + 1); + } else { + noMoreChildren = true; + } + } + + handler.endElement(uri, lastBit, lastBit); + + } + return whereIFinish; + } + + /*This function creates the attribute children for an element node + * + */ + int findAttributeChildren(int fieldnum, List<String> n, List<String> v, List<String> u, List<String> l, + List<String> t, List<String> q) { + int nextindex = fieldnum + 1; + boolean foundattributes = false; + if (nextindex < fields.size()) { + IndexableField nextguy; + + while (nextindex < fields.size()) { + nextguy = fields.get(nextindex); + String contents = nextguy.stringValue(); + int firstcolon = contents.indexOf(':'); + int lastdot = contents.lastIndexOf('.'); + String lastbit = contents.substring(firstcolon + 1, lastdot); + + if (isDirectChildAttribute(nextguy, fields.get(fieldnum))) { + foundattributes = true; + n.add(lastbit); + IndexableField nextnextguy = fields.get(nextindex + 1); + contents = nextnextguy.stringValue(); + firstcolon = contents.indexOf(':'); + lastdot = contents.lastIndexOf('.'); + String nextlastbit = contents.substring(firstcolon + 1, lastdot); + v.add(nextlastbit); + u.add(lastbit); + l.add(lastbit); + t.add(lastbit); + q.add(lastbit); + } else { + break; + } + nextindex += 2; + } + } + if (foundattributes) { + return nextindex - 1; + + } else { + return fieldnum; + } + } + + boolean isChild(IndexableField child, IndexableField adult) { + String childId = child.stringValue(); + String adultId = adult.stringValue(); + + int lastDotChild = childId.lastIndexOf('.'); + int lastDotAdult = adultId.lastIndexOf('.'); + + String childPath = childId.substring(0, lastDotChild); + String adultPath = adultId.substring(0, lastDotAdult); + adultPath = adultPath.replaceFirst(":", "."); + + return (childPath.startsWith(adultPath + ":") || childPath.startsWith(adultPath + ".")); + } + + boolean isDirectChildAttribute(IndexableField child, IndexableField adult) { + String childId = child.stringValue(); + String adultId = adult.stringValue(); + + String childPath = childId.substring(0, childId.lastIndexOf('.')); + String adultPath = adultId.substring(0, adultId.lastIndexOf('.')); + adultPath = adultPath.replaceFirst(":", "."); + String[] childSegments = child.stringValue().split("\\."); + + String childType = childSegments[childSegments.length - 1]; + + return (childPath.startsWith(adultPath + ":") && childType.equals("attribute")); + } + +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java new file mode 100644 index 0000000..38dcc2a --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.index.indexCentralizer; + +import java.io.DataOutput; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.commons.io.FileUtils; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder; +import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; +import org.apache.vxquery.datamodel.values.ValueTag; + +/** + * Class for maintaining the centralized index information file. + * Index centralization procedure. + * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = <index_directory>. + * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections. + * There will be a single xml file, located in the directory specified in local.xml, which contains all information + * about the existing indexes. + * This class can be used to read, add, delete, modify the entries and write the file back to the disk. + */ +public class IndexCentralizerUtil { + + private final String FILE_NAME = "VXQuery-Index-Directory.xml"; + private final List<String> collections = new ArrayList<>(); + private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil"); + private File XML_FILE; + private String INDEX_LOCATION; + private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>(); + + public IndexCentralizerUtil(File index) { + this.INDEX_LOCATION = index.getPath(); + if (!index.exists()) { + try { + FileUtils.forceMkdir(index); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e); + } + } + XML_FILE = new File(index.getPath() + "/" + FILE_NAME); + } + + /** + * Get the index directory containing index of the given collection + * + * @param collection : Collection folder + * @return Index folder. + */ + public String getIndexForCollection(String collection) { + return indexCollectionMap.get(collection).getIndex(); + } + + /** + * Put the index location corresponding to given collection. + * Index location is created by using the last 100 characters of collection. + * + * @param collection : Collection directory + */ + public String putIndexForCollection(String collection) { + int length = collection.replaceAll("/", "").length(); + String index = collection.replaceAll("/", ""); + index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index); + IndexLocator il = new IndexLocator(); + il.setCollection(collection); + il.setIndex(index); + if (indexCollectionMap.get(collection) != null) { + return index; + } + indexCollectionMap.put(collection, il); + return index; + } + + /** + * Remove the entry for given collection directory. + * + * @param collection : Collection directory + */ + public void deleteEntryForCollection(String collection) { + indexCollectionMap.remove(collection); + } + + /** + * Prints all collections which have an index created. + * + * @throws IOException + */ + public void getAllCollections(SequenceBuilder sb) throws IOException { + for (String s : collections) { + StringValueBuilder svb = new StringValueBuilder(); + ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); + DataOutput output = abvs.getDataOutput(); + output.write(ValueTag.XS_STRING_TAG); + svb.write(s, output); + sb.addItem(abvs); + } + } + + /** + * Read the collection, index directory file and populate the HashMap. + */ + public void readIndexDirectory() { + if (this.XML_FILE.exists()) { + try { + JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class); + Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); + IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE); + + for (IndexLocator il : indexDirectory.getDirectory()) { + indexCollectionMap.put(il.getCollection(), il); + this.collections.add(il.getCollection()); + } + } catch (JAXBException e) { + LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e); + } + } + + } + + /** + * Write back the contents of the HashMap to the file. + */ + public void writeIndexDirectory() { + IndexDirectory id = new IndexDirectory(); + List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values()); + id.setDirectory(indexLocators); + try { + FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE); + JAXBContext context = JAXBContext.newInstance(IndexDirectory.class); + Marshaller jaxbMarshaller = context.createMarshaller(); + jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + jaxbMarshaller.marshal(id, fileOutputStream); + } catch (JAXBException | FileNotFoundException e) { + LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e); + } + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java new file mode 100644 index 0000000..54d9ad9 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.index.indexCentralizer; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@XmlRootElement(name = "indexes") +@XmlAccessorType(XmlAccessType.FIELD) +public class IndexDirectory implements Serializable{ + + @XmlElement(name = "index", type = IndexLocator.class) + private List<IndexLocator> directory = new ArrayList<>(); + + public List<IndexLocator> getDirectory() { + return directory; + } + + + public void setDirectory(List<IndexLocator> directory) { + this.directory = directory; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java new file mode 100644 index 0000000..1a33c8b --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.index.indexCentralizer; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "Entry") +public class IndexLocator implements Serializable{ + + @XmlAttribute + private String collection; + + @XmlAttribute + private String index; + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java index 9aebfb4..2a45747 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java @@ -22,5 +22,4 @@ package org.apache.vxquery.runtime.functions.index.updateIndex; public class Constants { public static String FIELD_PATH = "path"; public static String META_FILE_NAME = "vxquery_index.xml"; - public static String COLLECTION_ENTRY = "collection"; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java index 4588282..ba7cd88 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java @@ -29,7 +29,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; -import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.index.IndexDocumentBuilder; @@ -37,16 +36,12 @@ import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer; import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; -import javax.xml.bind.JAXBException; import java.io.DataInputStream; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Paths; -import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; -import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -57,9 +52,7 @@ import java.util.concurrent.ConcurrentHashMap; public class IndexUpdater { private MetaFileUtil metaFileUtil; private ConcurrentHashMap<String, XmlMetadata> metadataMap; - private TaggedValuePointable[] args; private IPointable result; - private UTF8StringPointable stringp; private ByteBufferInputStream bbis; private DataInputStream di; private SequenceBuilder sb; @@ -71,19 +64,17 @@ public class IndexUpdater { private IndexWriter indexWriter; private Set<String> pathsFromFileList; private String collectionFolder; - private XmlMetadata collectionMetadata; private String indexFolder; private Logger LOGGER = Logger.getLogger("Index Updater"); private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); + private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); - //TODO : Implement for paralleizing - public IndexUpdater(TaggedValuePointable[] args, IPointable result, UTF8StringPointable stringp, + public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) { - this.args = args; + this.indexFolder = indexFolder; this.result = result; - this.stringp = stringp; this.bbis = bbis; this.di = di; this.sb = sb; @@ -100,35 +91,17 @@ public class IndexUpdater { * * @throws SystemException * @throws IOException - * @throws NoSuchAlgorithmException */ - public void setup() throws SystemException, IOException, NoSuchAlgorithmException, JAXBException { + public void setup() throws SystemException, IOException { - TaggedValuePointable indexTVP = args[0]; + // Read the metadata file and load the metadata map into memory. + metaFileUtil = new MetaFileUtil(indexFolder); + metaFileUtil.readMetadataFile(); + metadataMap = metaFileUtil.getMetadata(); - if (indexTVP.getTag() != ValueTag.XS_STRING_TAG) { - throw new SystemException(ErrorCode.FORG0006); - } - - try { - // Get the index folder - indexTVP.getValue(stringp); - bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(), - stringp.getLength() + stringp.getStartOffset())), 0); - indexFolder = di.readUTF(); - - // Read the metadata file and load the metadata map into memory. - metaFileUtil = MetaFileUtil.create(indexFolder); - metaFileUtil.readMetadataFile(); - metadataMap = metaFileUtil.getMetadata(indexFolder); - - // Retrieve the collection folder path. - // Remove the entry for ease of the next steps. - collectionFolder = metaFileUtil.getCollection(indexFolder); - - } catch (IOException | ClassNotFoundException e) { - throw new SystemException(ErrorCode.SYSE0001, e); - } + // Retrieve the collection folder path. + // Remove the entry for ease of the next steps. + collectionFolder = metaFileUtil.getCollection(); abvs.reset(); sb.reset(abvs); @@ -142,9 +115,8 @@ public class IndexUpdater { * Wrapper for update index function. * * @throws IOException - * @throws NoSuchAlgorithmException */ - public void updateIndex() throws IOException, NoSuchAlgorithmException { + public void updateIndex() throws IOException { File collectionDirectory = new File(collectionFolder); if (!collectionDirectory.exists()) { throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist."); @@ -155,6 +127,7 @@ public class IndexUpdater { //Detect deleted files and execute the delete index process. deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList); + updateMetadataFile(); } /** @@ -176,7 +149,7 @@ public class IndexUpdater { * * @throws IOException */ - public synchronized void updateMetadataFile() throws IOException, JAXBException { + public synchronized void updateMetadataFile() throws IOException { //Write the updated metadata to the file. metaFileUtil.updateMetadataMap(metadataMap, indexFolder); metaFileUtil.writeMetadataToFile(); @@ -188,14 +161,14 @@ public class IndexUpdater { * * @param collection : Collection folder path */ - private void updateIndex(File collection) throws IOException, NoSuchAlgorithmException { + private void updateIndex(File collection) throws IOException { File[] list = collection.listFiles(); assert list != null; for (File file : list) { pathsFromFileList.add(file.getCanonicalPath()); - if (IndexConstructorUtil.readableXmlFile(file.getCanonicalPath())) { + if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) { XmlMetadata data = metadataMap.get(file.getCanonicalPath()); String md5 = metaFileUtil.generateMD5(file); @@ -212,7 +185,7 @@ public class IndexUpdater { //Update index corresponding to the xml file. indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath())); - indexDocumentBuilder = IndexConstructorUtil + indexDocumentBuilder = indexConstructorUtil .getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, nodeId); indexDocumentBuilder.printStart(); @@ -230,7 +203,7 @@ public class IndexUpdater { // In this case, the xml file has not added to the index. (It is a newly added file) // Therefore generate a new index for this file and add it to the existing index. - indexDocumentBuilder = IndexConstructorUtil + indexDocumentBuilder = indexConstructorUtil .getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, nodeId); indexDocumentBuilder.printStart(); @@ -254,15 +227,14 @@ public class IndexUpdater { * @param metadata : Existing metadata object * @return : XML metadata object with updated fields. * @throws IOException - * @throws NoSuchAlgorithmException */ - private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException, NoSuchAlgorithmException { + private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException { - if (metadata == null) + if (metadata == null) { metadata = new XmlMetadata(); - - metadata.setPath(file.getCanonicalPath()); + } metadata.setFileName(file.getName()); + metadata.setPath(file.getCanonicalPath()); metadata.setMd5(metaFileUtil.generateMD5(file)); metadata.setLastModified(sdf.format(file.lastModified())); return metadata; http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java index 53b02df..dd099b5 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java @@ -19,7 +19,11 @@ package org.apache.vxquery.runtime.functions.index.updateIndex; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import javax.xml.bind.*; +import javax.xml.bind.DatatypeConverter; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -40,17 +44,14 @@ public class MetaFileUtil { private File metaFile; private Logger LOGGER = Logger.getLogger("MetadataFileUtil"); - private Map<String, ConcurrentHashMap<String, XmlMetadata>> indexes = new ConcurrentHashMap<>(); - private Map<String, String> indexToCollection = new ConcurrentHashMap<>(); + private String index; + private String collection; + private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>(); - private MetaFileUtil(String indexFolder) { + public MetaFileUtil(String indexFolder) { this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME); } - public static MetaFileUtil create(String indexFolder) { - return new MetaFileUtil(indexFolder); - } - /** * Checks for existing metadata file. * @@ -66,88 +67,74 @@ public class MetaFileUtil { * Otherwise insert new. * @param metadataMap : Set of XmlMetaData objects. * @param index : The path to index location. - * @throws IOException */ - public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) throws - IOException, JAXBException { - - if (this.indexes.get(index) == null) { - this.indexes.put(index, metadataMap); - } else { - this.indexes.replace(index, metadataMap); - } + public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) { + this.indexMap = metadataMap; + this.index = index; } /** * Method to get the set of xml metadata for a given collection * - * @param index : The collection from which the metadata should be read. - * @return : Map containing the set of XmlMetadata objects. - * @throws IOException - * @throws ClassNotFoundException + * @return : Map containing the set of XmlMetadata objects.\ */ - public ConcurrentHashMap<String, XmlMetadata> getMetadata(String index) - throws IOException, ClassNotFoundException, JAXBException { - - return this.indexes.get(index); + public ConcurrentHashMap<String, XmlMetadata> getMetadata() { + return this.indexMap; } /** * Read the metadata file and create an in-memory map containing collection paths and xml files. - * @throws JAXBException */ - public void readMetadataFile() throws JAXBException { - JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); - Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); - VXQueryIndex indexes = (VXQueryIndex) jaxbUnmarshaller.unmarshal(metaFile); - - List<XmlMetadataCollection> list = indexes.getIndex(); - - - for (XmlMetadataCollection collection : list) { - String indexPath = collection.getIndexLocation(); - ConcurrentHashMap<String, XmlMetadata> metadataMap = new ConcurrentHashMap<>(); - List<XmlMetadata> metadata = collection.getMetadataList(); + public void readMetadataFile() { + try { + JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); + Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); + XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile); - this.indexToCollection.put(indexPath, collection.getCollection()); + this.collection = indexes.getCollection(); + this.index = indexes.getIndexLocation(); - for (XmlMetadata mData : metadata) { - metadataMap.put(mData.getPath(), mData); + for (XmlMetadata metadata : indexes.getMetadataList()) { + this.indexMap.put(index, metadata); + } + } catch (JAXBException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e); } - this.indexes.put(indexPath, metadataMap); } } /** * Write the content of the ConcurrentHashMap to the xml metadata file. - * @throws FileNotFoundException - * @throws JAXBException */ - public void writeMetadataToFile() throws FileNotFoundException, JAXBException { - VXQueryIndex index = new VXQueryIndex(); - List<XmlMetadataCollection> xmlMetadataCollections = new ArrayList<>(); - - for (Map.Entry<String, ConcurrentHashMap<String, XmlMetadata>> entry : indexes.entrySet()) { - XmlMetadataCollection metadataCollection = new XmlMetadataCollection(); - List<XmlMetadata> metadataList = new ArrayList<>(); - metadataCollection.setIndexLocation(entry.getKey()); - metadataCollection.setCollection(indexToCollection.get(entry.getKey())); - metadataList.addAll(entry.getValue().values()); - metadataCollection.setMetadataList(metadataList); - xmlMetadataCollections.add(metadataCollection); - } - index.setIndex(xmlMetadataCollections); + public void writeMetadataToFile() { + XmlMetadataCollection collection = new XmlMetadataCollection(); + List<XmlMetadata> metadataList = new ArrayList<>(); + for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) { + metadataList.add(entry.getValue()); + } - FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile); - JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); - Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); - jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - jaxbMarshaller.marshal(index, fileOutputStream); + collection.setMetadataList(metadataList); + collection.setCollection(this.collection); + collection.setIndexLocation(this.index); + try{ + FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile); + JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + jaxbMarshaller.marshal(collection, fileOutputStream); + + if (LOGGER.isDebugEnabled()) { + LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!"); + } + } catch (JAXBException | FileNotFoundException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e); + } + } - if (LOGGER.isDebugEnabled()) - LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!"); } @@ -157,14 +144,20 @@ public class MetaFileUtil { * * @param file : File which the checksum should be generated. * @return : Checksum String - * @throws NoSuchAlgorithmException * @throws IOException */ - public String generateMD5(File file) throws NoSuchAlgorithmException, IOException { - MessageDigest md = MessageDigest.getInstance("MD5"); - md.update(Files.readAllBytes(file.toPath())); - byte[] md5 = md.digest(); - return DatatypeConverter.printHexBinary(md5); + public String generateMD5(File file) throws IOException { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(file.toPath())); + byte[] md5 = md.digest(); + return DatatypeConverter.printHexBinary(md5); + } catch (NoSuchAlgorithmException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage()); + } + return null; + } } /** @@ -189,21 +182,17 @@ public class MetaFileUtil { /** * Get the collection for a given index location. - * @param index : path to index - * @return + * @return collection folder for a given index. */ - public String getCollection(String index) { - return this.indexToCollection.get(index); + public String getCollection() { + return this.collection; } /** * Set the entry for given index and collection. - * @param index : path to index * @param collection : path to corresponding collection */ - public void setCollectionForIndex(String index, String collection) { - if (this.indexToCollection.get(index)==null) { - this.indexToCollection.put(index, collection); - } + public void setCollection(String collection) { + this.collection = collection; } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java index 270fb8d..1f5c3e9 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java @@ -16,7 +16,11 @@ */ package org.apache.vxquery.runtime.functions.index.updateIndex; -import javax.xml.bind.annotation.*; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; import java.util.List; /** http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java index 6fa92e1..543719d 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/MetaFileUtilTest.java @@ -14,7 +14,14 @@ */ package org.apache.vxquery.indexing; -import junit.framework.Assert; +import java.io.File; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import javax.xml.bind.JAXBException; + import org.apache.commons.io.FileUtils; import org.apache.vxquery.runtime.functions.index.updateIndex.MetaFileUtil; import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata; @@ -24,12 +31,7 @@ import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; -import javax.xml.bind.JAXBException; -import java.io.File; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; +import junit.framework.Assert; /** * Test cases for testing MetaFileUtil functions. @@ -49,7 +51,7 @@ public class MetaFileUtilTest { @BeforeClass public static void setup() { new File(TestConstants.INDEX_DIR).mkdir(); - metaFileUtil = MetaFileUtil.create(TestConstants.INDEX_DIR); + metaFileUtil = new MetaFileUtil(TestConstants.INDEX_DIR); initialMap = TestConstants.getInitialMap(); modifiedMap = TestConstants.getModifiedMap(); } @@ -58,7 +60,7 @@ public class MetaFileUtilTest { * Test case for generating MD5 string for an XML file. */ @Test - public void step1_testGenerateMD5ForXML() throws IOException, NoSuchAlgorithmException { + public void step1_testGenerateMD5ForXML() throws IOException { TestConstants.createXML("catalog.xml"); File xml = new File(TestConstants.XML_FILE); String md5 = metaFileUtil.generateMD5(xml); @@ -71,7 +73,7 @@ public class MetaFileUtilTest { * Test the creation of metadata file. */ @Test - public void step2_testCreateMetaDataFile() throws IOException, JAXBException { + public void step2_testCreateMetaDataFile() { ConcurrentHashMap<String, XmlMetadata> initialMap = TestConstants.getInitialMap(); metaFileUtil.updateMetadataMap(initialMap, ""); metaFileUtil.writeMetadataToFile(); @@ -82,8 +84,8 @@ public class MetaFileUtilTest { * Validate the content of the file. */ @Test - public void step3_testValidateMetadataFile() throws IOException, ClassNotFoundException, JAXBException { - ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(""); + public void step3_testValidateMetadataFile() { + ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(); Set<String> from = fromFile.keySet(); Set<String> initial = initialMap.keySet(); @@ -100,7 +102,7 @@ public class MetaFileUtilTest { * Change the xml file and test whether the changes are detected. */ @Test - public void step4_testDetectFileChanges() throws IOException, NoSuchAlgorithmException { + public void step4_testDetectFileChanges() throws IOException { TestConstants.createXML("catalog_edited.xml"); File xml = new File(TestConstants.XML_FILE); Assert.assertTrue(metaFileUtil.generateMD5(xml).equals(TestConstants.CHANGED_MD5)); @@ -112,7 +114,7 @@ public class MetaFileUtilTest { @Test public void step5_testUpdateMetadata() throws IOException, ClassNotFoundException, NoSuchAlgorithmException, JAXBException { - ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata(""); + ConcurrentHashMap<String, XmlMetadata> fromFileMap = metaFileUtil.getMetadata(); XmlMetadata modified = fromFileMap.get(TestConstants.XML_FILE); File xml = new File(TestConstants.XML_FILE); @@ -122,7 +124,7 @@ public class MetaFileUtilTest { metaFileUtil.updateMetadataMap(fromFileMap, TestConstants.INDEX_DIR); - Assert.assertNotNull(metaFileUtil.getMetadata(TestConstants.INDEX_DIR)); + Assert.assertNotNull(metaFileUtil.getMetadata()); } @@ -130,8 +132,8 @@ public class MetaFileUtilTest { * Validate the updated metadata. */ @Test - public void step6_testVerifyMetadataChange() throws IOException, ClassNotFoundException, JAXBException { - ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(TestConstants.INDEX_DIR); + public void step6_testVerifyMetadataChange() { + ConcurrentHashMap<String, XmlMetadata> fromFile = metaFileUtil.getMetadata(); Set<String> from = fromFile.keySet(); Set<String> modified = modifiedMap.keySet(); http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java index 68d40b5..b79107e 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/indexing/TestConstants.java @@ -16,7 +16,13 @@ package org.apache.vxquery.indexing; import org.apache.vxquery.runtime.functions.index.updateIndex.XmlMetadata; -import java.io.*; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Paths; import java.util.concurrent.ConcurrentHashMap; http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/conf/cluster_example.xml ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/resources/conf/cluster_example.xml b/vxquery-server/src/main/resources/conf/cluster_example.xml index 18d9173..f43b22a 100644 --- a/vxquery-server/src/main/resources/conf/cluster_example.xml +++ b/vxquery-server/src/main/resources/conf/cluster_example.xml @@ -17,6 +17,7 @@ <cluster xmlns="cluster"> <name>local</name> <username>joe</username> + <index_directory>/tmp/indexFolder</index_directory> <master_node> <id>master</id> <client_ip>128.195.52.177</client_ip> http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/conf/local.xml ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/resources/conf/local.xml b/vxquery-server/src/main/resources/conf/local.xml index 4a48620..e0d07f8 100644 --- a/vxquery-server/src/main/resources/conf/local.xml +++ b/vxquery-server/src/main/resources/conf/local.xml @@ -16,6 +16,7 @@ --> <cluster xmlns="cluster"> <name>local</name> + <index_directory>/tmp/indexFolder</index_directory> <master_node> <id>master</id> <client_ip>127.0.0.1</client_ip> http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/cluster_actions.py ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/resources/scripts/cluster_actions.py b/vxquery-server/src/main/resources/scripts/cluster_actions.py index deeee33..8ce1607 100644 --- a/vxquery-server/src/main/resources/scripts/cluster_actions.py +++ b/vxquery-server/src/main/resources/scripts/cluster_actions.py @@ -120,7 +120,7 @@ class ClusterActions: def start_nc(self, machine, cc): print "Start Node Controller." print " " + machine.get_id() + " " + machine.get_ip() - command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\"" + command = "./vxquery-server/target/appassembler/bin/startnc.sh " + machine.get_id() + " " + machine.get_ip() + " " + cc.get_client_ip() + " \"" + cc.get_client_port() + "\" \"" + machine.get_java_opts() + "\" \"" + self.ci.get_index_directory() + "\"" self.run_remote_command(machine.get_username(), machine.get_id(), command) def stop_cc_and_all_ncs(self, machine): http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/cluster_information.py ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/resources/scripts/cluster_information.py b/vxquery-server/src/main/resources/scripts/cluster_information.py index 94b231d..d70828d 100644 --- a/vxquery-server/src/main/resources/scripts/cluster_information.py +++ b/vxquery-server/src/main/resources/scripts/cluster_information.py @@ -28,6 +28,9 @@ class ClusterInformation: def get_java_opts(self): return get_tag_text(self.config, "java_opts") + def get_index_directory(self): + return get_tag_text(self.config, "index_directory"); + def get_master_node_machine(self): master_node = self.config.getElementsByTagName("master_node")[0] id = NodeXmlReader.get_cluster_id(master_node) http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-server/src/main/resources/scripts/startnc.sh ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/resources/scripts/startnc.sh b/vxquery-server/src/main/resources/scripts/startnc.sh index a199ec8..d719d22 100755 --- a/vxquery-server/src/main/resources/scripts/startnc.sh +++ b/vxquery-server/src/main/resources/scripts/startnc.sh @@ -24,6 +24,7 @@ IPADDR=$2 CCHOST=$3 CCPORT=$4 J_OPTS=$5 +IO_DEVICES=$6 #Import cluster properties MYDIR="$(dirname -- $0)" @@ -62,6 +63,9 @@ NC_OPTIONS=" -cc-host ${CCHOST} -cluster-net-ip-address ${IPADDR} -data-ip-addr NC_OPTIONS=" ${NC_OPTIONS} ${NCOPTS} " +[ "${IO_DEVICES}" ] && + NC_OPTIONS=" ${NC_OPTIONS} -iodevices ${IO_DEVICES} " + echo "${JAVA_OPTS}" &> ${NCLOGS_DIR}/nc.log http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java index 75dce1b..d7db8a8 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestClusterUtil.java @@ -17,16 +17,16 @@ package org.apache.vxquery.xtest; -import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; - import org.apache.hyracks.control.cc.ClusterControllerService; import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.hyracks.control.nc.NodeControllerService; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; + public class TestClusterUtil { private static int clientNetPort = 39000; @@ -34,6 +34,7 @@ public class TestClusterUtil { private static int profileDumpPeriod = 10000; private static String ccHost = "localhost"; private static String nodeId = "nc1"; + private static String ioDevices = "target/tmp/indexFolder"; private TestClusterUtil() { } @@ -58,6 +59,7 @@ public class TestClusterUtil { ncConfig1.dataIPAddress = publicAddress; ncConfig1.resultIPAddress = publicAddress; ncConfig1.nodeId = nodeId; + ncConfig1.ioDevices = ioDevices; return ncConfig1; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java index c6b339b..78ed8ff 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java @@ -80,6 +80,7 @@ public abstract class AbstractXQueryTest { case EXPECTED_RESULT_GOT_SAME_RESULT: break; case NO_RESULT_FILE: + fail(result.state + " (" + result.time + " ms): " + result.testCase.getXQueryDisplayName()); break; } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt new file mode 100644 index 0000000..f32a580 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/createIndex.txt @@ -0,0 +1 @@ +true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt new file mode 100644 index 0000000..f32a580 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/deleteIndex.txt @@ -0,0 +1 @@ +true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt new file mode 100644 index 0000000..17d6fec --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex1.txt @@ -0,0 +1 @@ +src/test/resources/TestSources/ghcnd http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/showIndex2.txt @@ -0,0 +1 @@ + http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt new file mode 100644 index 0000000..f32a580 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/updateIndex.txt @@ -0,0 +1 @@ +true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt new file mode 100644 index 0000000..baf9dca --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex1.txt @@ -0,0 +1,2 @@ +<data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data> +<data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt new file mode 100644 index 0000000..ef8dde4 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex2.txt @@ -0,0 +1 @@ +<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex3.txt @@ -0,0 +1 @@ +2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt new file mode 100644 index 0000000..f30101c --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex4.txt @@ -0,0 +1 @@ +3.3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt new file mode 100644 index 0000000..c84c360 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex5.txt @@ -0,0 +1,3 @@ +<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data> +<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data> +<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt new file mode 100644 index 0000000..9abedff --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex6.txt @@ -0,0 +1,2 @@ +<station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station> +<station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName/></locationLabels></station> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt new file mode 100644 index 0000000..c84c360 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-1/useIndex7.txt @@ -0,0 +1,3 @@ +<data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data> +<data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute/><attribute/><attribute>a</attribute><attribute/></attributes></data> +<data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/eb76640f/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt new file mode 100644 index 0000000..0be5d98 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Indexing/Partition-2/createIndex.txt @@ -0,0 +1,2 @@ +true +true \ No newline at end of file
