http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java new file mode 100644 index 0000000..1d58433 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexCentralizerUtil.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.index.centralizer; + +import java.io.DataOutput; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.commons.io.FileUtils; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder; +import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; +import org.apache.vxquery.datamodel.values.ValueTag; + +/** + * Class for maintaining the centralized index information file. + * Index centralization procedure. + * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = < index_directory > . + * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections. + * There will be a single xml file, located in the directory specified in local.xml, which contains all information + * about the existing indexes. + * This class can be used to read, add, delete, modify the entries and write the file back to the disk. + */ +public class IndexCentralizerUtil { + + private static final String FILE_NAME = "VXQuery-Index-Directory.xml"; + private final List<String> collections = new ArrayList<>(); + private static final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil"); + private File xmlFile; + private String indexPath; + public static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>(); + private static final StringValueBuilder svb = new StringValueBuilder(); + private final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); + private final DataOutput output = abvs.getDataOutput(); + + public IndexCentralizerUtil(File index) { + indexPath = index.getPath(); + if (!index.exists()) { + try { + FileUtils.forceMkdir(index); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + indexPath + " " + e); + } + } + xmlFile = new File(index.getPath() + "/" + FILE_NAME); + } + + /** + * Get the index directory containing index of the given collection + * + * @param collection + * : Collection folder + * @return Index folder. + */ + public String getIndexForCollection(String collection) { + if (indexCollectionMap.size() > 0 && indexCollectionMap.containsKey(collection)) { + return indexCollectionMap.get(collection).getIndex(); + } + return null; + } + + /** + * Put the index location corresponding to given collection. + * Index location is created by using the last 100 characters of collection. + * + * @param collection + * : Collection directory + * @return index + */ + public String putIndexForCollection(String collection) { + int length = collection.replaceAll("/", "").length(); + String index = collection.replaceAll("/", ""); + index = indexPath + "/" + (length > 100 ? index.substring(length - 100) : index); + IndexLocator il = new IndexLocator(); + il.setCollection(collection); + il.setIndex(index); + if (indexCollectionMap.get(collection) != null) { + return index; + } + indexCollectionMap.put(collection, il); + return index; + } + + /** + * Remove the entry for given collection directory. + * + * @param collection + * : Collection directory + */ + public void deleteEntryForCollection(String collection) { + indexCollectionMap.remove(collection); + } + + /** + * Prints all collections which have an index created. + * + * @param sb + * : The output is stored in a sequence + * @throws IOException + * : If writing the dataOutput generates {@link IOException} + */ + public void getAllCollections(SequenceBuilder sb) throws IOException { + for (String s : collections) { + abvs.reset(); + output.write(ValueTag.XS_STRING_TAG); + svb.write(s, output); + sb.addItem(abvs); + } + } + + /** + * Read the collection, index directory file and populate the HashMap. + */ + public void readIndexDirectory() { + if (xmlFile.exists()) { + try { + JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class); + Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); + IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(xmlFile); + + for (IndexLocator il : indexDirectory.getDirectory()) { + indexCollectionMap.put(il.getCollection(), il); + this.collections.add(il.getCollection()); + } + } catch (JAXBException e) { + LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e); + } + } + + } + + /** + * Write back the contents of the HashMap to the file. + */ + public void writeIndexDirectory() { + IndexDirectory id = new IndexDirectory(); + List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values()); + id.setDirectory(indexLocators); + try { + FileOutputStream fileOutputStream = new FileOutputStream(this.xmlFile); + JAXBContext context = JAXBContext.newInstance(IndexDirectory.class); + Marshaller jaxbMarshaller = context.createMarshaller(); + jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + jaxbMarshaller.marshal(id, fileOutputStream); + } catch (JAXBException | FileNotFoundException e) { + LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e); + } + } +}
http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java new file mode 100644 index 0000000..d118926 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexDirectory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.index.centralizer; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "indexes") +@XmlAccessorType(XmlAccessType.FIELD) +public class IndexDirectory implements Serializable { + private static final long serialVersionUID = 1L; + + @XmlElement(name = "index", type = IndexLocator.class) + private List<IndexLocator> directory = new ArrayList<>(); + + public List<IndexLocator> getDirectory() { + return directory; + } + + public void setDirectory(List<IndexLocator> directory) { + this.directory = directory; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java new file mode 100644 index 0000000..49dbecd --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/centralizer/IndexLocator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.runtime.functions.index.centralizer; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "Entry") +public class IndexLocator implements Serializable { + private static final long serialVersionUID = 1L; + + @XmlAttribute + private String collection; + + @XmlAttribute + private String index; + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java deleted file mode 100644 index 51510d5..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.vxquery.runtime.functions.index.indexCentralizer; - -import java.io.DataOutput; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; - -import org.apache.commons.io.FileUtils; -import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder; -import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; -import org.apache.vxquery.datamodel.values.ValueTag; - -/** - * Class for maintaining the centralized index information file. - * Index centralization procedure. - * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = < index_directory > . - * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections. - * There will be a single xml file, located in the directory specified in local.xml, which contains all information - * about the existing indexes. - * This class can be used to read, add, delete, modify the entries and write the file back to the disk. - */ -public class IndexCentralizerUtil { - - private final String FILE_NAME = "VXQuery-Index-Directory.xml"; - private final List<String> collections = new ArrayList<>(); - private final Logger LOGGER = Logger.getLogger("IndexCentralizerUtil"); - private File XML_FILE; - private String INDEX_LOCATION; - private static ConcurrentHashMap<String, IndexLocator> indexCollectionMap = new ConcurrentHashMap<>(); - - public IndexCentralizerUtil(File index) { - this.INDEX_LOCATION = index.getPath(); - if (!index.exists()) { - try { - FileUtils.forceMkdir(index); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, "Could not create the index directory for path: " + INDEX_LOCATION + " " + e); - } - } - XML_FILE = new File(index.getPath() + "/" + FILE_NAME); - } - - /** - * Get the index directory containing index of the given collection - * - * @param collection : Collection folder - * @return Index folder. - */ - public String getIndexForCollection(String collection) { - return indexCollectionMap.get(collection).getIndex(); - } - - /** - * Put the index location corresponding to given collection. - * Index location is created by using the last 100 characters of collection. - * - * @param collection : Collection directory - * @return index - */ - public String putIndexForCollection(String collection) { - int length = collection.replaceAll("/", "").length(); - String index = collection.replaceAll("/", ""); - index = INDEX_LOCATION + "/" + (length > 100 ? index.substring(length - 100) : index); - IndexLocator il = new IndexLocator(); - il.setCollection(collection); - il.setIndex(index); - if (indexCollectionMap.get(collection) != null) { - return index; - } - indexCollectionMap.put(collection, il); - return index; - } - - /** - * Remove the entry for given collection directory. - * - * @param collection : Collection directory - */ - public void deleteEntryForCollection(String collection) { - indexCollectionMap.remove(collection); - } - - /** - * Prints all collections which have an index created. - * @param sb : The output is stored in a sequence - * @throws IOException : If writing the dataOutput generates {@link IOException} - */ - public void getAllCollections(SequenceBuilder sb) throws IOException { - for (String s : collections) { - StringValueBuilder svb = new StringValueBuilder(); - ArrayBackedValueStorage abvs = new ArrayBackedValueStorage(); - DataOutput output = abvs.getDataOutput(); - output.write(ValueTag.XS_STRING_TAG); - svb.write(s, output); - sb.addItem(abvs); - } - } - - /** - * Read the collection, index directory file and populate the HashMap. - */ - public void readIndexDirectory() { - if (this.XML_FILE.exists()) { - try { - JAXBContext jaxbContext = JAXBContext.newInstance(IndexDirectory.class); - Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); - IndexDirectory indexDirectory = (IndexDirectory) jaxbUnmarshaller.unmarshal(this.XML_FILE); - - for (IndexLocator il : indexDirectory.getDirectory()) { - indexCollectionMap.put(il.getCollection(), il); - this.collections.add(il.getCollection()); - } - } catch (JAXBException e) { - LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e); - } - } - - } - - /** - * Write back the contents of the HashMap to the file. - */ - public void writeIndexDirectory() { - IndexDirectory id = new IndexDirectory(); - List<IndexLocator> indexLocators = new ArrayList<>(indexCollectionMap.values()); - id.setDirectory(indexLocators); - try { - FileOutputStream fileOutputStream = new FileOutputStream(this.XML_FILE); - JAXBContext context = JAXBContext.newInstance(IndexDirectory.class); - Marshaller jaxbMarshaller = context.createMarshaller(); - jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - jaxbMarshaller.marshal(id, fileOutputStream); - } catch (JAXBException | FileNotFoundException e) { - LOGGER.log(Level.SEVERE, "Could not read the XML file due to " + e); - } - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java deleted file mode 100644 index 54d9ad9..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexDirectory.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.vxquery.runtime.functions.index.indexCentralizer; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -@XmlRootElement(name = "indexes") -@XmlAccessorType(XmlAccessType.FIELD) -public class IndexDirectory implements Serializable{ - - @XmlElement(name = "index", type = IndexLocator.class) - private List<IndexLocator> directory = new ArrayList<>(); - - public List<IndexLocator> getDirectory() { - return directory; - } - - - public void setDirectory(List<IndexLocator> directory) { - this.directory = directory; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java deleted file mode 100644 index 1a33c8b..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexLocator.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.vxquery.runtime.functions.index.indexCentralizer; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; - -@XmlAccessorType(XmlAccessType.FIELD) -@XmlRootElement(name = "Entry") -public class IndexLocator implements Serializable{ - - @XmlAttribute - private String collection; - - @XmlAttribute - private String index; - - public String getCollection() { - return collection; - } - - public void setCollection(String collection) { - this.collection = collection; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java new file mode 100644 index 0000000..0346a62 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/Constants.java @@ -0,0 +1,28 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index.update; + +/** + * Constants used in updating index + */ +public class Constants { + public static final String FIELD_PATH = "path"; + public static final String META_FILE_NAME = "vxquery_index.xml"; + + private Constants() { + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java new file mode 100644 index 0000000..65a8325 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/IndexUpdater.java @@ -0,0 +1,304 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index.update; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.index.IndexDocumentBuilder; +import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer; +import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil; +import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; + +/** + * Update the index if the source files are changed. + */ +public class IndexUpdater { + private MetaFileUtil metaFileUtil; + private ConcurrentHashMap<String, XmlMetadata> metadataMap; + private IPointable result; + private final SequenceBuilder sb = new SequenceBuilder(); + private ArrayBackedValueStorage abvs; + private ITreeNodeIdProvider nodeIdProvider; + private ArrayBackedValueStorage abvsFileNode; + private String nodeId; + private IndexWriter indexWriter; + private Set<String> pathsFromFileList; + private String collectionFolder; + private String indexFolder; + private final Logger LOGGER = Logger.getLogger("Index Updater"); + private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); + private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); + + public IndexUpdater(String indexFolder, IPointable result, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, + ArrayBackedValueStorage abvsFileNode, String nodeId) { + this.indexFolder = indexFolder; + this.result = result; + this.abvs = abvs; + this.nodeIdProvider = nodeIdProvider; + this.abvsFileNode = abvsFileNode; + this.nodeId = nodeId; + this.pathsFromFileList = new HashSet<>(); + } + + /** + * Perform the initial configuration for index update/ delete processes. + * + * @throws IOException + * : If getting the index folder generates {@link IOException} + */ + public void setup() throws IOException { + // Read the metadata file and load the metadata map into memory. + metaFileUtil = new MetaFileUtil(indexFolder); + metaFileUtil.readMetadataFile(); + metadataMap = metaFileUtil.getMetadata(); + + // Retrieve the collection folder path. + // Remove the entry for ease of the next steps. + collectionFolder = metaFileUtil.getCollection(); + + abvs.reset(); + sb.reset(abvs); + + Directory fsdir = FSDirectory.open(Paths.get(indexFolder)); + indexWriter = new IndexWriter(fsdir, new IndexWriterConfig(new CaseSensitiveAnalyzer()) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND)); + } + + /** + * Wrapper for update index function. + * + * @throws IOException + * : If the directory doesn't exist + */ + public void updateIndex() throws IOException { + File collectionDirectory = new File(collectionFolder); + if (!collectionDirectory.exists()) { + throw new IOException("The collection directory (" + collectionFolder + ") does not exist."); + } + + //Execute update index process + updateIndex(collectionDirectory); + + //Detect deleted files and execute the delete index process. + deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList); + updateMetadataFile(); + } + + /** + * Close opened IndexWriter and terminate the index update/ delete process. + * + * @throws IOException + * : If exiting the index folder generates {@link IOException} + */ + public void exit() throws IOException { + indexWriter.forceMerge(1); + + indexWriter.close(); + + sb.finish(); + result.set(abvs); + } + + /** + * Functional wrapper to update Metadata file. + * + * @throws IOException + * : If updating metadata folder generates {@link IOException} + */ + public synchronized void updateMetadataFile() throws IOException { + //Write the updated metadata to the file. + metaFileUtil.updateMetadataMap(metadataMap, indexFolder); + metaFileUtil.writeMetadataToFile(); + } + + /** + * Check the collection for changes. + * If changes are detected, update the index + * + * @param collection + * : Collection folder path + */ + private void updateIndex(File collection) throws IOException { + + File[] list = collection.listFiles(); + + assert list != null; + for (File file : list) { + pathsFromFileList.add(file.getCanonicalPath()); + if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) { + XmlMetadata data = metadataMap.get(file.getCanonicalPath()); + String md5 = metaFileUtil.generateMD5(file); + + abvsFileNode.reset(); + + IndexDocumentBuilder indexDocumentBuilder; + if (data != null) { + + // This case checks whether the file has been changed. + // If the file has changed, delete the existing document, create a new index document and add it + // to the current index. + // At the same time, update the metadata for the file. + if (!md5.equals(data.getMd5())) { + + //Update index corresponding to the xml file. + indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath())); + indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, + nodeIdProvider, nodeId); + indexDocumentBuilder.printStart(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.log(Level.DEBUG, "New Index is created for updated file " + file.getCanonicalPath()); + } + + //Update the metadata map. + XmlMetadata metadata = updateEntry(file, data); + metadataMap.replace(file.getCanonicalPath(), metadata); + + } + } else { + + // In this case, the xml file has not added to the index. (It is a newly added file) + // Therefore generate a new index for this file and add it to the existing index. + indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, abvsFileNode, nodeIdProvider, + nodeId); + indexDocumentBuilder.printStart(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.log(Level.DEBUG, "New Index is created for newly added file " + file.getCanonicalPath()); + } + + XmlMetadata metadata = updateEntry(file, null); + metadataMap.put(file.getCanonicalPath(), metadata); + } + } else if (file.isDirectory()) { + updateIndex(file); + } + } + } + + /** + * Update the current XmlMetadata object related to the currently reading XML file. + * + * @param file + * : XML file + * @param metadata + * : Existing metadata object + * @return : XML metadata object with updated fields. + * @throws IOException + * : If getting the file info generates {@link IOException} + */ + private XmlMetadata updateEntry(File file, XmlMetadata metadataArg) throws IOException { + XmlMetadata metadata = metadataArg; + if (metadata == null) { + metadata = new XmlMetadata(); + } + metadata.setFileName(file.getName()); + metadata.setPath(file.getCanonicalPath()); + metadata.setMd5(metaFileUtil.generateMD5(file)); + metadata.setLastModified(sdf.format(file.lastModified())); + return metadata; + } + + /** + * Delete the index of deleted files. + * + * @param pathsFromMap + * : Set of paths taken from metafile. + * @param pathsFromFileList + * : Set of paths taken from list of existing files. + * @throws IOException + * : If deleting Documents generates {@link IOException} + */ + private void deleteIndexOfDeletedFiles(Set<String> pathsFromMap, Set<String> pathsFromFileList) throws IOException { + Set<String> sfm = new HashSet<>(pathsFromMap); + + // If any file has been deleted from the collection, the number of files stored in metadata is higher than + // the actual number of files. + // With set difference, the paths of deleted files are taken from the stored metadata. + // Delete the corresponding indexes of each file from the index and as well as remove the entry from the + // metadata file. + + if (sfm.size() > pathsFromFileList.size()) { + sfm.removeAll(pathsFromFileList); + + for (String s : sfm) { + metadataMap.remove(s); + indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, s)); + if (LOGGER.isDebugEnabled()) { + LOGGER.log(Level.DEBUG, "Index of the deleted file " + s + " was deleted from the index!"); + } + } + } + } + + /** + * Delete all indexes in the given directory. + * This will also remove the existing metadata file. + * It will be created when recreating the index. + * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be + * restored. + * Otherwise the changes will be committed. + * + * @throws SystemException + * : An attempt to divide by zero + */ + public void deleteAllIndexes() throws SystemException { + try { + indexWriter.deleteAll(); + indexWriter.commit(); + indexWriter.close(); + metaFileUtil.deleteMetaDataFile(); + + for (File f : (new File(indexFolder)).listFiles()) + Files.delete(f.toPath()); + + sb.finish(); + result.set(abvs); + } catch (IOException e) { + try { + indexWriter.rollback(); + indexWriter.close(); + + sb.finish(); + result.set(abvs); + } catch (IOException e1) { + throw new SystemException(ErrorCode.FOAR0001, e1); + } + } + + } + +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java new file mode 100644 index 0000000..5f41355 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/MetaFileUtil.java @@ -0,0 +1,205 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index.update; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.xml.bind.DatatypeConverter; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * Utility class for writing, reading metadata file and generating checksum. + */ +public class MetaFileUtil { + + private File metaFile; + private static final Logger LOGGER = Logger.getLogger("MetadataFileUtil"); + private String index; + private String collection; + private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>(); + + public MetaFileUtil(String indexFolder) { + this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME); + } + + /** + * Checks for existing metadata file. + * + * @return true if the metadata file is present + */ + public boolean isMetaFilePresent() { + return metaFile.exists(); + } + + /** + * Update the content of the metadata map. + * If the current collection data is present, replace it. + * Otherwise insert new. + * + * @param metadataMap + * : Set of XmlMetaData objects. + * @param index + * : The path to index location. + */ + public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) { + this.indexMap = metadataMap; + this.index = index; + + } + + /** + * Method to get the set of xml metadata for a given collection + * + * @return : Map containing the set of XmlMetadata objects.\ + */ + public ConcurrentHashMap<String, XmlMetadata> getMetadata() { + return this.indexMap; + } + + /** + * Read the metadata file and create an in-memory map containing collection paths and xml files. + */ + public void readMetadataFile() { + try { + JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); + Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); + XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile); + + this.collection = indexes.getCollection(); + this.index = indexes.getIndexLocation(); + + for (XmlMetadata metadata : indexes.getMetadataList()) { + this.indexMap.put(index, metadata); + } + } catch (JAXBException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e); + } + } + } + + /** + * Write the content of the ConcurrentHashMap to the xml metadata file. + */ + public void writeMetadataToFile() { + XmlMetadataCollection xmlMetadataCollection = new XmlMetadataCollection(); + List<XmlMetadata> metadataList = new ArrayList<>(); + + for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) { + metadataList.add(entry.getValue()); + } + + xmlMetadataCollection.setMetadataList(metadataList); + xmlMetadataCollection.setCollection(collection); + xmlMetadataCollection.setIndexLocation(this.index); + try { + FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile); + JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + jaxbMarshaller.marshal(xmlMetadataCollection, fileOutputStream); + + if (LOGGER.isDebugEnabled()) { + LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!"); + } + } catch (JAXBException | FileNotFoundException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e); + } + } + + } + + /** + * Generate MD5 checksum string for a given file. + * + * @param file + * : File which the checksum should be generated. + * @return : Checksum String + * @throws IOException + * : The file is not available + */ + public String generateMD5(File file) throws IOException { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(Files.readAllBytes(file.toPath())); + byte[] md5 = md.digest(); + return DatatypeConverter.printHexBinary(md5); + } catch (NoSuchAlgorithmException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage()); + } + return null; + } + } + + /** + * Delete the existing Metadata file. + * + * @return True if deleted, false otherwise. + */ + public boolean deleteMetaDataFile() { + try { + Files.delete(Paths.get(metaFile.getCanonicalPath())); + if (LOGGER.isDebugEnabled()) { + LOGGER.log(Level.DEBUG, "Metadata file deleted!"); + } + return true; + } catch (IOException e) { + if (LOGGER.isTraceEnabled()) { + LOGGER.log(Level.ERROR, "Metadata file could not be deleted!"); + } + return false; + } + } + + /** + * Get the collection for a given index location. + * + * @return collection folder for a given index. + */ + public String getCollection() { + return this.collection; + } + + /** + * Set the entry for given index and collection. + * + * @param collection + * : path to corresponding collection + */ + public void setCollection(String collection) { + this.collection = collection; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java new file mode 100644 index 0000000..fc96763 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/VXQueryIndex.java @@ -0,0 +1,42 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index.update; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +/** + * Class for storing metadata information for vxquery index. + */ +@XmlAccessorType(XmlAccessType.PROPERTY) +@XmlRootElement(name = "indexes") +public class VXQueryIndex { + + private List<XmlMetadataCollection> indexes; + + public List<XmlMetadataCollection> getIndex() { + return indexes; + } + + @XmlElement(name = "index", type = XmlMetadataCollection.class) + public void setIndex(List<XmlMetadataCollection> index) { + this.indexes = index; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java new file mode 100644 index 0000000..063120c --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadata.java @@ -0,0 +1,74 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index.update; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Class to store metadata related to an XML file. + * This contains + * - Path to the xml file + * - MD5 Checksum String + * - File name + * - Last modified date + */ +@XmlRootElement(name = "file") +@XmlAccessorType(XmlAccessType.FIELD) +public class XmlMetadata implements Serializable { + private static final long serialVersionUID = 1L; + + private String path; + private String md5; + private String fileName; + private String lastModified; + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getMd5() { + return md5; + } + + public void setMd5(String md5) { + this.md5 = md5; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getLastModified() { + return lastModified; + } + + public void setLastModified(String lastModified) { + this.lastModified = lastModified; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java new file mode 100644 index 0000000..a1ca776 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/update/XmlMetadataCollection.java @@ -0,0 +1,66 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.vxquery.runtime.functions.index.update; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +/** + * Class for holding the collection information and the list of XML metadata related to the xml files in the + * collection. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "index") +public class XmlMetadataCollection { + + @XmlAttribute(name = "location") + private String indexLocation; + + @XmlAttribute(name = "collection") + private String collection; + + @XmlElement(name = "file", type = XmlMetadata.class) + private List<XmlMetadata> metadataList; + + public List<XmlMetadata> getMetadataList() { + return metadataList; + } + + public void setMetadataList(List<XmlMetadata> metadataList) { + this.metadataList = metadataList; + } + + public String getIndexLocation() { + return indexLocation; + } + + public void setIndexLocation(String indexLocation) { + this.indexLocation = indexLocation; + } + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java deleted file mode 100644 index 2a45747..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/Constants.java +++ /dev/null @@ -1,25 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index.updateIndex; - -/** - * Constants used in updating index - */ -public class Constants { - public static String FIELD_PATH = "path"; - public static String META_FILE_NAME = "vxquery_index.xml"; -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java deleted file mode 100644 index d3b9fdf..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java +++ /dev/null @@ -1,319 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index.updateIndex; - -import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; -import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; -import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; -import org.apache.vxquery.exceptions.ErrorCode; -import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.index.IndexDocumentBuilder; -import org.apache.vxquery.runtime.functions.index.CaseSensitiveAnalyzer; -import org.apache.vxquery.runtime.functions.index.IndexConstructorUtil; -import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; - -import java.io.DataInputStream; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Update the index if the source files are changed. - */ -public class IndexUpdater { - private MetaFileUtil metaFileUtil; - private ConcurrentHashMap<String, XmlMetadata> metadataMap; - private IPointable result; - private ByteBufferInputStream bbis; - private DataInputStream di; - private SequenceBuilder sb; - private ArrayBackedValueStorage abvs; - private ITreeNodeIdProvider nodeIdProvider; - private ArrayBackedValueStorage abvsFileNode; - private TaggedValuePointable nodep; - private String nodeId; - private IndexWriter indexWriter; - private Set<String> pathsFromFileList; - private String collectionFolder; - private String indexFolder; - private Logger LOGGER = Logger.getLogger("Index Updater"); - private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); - private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); - - public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis, - DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, - ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) { - this.indexFolder = indexFolder; - this.result = result; - this.bbis = bbis; - this.di = di; - this.sb = sb; - this.abvs = abvs; - this.nodeIdProvider = nodeIdProvider; - this.abvsFileNode = abvsFileNode; - this.nodep = nodep; - this.nodeId = nodeId; - this.pathsFromFileList = new HashSet<>(); - } - - /** - * Perform the initial configuration for index update/ delete processes. - * - * @throws SystemException - * : If getting the index folder generates {@link SystemException} - * @throws IOException - * : If getting the index folder generates {@link IOException} - */ - public void setup() throws SystemException, IOException { - - // Read the metadata file and load the metadata map into memory. - metaFileUtil = new MetaFileUtil(indexFolder); - metaFileUtil.readMetadataFile(); - metadataMap = metaFileUtil.getMetadata(); - - // Retrieve the collection folder path. - // Remove the entry for ease of the next steps. - collectionFolder = metaFileUtil.getCollection(); - - abvs.reset(); - sb.reset(abvs); - - Directory fsdir = FSDirectory.open(Paths.get(indexFolder)); - indexWriter = new IndexWriter(fsdir, new IndexWriterConfig(new CaseSensitiveAnalyzer()) - .setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND)); - } - - /** - * Wrapper for update index function. - * - * @throws IOException - * : If the directory doesn't exist - */ - public void updateIndex() throws IOException { - File collectionDirectory = new File(collectionFolder); - if (!collectionDirectory.exists()) { - throw new RuntimeException("The collection directory (" + collectionFolder + ") does not exist."); - } - - //Execute update index process - updateIndex(collectionDirectory); - - //Detect deleted files and execute the delete index process. - deleteIndexOfDeletedFiles(metadataMap.keySet(), pathsFromFileList); - updateMetadataFile(); - } - - /** - * Close opened IndexWriter and terminate the index update/ delete process. - * - * @throws IOException - * : If exiting the index folder generates {@link IOException} - */ - public void exit() throws IOException { - indexWriter.forceMerge(1); - - indexWriter.close(); - - sb.finish(); - result.set(abvs); - } - - /** - * Functional wrapper to update Metadata file. - * - * @throws IOException - * : If updating metadata folder generates {@link IOException} - */ - public synchronized void updateMetadataFile() throws IOException { - //Write the updated metadata to the file. - metaFileUtil.updateMetadataMap(metadataMap, indexFolder); - metaFileUtil.writeMetadataToFile(); - } - - /** - * Check the collection for changes. - * If changes are detected, update the index - * - * @param collection - * : Collection folder path - */ - private void updateIndex(File collection) throws IOException { - - File[] list = collection.listFiles(); - - assert list != null; - for (File file : list) { - pathsFromFileList.add(file.getCanonicalPath()); - if (indexConstructorUtil.readableXmlFile(file.getCanonicalPath())) { - XmlMetadata data = metadataMap.get(file.getCanonicalPath()); - String md5 = metaFileUtil.generateMD5(file); - - abvsFileNode.reset(); - - IndexDocumentBuilder indexDocumentBuilder; - if (data != null) { - - // This case checks whether the file has been changed. - // If the file has changed, delete the existing document, create a new index document and add it - // to the current index. - // At the same time, update the metadata for the file. - if (!md5.equals(data.getMd5())) { - - //Update index corresponding to the xml file. - indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath())); - indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, - abvsFileNode, nodeIdProvider, bbis, di, nodeId); - indexDocumentBuilder.printStart(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.log(Level.DEBUG, "New Index is created for updated file " + file.getCanonicalPath()); - } - - //Update the metadata map. - XmlMetadata metadata = updateEntry(file, data); - metadataMap.replace(file.getCanonicalPath(), metadata); - - } - } else { - - // In this case, the xml file has not added to the index. (It is a newly added file) - // Therefore generate a new index for this file and add it to the existing index. - indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, - nodeIdProvider, bbis, di, nodeId); - indexDocumentBuilder.printStart(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.log(Level.DEBUG, "New Index is created for newly added file " + file.getCanonicalPath()); - } - - XmlMetadata metadata = updateEntry(file, null); - metadataMap.put(file.getCanonicalPath(), metadata); - } - } else if (file.isDirectory()) { - updateIndex(file); - } - } - } - - /** - * Update the current XmlMetadata object related to the currently reading XML file. - * - * @param file - * : XML file - * @param metadata - * : Existing metadata object - * @return : XML metadata object with updated fields. - * @throws IOException - * : If getting the file info generates {@link IOException} - */ - private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException { - - if (metadata == null) { - metadata = new XmlMetadata(); - } - metadata.setFileName(file.getName()); - metadata.setPath(file.getCanonicalPath()); - metadata.setMd5(metaFileUtil.generateMD5(file)); - metadata.setLastModified(sdf.format(file.lastModified())); - return metadata; - } - - /** - * Delete the index of deleted files. - * - * @param pathsFromMap - * : Set of paths taken from metafile. - * @param pathsFromFileList - * : Set of paths taken from list of existing files. - * @throws IOException - * : If deleting Documents generates {@link IOException} - */ - private void deleteIndexOfDeletedFiles(Set<String> pathsFromMap, Set<String> pathsFromFileList) throws IOException { - Set<String> sfm = new HashSet<>(pathsFromMap); - - // If any file has been deleted from the collection, the number of files stored in metadata is higher than - // the actual number of files. - // With set difference, the paths of deleted files are taken from the stored metadata. - // Delete the corresponding indexes of each file from the index and as well as remove the entry from the - // metadata file. - - if (sfm.size() > pathsFromFileList.size()) { - sfm.removeAll(pathsFromFileList); - - for (String s : sfm) { - metadataMap.remove(s); - indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, s)); - if (LOGGER.isDebugEnabled()) { - LOGGER.log(Level.DEBUG, "Index of the deleted file " + s + " was deleted from the index!"); - } - } - } - } - - /** - * Delete all indexes in the given directory. - * This will also remove the existing metadata file. - * It will be created when recreating the index. - * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be - * restored. - * Otherwise the changes will be committed. - * - * @throws SystemException - * : An attempt to divide by zero - */ - public void deleteAllIndexes() throws SystemException { - try { - indexWriter.deleteAll(); - indexWriter.commit(); - indexWriter.close(); - metaFileUtil.deleteMetaDataFile(); - - for (File f : (new File(indexFolder)).listFiles()) - Files.delete(f.toPath()); - - sb.finish(); - result.set(abvs); - } catch (IOException e) { - try { - indexWriter.rollback(); - indexWriter.close(); - - sb.finish(); - result.set(abvs); - } catch (IOException e1) { - throw new SystemException(ErrorCode.FOAR0001); - } - } - - } - -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java deleted file mode 100644 index 0dfb54a..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java +++ /dev/null @@ -1,198 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index.updateIndex; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -import javax.xml.bind.DatatypeConverter; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Utility class for writing, reading metadata file and generating checksum. - */ -public class MetaFileUtil { - - private File metaFile; - private Logger LOGGER = Logger.getLogger("MetadataFileUtil"); - private String index; - private String collection; - private ConcurrentHashMap<String, XmlMetadata> indexMap = new ConcurrentHashMap<>(); - - public MetaFileUtil(String indexFolder) { - this.metaFile = new File(indexFolder + "/" + Constants.META_FILE_NAME); - } - - /** - * Checks for existing metadata file. - * - * @return true if the metadata file is present - */ - public boolean isMetaFilePresent() { - return metaFile.exists(); - } - - /** - * Update the content of the metadata map. - * If the current collection data is present, replace it. - * Otherwise insert new. - * @param metadataMap : Set of XmlMetaData objects. - * @param index : The path to index location. - */ - public void updateMetadataMap(ConcurrentHashMap<String, XmlMetadata> metadataMap, String index) { - this.indexMap = metadataMap; - this.index = index; - - } - - /** - * Method to get the set of xml metadata for a given collection - * - * @return : Map containing the set of XmlMetadata objects.\ - */ - public ConcurrentHashMap<String, XmlMetadata> getMetadata() { - return this.indexMap; - } - - /** - * Read the metadata file and create an in-memory map containing collection paths and xml files. - */ - public void readMetadataFile() { - try { - JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); - Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); - XmlMetadataCollection indexes = (XmlMetadataCollection) jaxbUnmarshaller.unmarshal(metaFile); - - this.collection = indexes.getCollection(); - this.index = indexes.getIndexLocation(); - - for (XmlMetadata metadata : indexes.getMetadataList()) { - this.indexMap.put(index, metadata); - } - } catch (JAXBException e) { - if (LOGGER.isTraceEnabled()) { - LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e); - } - } - } - - /** - * Write the content of the ConcurrentHashMap to the xml metadata file. - */ - public void writeMetadataToFile() { - XmlMetadataCollection collection = new XmlMetadataCollection(); - List<XmlMetadata> metadataList = new ArrayList<>(); - - for (Map.Entry<String, XmlMetadata> entry : this.indexMap.entrySet()) { - metadataList.add(entry.getValue()); - } - - collection.setMetadataList(metadataList); - collection.setCollection(this.collection); - collection.setIndexLocation(this.index); - try{ - FileOutputStream fileOutputStream = new FileOutputStream(this.metaFile); - JAXBContext jaxbContext = JAXBContext.newInstance(VXQueryIndex.class); - Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); - jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); - jaxbMarshaller.marshal(collection, fileOutputStream); - - if (LOGGER.isDebugEnabled()) { - LOGGER.log(Level.DEBUG, "Writing metadata file completed successfully!"); - } - } catch (JAXBException | FileNotFoundException e) { - if (LOGGER.isTraceEnabled()) { - LOGGER.log(Level.ERROR, "Could not read the XML file due to " + e); - } - } - - - } - - - /** - * Generate MD5 checksum string for a given file. - * - * @param file : File which the checksum should be generated. - * @return : Checksum String - * @throws IOException : The file is not available - */ - public String generateMD5(File file) throws IOException { - try { - MessageDigest md = MessageDigest.getInstance("MD5"); - md.update(Files.readAllBytes(file.toPath())); - byte[] md5 = md.digest(); - return DatatypeConverter.printHexBinary(md5); - } catch (NoSuchAlgorithmException e) { - if (LOGGER.isTraceEnabled()) { - LOGGER.log(Level.ERROR, "No Such Algorithm Error " + e.getMessage()); - } - return null; - } - } - - /** - * Delete the existing Metadata file. - * - * @return True if deleted, false otherwise. - */ - public boolean deleteMetaDataFile() { - try { - Files.delete(Paths.get(metaFile.getCanonicalPath())); - if (LOGGER.isDebugEnabled()){ - LOGGER.log(Level.DEBUG, "Metadata file deleted!"); - } - return true; - } catch (IOException e) { - if (LOGGER.isTraceEnabled()){ - LOGGER.log(Level.ERROR, "Metadata file could not be deleted!"); - } - return false; - } - } - - /** - * Get the collection for a given index location. - * @return collection folder for a given index. - */ - public String getCollection() { - return this.collection; - } - - /** - * Set the entry for given index and collection. - * @param collection : path to corresponding collection - */ - public void setCollection(String collection) { - this.collection = collection; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java deleted file mode 100644 index fa92b2f..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/VXQueryIndex.java +++ /dev/null @@ -1,42 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index.updateIndex; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.util.List; - -/** - * Class for storing metadata information for vxquery index. - */ -@XmlAccessorType(XmlAccessType.PROPERTY) -@XmlRootElement(name = "indexes") -public class VXQueryIndex { - - private List<XmlMetadataCollection> indexes; - - public List<XmlMetadataCollection> getIndex() { - return indexes; - } - - @XmlElement(name = "index", type = XmlMetadataCollection.class) - public void setIndex(List<XmlMetadataCollection> index) { - this.indexes = index; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java deleted file mode 100644 index b6da6d9..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadata.java +++ /dev/null @@ -1,72 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index.updateIndex; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; - -/** - * Class to store metadata related to an XML file. - * This contains - * - Path to the xml file - * - MD5 Checksum String - * - File name - * - Last modified date - */ -@XmlRootElement(name = "file") -@XmlAccessorType(XmlAccessType.FIELD) -public class XmlMetadata implements Serializable { - - private String path; - private String md5; - private String fileName; - private String lastModified; - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - - public String getMd5() { - return md5; - } - - public void setMd5(String md5) { - this.md5 = md5; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public String getLastModified() { - return lastModified; - } - - public void setLastModified(String lastModified) { - this.lastModified = lastModified; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java deleted file mode 100644 index 1f5c3e9..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/XmlMetadataCollection.java +++ /dev/null @@ -1,66 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.vxquery.runtime.functions.index.updateIndex; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.util.List; - -/** - * Class for holding the collection information and the list of XML metadata related to the xml files in the - * collection. - */ -@XmlAccessorType(XmlAccessType.FIELD) -@XmlRootElement(name = "index") -public class XmlMetadataCollection { - - @XmlAttribute(name = "location") - private String indexLocation; - - @XmlAttribute(name = "collection") - private String collection; - - @XmlElement(name = "file", type = XmlMetadata.class) - private List<XmlMetadata> metadataList; - - public List<XmlMetadata> getMetadataList() { - return metadataList; - } - - public void setMetadataList(List<XmlMetadata> metadataList) { - this.metadataList = metadataList; - } - - public String getIndexLocation() { - return indexLocation; - } - - public void setIndexLocation(String indexLocation) { - this.indexLocation = indexLocation; - } - - public String getCollection() { - return collection; - } - - public void setCollection(String collection) { - this.collection = collection; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java index 665c812..85ef4ca 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/json/JnDocScalarEvaluatorFactory.java @@ -64,7 +64,7 @@ public class JnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal tvp.getValue(stringp); try { IParser parser = new JSONParser(); - FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); + FunctionHelper.readInDocFromPointable(stringp, abvs, parser); } catch (IOException e) { throw new SystemException(ErrorCode.FODC0002, e); } http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java index db908f6..15fd624 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocAvailableScalarEvaluatorFactory.java @@ -78,7 +78,7 @@ public class FnDocAvailableScalarEvaluatorFactory extends AbstractTaggedValueArg tvp.getValue(stringp); try { IParser parser = new XMLParser(false, nodeIdProvider, nodeId); - FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); + FunctionHelper.readInDocFromPointable(stringp, abvs, parser); XDMConstants.setTrue(result); } catch (Exception e) { XDMConstants.setFalse(result); http://git-wip-us.apache.org/repos/asf/vxquery/blob/4a38f670/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java index 5f08a8e..2fd1755 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/node/FnDocScalarEvaluatorFactory.java @@ -79,7 +79,7 @@ public class FnDocScalarEvaluatorFactory extends AbstractTaggedValueArgumentScal try { // Only one document should be parsed so its ok to have a unique parser. IParser parser = new XMLParser(false, nodeIdProvider, nodeId); - FunctionHelper.readInDocFromPointable(stringp, bbis, di, abvs, parser); + FunctionHelper.readInDocFromPointable(stringp, abvs, parser); } catch (Exception e) { throw new SystemException(ErrorCode.SYSE0001, e); }
