Repository: vxquery Updated Branches: refs/heads/master f463f01d0 -> 233a43064
Preparing the site for deployment after recent changes. - Fix javadocs comments for java 8 - Clean up the maven apt files Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/233a4306 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/233a4306 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/233a4306 Branch: refs/heads/master Commit: 233a43064ec8e7dcc114630b559801233b7f0610 Parents: f463f01 Author: Preston Carman <prest...@apache.org> Authored: Mon Sep 19 16:03:31 2016 -0700 Committer: Christina Pavlopoulou <cpavl...@ucr.edu> Committed: Mon Oct 10 13:01:32 2016 -0700 ---------------------------------------------------------------------- src/site/apt/development_xml_document.apt | 551 +++++++++---------- src/site/apt/user_get_started_with_jsoniq.apt | 8 +- src/site/apt/user_indexing.apt | 242 ++++---- src/site/site.xml | 3 + .../rewriter/rules/AbstractCollectionRule.java | 2 + .../accessors/atomic/XSDateTimePointable.java | 76 +-- .../org/apache/vxquery/hdfs2/HDFSFunctions.java | 54 +- .../functions/index/CaseSensitiveAnalyzer.java | 5 +- .../functions/index/IndexConstructorUtil.java | 3 - .../indexCentralizer/IndexCentralizerUtil.java | 7 +- .../index/updateIndex/IndexUpdater.java | 53 +- .../index/updateIndex/MetaFileUtil.java | 2 +- .../runtime/functions/util/FunctionHelper.java | 4 +- 13 files changed, 534 insertions(+), 476 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/src/site/apt/development_xml_document.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/development_xml_document.apt b/src/site/apt/development_xml_document.apt index f618704..1ad9f39 100644 --- a/src/site/apt/development_xml_document.apt +++ b/src/site/apt/development_xml_document.apt @@ -13,6 +13,8 @@ ~~ See the License for the specific language governing permissions and ~~ limitations under the License. +XML Data Model Example + * XML Mapping The XML mapping to Hyracks pointables is fairly straight forward. The following example @@ -21,7 +23,7 @@ ** Example XML File The example XML file comes from W3School XQuery tutorial. - + ---------------------------------------- <?xml version="1.0" encoding="ISO-8859-1"?> <!-- Edited by XMLSpyà --> @@ -33,14 +35,14 @@ <year>2005</year> <price>30.00</price> </book> - + <book category="CHILDREN"> <title lang="en">Harry Potter</title> <author>J K. Rowling</author> <year>2005</year> <price>29.99</price> </book> - + <book category="WEB"> <title lang="en">XQuery Kick Start</title> <author>James McGovern</author> @@ -51,7 +53,7 @@ <year>2003</year> <price>49.99</price> </book> - + <book category="WEB"> <title lang="en">Learning XML</title> <author>Erik T. Ray</author> @@ -61,13 +63,13 @@ </bookstore> ---------------------------------------- - + ** Example Hyracks Mapping The mapping is explained through using some short hand for the above example XML file. Realize the direct bytes will not be explained although the pointable names are used for each piece of information. - + ---------------------------------------- NodeTree { DocumentNode {bookstore} @@ -133,32 +135,29 @@ NodeTree { ---------------------------------------- Following is a detailed example of the byte representation of an XML document. - - -========================================================================= -XML Data Model Example +========================================================================= *Byte Array Break Down - Every XML document in VXQuery is stored in memory as one continuous array of + Every XML document in VXQuery is stored in memory as one continuous array of bytes. Pointables are used to refer to these bytes in the memory. - This document covers VXQuery's representation of all the different types of - elements of an XML document. As a result, we use a lots of pointables - (same and different) through out the document. To simplify explanations, - each pointable is explicitly assigned a NodeID only on this web page. - Refer to the following link for details on the various pointables used: + This document covers VXQuery's representation of all the different types of + elements of an XML document. As a result, we use a lots of pointables + (same and different) through out the document. To simplify explanations, + each pointable is explicitly assigned a NodeID only on this web page. + Refer to the following link for details on the various pointables used: {{{http://vxquery.apache.org/development_xml_node_details.html} XML Node Details }}. **XML Document - - We use the following XML document as an example to explain VXQuery's node - types. The different node types are Node Tree Pointable (NTP), - Document Node Pointable (DNP), Element Node Pointable (ENP), + + We use the following XML document as an example to explain VXQuery's node + types. The different node types are Node Tree Pointable (NTP), + Document Node Pointable (DNP), Element Node Pointable (ENP), Attribute Node Pointable (ANP), Text Node Pointable (TNP), - Comment Node Pointable (CNP) and Processing Instruction Node Pointable (PINP). - + Comment Node Pointable (CNP) and Processing Instruction Node Pointable (PINP). + --- <?xml version="1.0"?> <catalog xmlns:ex="http://example.org/"; > @@ -171,57 +170,57 @@ XML Data Model Example --- ** {Bytes} - + Following are the bytes for the XML document above. Elements in VXQuery are - accessed using Tagged Value Pointables. Similarly, the XML document is also - accessed using a Tagged Value Pointable. The first byte is represents the - value tag. It indicates the type of the bytes that follow. - + accessed using Tagged Value Pointables. Similarly, the XML document is also + accessed using a Tagged Value Pointable. The first byte is represents the + value tag. It indicates the type of the bytes that follow. + *** <<<107, 3, 0, 0, 0, 0, 0, 0, 0, -109, 0, 0, 0, 7, 0, 0, 0, 6, 0, 0, 0, 19, 0, 0, 0, 44, 0, 0, 0, 54, 0, 0, 0, 62, 0, 0, 0, 72, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, 99, 97, 116, 97, 108, 111, 103, 0, 0, 0, 1, 0, 19, 104, 116, 116, 112, 58, 47, 47, 101, 120, 97, 109, 112, 108, 101, 46, 111, 114, 103, 47, 0, 0, 0, 2, 0, 4, 98, 111, 111, 107, 0, 0, 0, 3, 0, 2, 101, 120, 0, 0, 0, 4, 0, 4, 105, 115, 98, 110, 0, 0, 0, 5, 0, 5, 116, 105, 116, 108, 101, 0, 0, 0, 6, 101, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 0, -42, 0, 0, 0, -34, 104, 0, 0, 0, 2, 0, 3, 10, 32, 32, 102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> ========================================================================= ** {Node Tree} - + <<<107>>> The first byte as described above is the value tag for Node Tree Pointable. - + The rest of the bytes represent a Node Tree Pointable. Refer to this link to view the {{Bytes}} for the Node Tree Pointable(NTP). XML Documents in VXQuery are wrapped in Node Tree Pointables. As a side note, every result produced as an output of a function is also wrapped in a NTP. - + Following are the bytes and contents of the Node Tree Pointable for this XML document. - + <<< 3 >>> Header byte (One byte) that uses the lowest three bit to denote if ** bit <Node Id> exists: <Yes> - + ** bit <Dictionary> exists: <Yes> - + ** bit <Header Type> exists: <No> - + <<< 0, 0, 0, 0 >>> These 4 bytes represent the <Node Id> which has value <<0>> - Following are the byte contents of the {{Dictionary}}. The byte array break + Following are the byte contents of the {{Dictionary}}. The byte array break down is explained in details further ahead. - + *** <<<0, 0, 0, -109, 0, 0, 0, 7, 0, 0, 0, 6, 0, 0, 0, 19, 0, 0, 0, 44, 0, 0, 0, 54, 0, 0, 0, 62, 0, 0, 0, 72, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, 99, 97, 116, 97, 108, 111, 103, 0, 0, 0, 1, 0, 19, 104, 116, 116, 112, 58, 47, 47, 101, 120, 97, 109, 112, 108, 101, 46, 111, 114, 103, 47, 0, 0, 0, 2, 0, 4, 98, 111, 111, 107, 0, 0, 0, 3, 0, 2, 101, 120, 0, 0, 0, 4, 0, 4, 105, 115, 98, 110, 0, 0, 0, 5, 0, 5, 116, 105, 116, 108, 101, 0, 0, 0, 6>>> - + Element Node in NTP(root node): - - In this NTP, the Element Node or the root node is a Document Node Pointable - (DNP) ({{NodeID:0}}). <<101>> is the <Value Tag> for Document Node Pointable. - Note that this root node can represent any pointable type. For example: + + In this NTP, the Element Node or the root node is a Document Node Pointable + (DNP) ({{NodeID:0}}). <<101>> is the <Value Tag> for Document Node Pointable. + Note that this root node can represent any pointable type. For example: ElementNodePointable, Attribute Node Pointable or Text Node Pointable. - - Following are the byte contents for the Document Node Pointable + + Following are the byte contents for the Document Node Pointable ({{NodeID:0}}). The byte array break down is explained further ahead. - + *** <<<0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 0, -42, 0, 0, 0, -34, 104, 0, 0, 0, 2, 0, 3, 10, 32, 32, 102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> ========================================================================= @@ -229,239 +228,239 @@ XML Data Model Example ** {Dictionary} Byte Array for the Dictionary - + *** <<<0, 0, 0, -109, 0, 0, 0, 7, 0, 0, 0, 6, 0, 0, 0, 19, 0, 0, 0, 44, 0, 0, 0, 54, 0, 0, 0, 62, 0, 0, 0, 72, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, 99, 97, 116, 97, 108, 111, 103, 0, 0, 0, 1, 0, 19, 104, 116, 116, 112, 58, 47, 47, 101, 120, 97, 109, 112, 108, 101, 46, 111, 114, 103, 47, 0, 0, 0, 2, 0, 4, 98, 111, 111, 107, 0, 0, 0, 3, 0, 2, 101, 120, 0, 0, 0, 4, 0, 4, 105, 115, 98, 110, 0, 0, 0, 5, 0, 5, 116, 105, 116, 108, 101, 0, 0, 0, 6>>> - -*** <<< 0, 0, 0, -109 >>> - - These 4 bytes represent the <Size of Dictionary> in signed integer format. + +*** <<< 0, 0, 0, -109 >>> + + These 4 bytes represent the <Size of Dictionary> in signed integer format. After conversion to unsigned integer format the value is <<147>>. - + *** <<<0, 0, 0, 7>>> These 4 bytes represent the <Number of items> in the dictionary: <<7>> -*** <<<0, 0, 0, 6, 0, 0, 0, 19, 0, 0, 0, 44, 0, 0, 0, 54, 0, 0, 0, 62, 0, 0, 0, 72, 0, 0, 0, 83>>> - - This is a list of <Offsets> for each item in the dictionary. - There are 7 offsets. Each offset is 4 bytes long. Following are +*** <<<0, 0, 0, 6, 0, 0, 0, 19, 0, 0, 0, 44, 0, 0, 0, 54, 0, 0, 0, 62, 0, 0, 0, 72, 0, 0, 0, 83>>> + + This is a list of <Offsets> for each item in the dictionary. + There are 7 offsets. Each offset is 4 bytes long. Following are the 7 offsets: <<6, 19, 44, 54, 62, 72, 83>> *** <<<0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 6>>> - - This is a sorted list of keys in alphabetical order. Each key is 4 byte long. - Each key is mapped to a string in the dictionary. The keys + + This is a sorted list of keys in alphabetical order. Each key is 4 byte long. + Each key is mapped to a string in the dictionary. The keys are the numbers <<1>> through <<6>>. - - Following are the data values in the dictionary.Each data value is a - StringPointable. Each StringPointable maps to XML document strings. -*** <<<0, 0, 0, 0, 0, 0>>> + Following are the data values in the dictionary.Each data value is a + StringPointable. Each StringPointable maps to XML document strings. - The <Size> of the string is <<0>>. The <String Value> is <<null>>. +*** <<<0, 0, 0, 0, 0, 0>>> + + The <Size> of the string is <<0>>. The <String Value> is <<null>>. The StringPointable is followed by the key which is <<0>>. - + *** <<<0, 7, 99, 97, 116, 97, 108, 111, 103, 0, 0, 0, 1>>> - - The <Size> of the string is <<7>>. The <String Value> is <<catalog>>. + + The <Size> of the string is <<7>>. The <String Value> is <<catalog>>. The StringPointable is followed by the key which is <<1>>. - -*** <<<0, 19, 104, 116, 116, 112, 58, 47, 47, 101, 120, 97, 109, 112, 108, 101, 46, 111, 114, 103, 47, 0, 0, 0, 2>>> - - The <Size> of the string is <<19>>. - The <String Value> is <<http://example.org/>>. + +*** <<<0, 19, 104, 116, 116, 112, 58, 47, 47, 101, 120, 97, 109, 112, 108, 101, 46, 111, 114, 103, 47, 0, 0, 0, 2>>> + + The <Size> of the string is <<19>>. + The <String Value> is <<http://example.org/>>. The StringPointable is followed by the key which is <<2>>. - + *** <<<0, 4, 98, 111, 111, 107, 0, 0, 0, 3>>> - - The <Size> of the string is <<4>>. The <String Value> is <<book>>. - The StringPointable is followed by the key which is <<3>>. - + + The <Size> of the string is <<4>>. The <String Value> is <<book>>. + The StringPointable is followed by the key which is <<3>>. + *** <<<0, 2, 101, 120, 0, 0, 0, 4>>> - - The <Size> of the string is <<2>>. The <String Value> is <<ex>>. + + The <Size> of the string is <<2>>. The <String Value> is <<ex>>. The StringPointable is followed by the key which is <<4>>. - + *** <<<0, 4, 105, 115, 98, 110, 0, 0, 0, 5>>> - - The <Size> of the string is <<4>>. The <String Value> is <<isbn>>. + + The <Size> of the string is <<4>>. The <String Value> is <<isbn>>. The StringPointable is followed by the key which is <<5>>. - + *** <<<0, 5, 116, 105, 116, 108, 101, 0, 0, 0, 6>>> - The <Size> of the string is <<4>>. The <String Value> is <<title>>. + The <Size> of the string is <<4>>. The <String Value> is <<title>>. The StringPointable is followed by the key which is <<6>>. -================================================================ +================================================================ ** Document Node ({NodeID:0}) - + This child is contained in the parent {{Node Tree}}. - + Byte Array for Document Node (NodeID:0) - + *** <<<101, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 0, -42, 0, 0, 0, -34, 104, 0, 0, 0, 2, 0, 3, 10, 32, 32, 102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> - + <<<101>>> is the value tag for the Document Node Pointable. - + Following are the bytes and contents of the Document Node Pointable. - - <<< 0, 0, 0, 0 >>> These 4 bytes represent the <Node Id> + + <<< 0, 0, 0, 0 >>> These 4 bytes represent the <Node Id> which has value <<0>> - - Every Document Node Pointable contains a Sequence Pointable. This is analogous - to a collection of items(data). In VXQuery, the items(data) - in the Sequence Pointable are preceded by the number of items in the sequence + + Every Document Node Pointable contains a Sequence Pointable. This is analogous + to a collection of items(data). In VXQuery, the items(data) + in the Sequence Pointable are preceded by the number of items in the sequence and item size. - + Sequence Content: - - <<<0, 0, 0, 1>>> These 4 bytes represents the <Number of Items> - in the sequence which is <<1>> - - <<<0, 0, 1, 0>>> These 4 bytes represents + + <<<0, 0, 0, 1>>> These 4 bytes represents the <Number of Items> + in the sequence which is <<1>> + + <<<0, 0, 1, 0>>> These 4 bytes represents the <Size of the item> which is <<257>> - - <Data in the Sequence>: Here the (item)data in the sequence is an - Element Node Pointable ({{NodeID:1}}). Note that the data can represent any + + <Data in the Sequence>: Here the (item)data in the sequence is an + Element Node Pointable ({{NodeID:1}}). Note that the data can represent any type of pointable or element. - + *** <<<102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 0, -42, 0, 0, 0, -34, 104, 0, 0, 0, 2, 0, 3, 10, 32, 32, 102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> - - -================================================================ + + +================================================================ ** Element Node ({NodeID:1}) This child is contained in the parent Document Node ({{NodeID:0}}). - + Byte Array for Element Node NodeID:1 - + *** <<<102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 0, -42, 0, 0, 0, -34, 104, 0, 0, 0, 2, 0, 3, 10, 32, 32, 102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> - + <<<102>>> is the value tag for Element Node Pointable. - + Following are the bytes and contents of the Element Node Pointable. - + <<< 4 >>> Header byte (One byte) that uses the lowest three bits to denote if ** bit <Namespace Chunk> exists: <No> - + ** bit <Attribute Chunk> exists: <No> - + ** bit <Children Chunk> exists: <Yes> - + <<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>>> This is a <Name Pointer> which is an - array of integers(4 bytes) of size <<3>> - + array of integers(4 bytes) of size <<3>> + <<<0, 0, 0, 1>>> This is the <Local Node Id> which uses 4 bytes. - - Children Chunk is a Sequence Pointable. This is analogous to a collection - of items(data). In VXQuery, the items(data) - in the Sequence Pointable are preceded by the number of items - in the sequence and item size. + + Children Chunk is a Sequence Pointable. This is analogous to a collection + of items(data). In VXQuery, the items(data) + in the Sequence Pointable are preceded by the number of items + in the sequence and item size. Sequence Content childrenChunk: - <<<0, 0, 0, 3>>> <Number of Items> in the + <<<0, 0, 0, 3>>> <Number of Items> in the SequencePointable <children chunk> is <<3>> - + <<<0, 0, 0, 10>>> <Offset> of the first item is <<10>> - + <<<0, 0, 0, -42>>> <Offset> of the second item is <<214>> - + <<<0, 0, 0, -34>>> <Offset> of the third item is <<222>> - - <Data in the Sequence>: Here the items(data) in the sequence are Text + + <Data in the Sequence>: Here the items(data) in the sequence are Text Node Pointables ({{NodeID:2}}), ({{NodeID:13}}) and Element Node Pointable - ({{NodeID:3}}). Note that the data can represent any type of pointable + ({{NodeID:3}}). Note that the data can represent any type of pointable or element. - + *** <<<104, 0, 0, 0, 2, 0, 3, 10, 32, 32, 102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> ================================================================ -** Text Node ({NodeID:2}) - +** Text Node ({NodeID:2}) + This child is contained in the parent Element Node ({{NodeID:1}}). - + Byte Array for Text Node NodeID:2 - + *** <<<104, 0, 0, 0, 2, 0, 3, 10, 32, 32>>> <<<104>>> is the value tag for the Text Node Pointable. - - Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 2>>> This is the <Node Id> that uses 4 bytes and has value <<2>> - - <<<0, 3, 10, 32, 32>>> This is the <UTF8String> which has a size <<3>> + Following are the bytes and contents of the Text Node Pointable. + + <<<0, 0, 0, 2>>> This is the <Node Id> that uses 4 bytes and has value <<2>> + + <<<0, 3, 10, 32, 32>>> This is the <UTF8String> which has a size <<3>> and value represents a <<new line>> and 2 <<spaces>> ================================================================ ** Element Node ({NodeID:3}) - + This child is contained in the parent Element Node ({{NodeID:1}}). - + Byte Array for Element Node NodeID:3 - + *** <<<102, 6, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 30, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57, 0, 0, 0, 7, 0, 0, 0, 12, 0, 0, 0, 29, 0, 0, 0, 41, 0, 0, 0, 81, 0, 0, 0, 93, 0, 0, 0, 106, 0, 0, 0, 116, 104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32>>> - + <<<102>>> is the value tag for the Element Node Pointable. - + Following are the bytes and contents of the Element Node Pointable. - + <<< 6 >>> Header byte (One byte) that uses the three lowest bit to denote if ** bit <Namespace Chunk> exists: <No> - + ** bit <Attribute Chunk> exists: <Yes> - + ** bit <Children Chunk> exists: <Yes> - - <<<0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3>>> This is a <Name Pointer> which is an - array of integers(4 bytes) of size <<3>> - + + <<<0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3>>> This is a <Name Pointer> which is an + array of integers(4 bytes) of size <<3>> + <<<0, 0, 0, 3>>> This is the <Local Node Id> which uses 4 bytes. - + Attribute Chunk is a Sequence Pointable. - + Sequence Content attributeChunk: - - <<<0, 0, 0, 1>>> <Number of Items> in the + + <<<0, 0, 0, 1>>> <Number of Items> in the SequencePointable <attribute chunk> is <<1>> - + <<<0, 0, 0, 30>>> <Size> of the first item is <<30>> - - <Data in the Sequence>: Here the item(data) in the sequence is an - Attribute Node Pointable ({{NodeID:4}}). Note that the data + + <Data in the Sequence>: Here the item(data) in the sequence is an + Attribute Node Pointable ({{NodeID:4}}). Note that the data can represent any type of pointable or element. - + *** <<< 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57>>> - Children Chunk is a Sequence Pointable. This is analogous to a collection of - items(data). In VXQuery, the items(data) - in the Sequence Pointable are preceded by the number of - items in the sequence and item size. - + Children Chunk is a Sequence Pointable. This is analogous to a collection of + items(data). In VXQuery, the items(data) + in the Sequence Pointable are preceded by the number of + items in the sequence and item size. + Sequence Content childrenChunk: <<<0, 0, 0, 7>>> <Number of Items> in the SequencePointable <children chunk> is <<3>> - + <<<0, 0, 0, 12>>> <Offset> of the first item is <<12>> - + <<<0, 0, 0, 29>>> <Offset> of the second item is <<12>> - + <<<0, 0, 0, 41>>> <Offset> of the third item is <<41>> - + <<<0, 0, 0, 81>>> <Offset> of the fourth item is <<81>> <<<0, 0, 0, 93>>> <Offset> of the fifth item is <<93>> @@ -469,12 +468,12 @@ XML Data Model Example <<<0, 0, 0, 106>>> <Offset> of the sixth item is <<106>> <<<0, 0, 0, 116>>> <Offset> of the seventh item is <<116>> - + <Data in the Sequence>: Here the items(data) in the sequence are Text Node - ({{NodeID:5}}), ({{NodeID:7}}), ({{NodeID:10}}), ({{NodeID:12}}), + ({{NodeID:5}}), ({{NodeID:7}}), ({{NodeID:10}}), ({{NodeID:12}}), Element Node ({{NodeID:8}}), Comment Node ({{NodeID:6}}) and - Processing Instruction Node ({{NodeID:11}}). - + Processing Instruction Node ({{NodeID:11}}). + *** <<<104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32, 105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116, 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32, 102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104, 104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32, 106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101, 0, 0, 104, 0, 0, 0, 12, 0, 3, 10, 32, 32, 104, 0, 0, 0, 13, 0, 1, 10>>> @@ -484,22 +483,22 @@ XML Data Model Example ** Attribute Node ({NodeID:3}) This child is contained in the parent Element Node ({{NodeID:3}}). - + Byte Array for Attribute Node NodeID:3 - + *** <<<103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57>>> - + <<<103>>> is the value tag for the Attribute Node Pointable. - + Following are the bytes and contents of the Attribute Node Pointable. - - <<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,>>> This is a <Name Pointer> - which is an array of integers(4 bytes) of size <<3>>. It consists of + + <<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,>>> This is a <Name Pointer> + which is an array of integers(4 bytes) of size <<3>>. It consists of <PrefixCode>, <NamespaceCode>, <LocalCode>. <<<0, 0, 0, 4>>> This is the <NodeId> which has a value of <<4>>. - - <<<14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57 >>> This is a + + <<<14, 0, 10, 48, 56, 49, 50, 52, 49, 54, 49, 51, 57 >>> This is a string of length <<10>> and the length of the string is <<0812416139>> @@ -510,18 +509,18 @@ XML Data Model Example This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Text Node NodeID:5 - + *** <<<104, 0, 0, 0, 5, 0, 5, 10, 32, 32, 32, 32>>> - + <<<104>>> is the value tag for the Text Node Pointable. - + Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 5>>> This is the <Node Id> that uses 4 bytes and has value <<2>> - - <<< 0, 5, 10, 32, 32, 32, 32>>> This is the <UTF8String> which has + <<<0, 0, 0, 5>>> This is the <Node Id> that uses 4 bytes and has value <<2>> + + <<< 0, 5, 10, 32, 32, 32, 32>>> This is the <UTF8String> which has a size <<5>> and value represents a <<new line>> and 4 <<spaces>> @@ -530,42 +529,42 @@ XML Data Model Example ** Comment Node Pointable ({NodeID:6}) - This child is contained in the parent Element Node ({{NodeID:3}}) + This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Comment Node NodeID:6 - + *** <<<105, 0, 0, 0, 6, 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116>>> - + <<<105>>> is the value tag for the Comment Node Pointable. - + Following are the bytes and contents of the Comment Node Pointable. - - <<<0, 0, 0, 6>>> This is the <Node Id> that uses 4 bytes and has value <<6>> - - <<< 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116>>> + + <<<0, 0, 0, 6>>> This is the <Node Id> that uses 4 bytes and has value <<6>> + + <<< 0, 10, 116, 111, 112, 32, 115, 101, 99, 114, 101, 116>>> This is the <UTF8String> which has a size <<10>> and value <<top secret>> - - + + ================================================================ - - + + ** Text Node Pointable ({NodeID:7}) - - This child is contained in the parent Element Node ({{NodeID:3}}) + + This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Text Node NodeID:7 - -*** <<< 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32>>> - + +*** <<< 104, 0, 0, 0, 7, 0, 5, 10, 32, 32, 32, 32>>> + <<<104>>> is the value tag for the Text Node Pointable. - + Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 7>>> This is the <Node Id> that uses 4 bytes and has value <<7>> - - <<< 0, 5, 10, 32, 32, 32, 32>>> This is the <UTF8String> + <<<0, 0, 0, 7>>> This is the <Node Id> that uses 4 bytes and has value <<7>> + + <<< 0, 5, 10, 32, 32, 32, 32>>> This is the <UTF8String> which has a size <<5>> and value represents a <<new line>> and 4 <<spaces>> @@ -574,63 +573,63 @@ XML Data Model Example ** Element Node ({NodeID:8}) - This child is contained in the parent Element Node ({{NodeID:3}}) + This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Element Node NodeID:8 - + *** <<<102, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 14, 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104>>> - + <<<102>>> is the value tag for the Element Node Pointable. - + Following are the bytes and contents of the Element Node Pointable. - + <<< 4 >>> Header byte (One byte) that uses the lowest three bits to denote if ** bit <Namespace Chunk> exists: <No> - + ** bit <Attribute Chunk> exists: <No> - + ** bit <Children Chunk> exists: <Yes> - - <<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6>>> This is a <Name Pointer> - which is an array of integers(4 bytes) of size <<3>> - - <<<0, 0, 0, 8>>> This is the <Node Id> that uses 4 bytes and has value <<8>> - + + <<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6>>> This is a <Name Pointer> + which is an array of integers(4 bytes) of size <<3>> + + <<<0, 0, 0, 8>>> This is the <Node Id> that uses 4 bytes and has value <<8>> + Following is a SequencePointable. - + Sequence Content childrenChunk - <<<0, 0, 0, 1>>> <Number of Items> in the + <<<0, 0, 0, 1>>> <Number of Items> in the SequencePointable <children chunk> is <<1>> - + <<<0, 0, 0, 14>>> <Offset> of the first item is <<14>> - - <Data in the Sequence>: Here the (item)data in the sequence is a + + <Data in the Sequence>: Here the (item)data in the sequence is a Text Node Pointable ({{NodeID:9}}). - + *** <<< 104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104>>> - - + + ================================================================ - - + + ** Text Node Pointable ({NodeID:9}) This child is contained in the parent Element Node ({{NodeID:8}}). - + Byte Array for Text NodeID:9 - + *** <<<104, 0, 0, 0, 9, 0, 7, 77, 97, 99, 98, 101, 116, 104>>> - + <<<104>>> is the value tag for the Text Node Pointable. - + Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 9>>> This is the <Node Id> that uses 4 bytes and has value <<9>> - - <<<0, 7, 77, 97, 99, 98, 101, 116, 104>>> This is the <UTF8String> + <<<0, 0, 0, 9>>> This is the <Node Id> that uses 4 bytes and has value <<9>> + + <<<0, 7, 77, 97, 99, 98, 101, 116, 104>>> This is the <UTF8String> which has a size <<7>> and value <<Macbeth>> @@ -638,22 +637,22 @@ XML Data Model Example ** Text Node Pointable ({NodeID:10}) - - This child is contained in the parent Element Node ({{NodeID:3}}) + + This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Text Node NodeID:10 - + *** <<<104, 0, 0, 0, 10, 0, 5, 10, 32, 32, 32, 32>>> - + <<<104>>> and the rest of the bytes represent a Text Node Pointable. - + Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 10>>> This is the <Node Id> that uses 4 bytes and has value <<2>> - - <<<0, 5, 10, 32, 32, 32, 32>>> This is the <UTF8String> which has a size <<5>> + <<<0, 0, 0, 10>>> This is the <Node Id> that uses 4 bytes and has value <<2>> + + <<<0, 5, 10, 32, 32, 32, 32>>> This is the <UTF8String> which has a size <<5>> and value represents a <<new line>> and 4 <<spaces>> @@ -662,26 +661,26 @@ XML Data Model Example ** Processing Instruction Node ({NodeID:11}) - This child is contained in the parent Element Node ({{NodeID:3}}) + This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Processing Instruction Node NodeID:11 - + *** <<<106, 0, 0, 0, 11, 0, 4, 104, 105, 100, 101>>> - + Note that this is a Tagged Value Pointable in which the value tag is <<106>> and the rest of the bytes represent a Processing Instruction Node Pointable. - - Following are the bytes and contents of the Processing + + Following are the bytes and contents of the Processing Instruction Node Pointable. <<<0, 0, 0, 11>>> This is the <Node Id> that uses 4 bytes and has value <<11>> - - <<<0, 4, 104, 105, 100, 101>>> This is the <UTF8String> + + <<<0, 4, 104, 105, 100, 101>>> This is the <UTF8String> which has a size <<4>> and value <<hide>> - - <<<0, 0>>> This is also a string representing content. - It is a <<null string>>. + + <<<0, 0>>> This is also a string representing content. + It is a <<null string>>. ================================================================ @@ -689,21 +688,21 @@ XML Data Model Example ** Text Node Pointable ({NodeID:12}) - This child is contained in the parent Element Node ({{NodeID:3}}) + This child is contained in the parent Element Node ({{NodeID:3}}) childrenChunk. - + Byte Array for Text Node NodeID:12 - + *** <<<104, 0, 0, 0, 12, 0, 3, 10, 32, 32>>> - + Note that this is a Tagged Value Pointable in which the value tag is <<104>> and the rest of the bytes represent a Text Node Pointable. - + Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 12>>> This is the <Node Id> that uses 4 bytes and has value <<12>> + <<<0, 0, 0, 12>>> This is the <Node Id> that uses 4 bytes and has value <<12>> - <<<0, 3, 10, 32, 32>>> This is the <UTF8String> which has a size <<3>> + <<<0, 3, 10, 32, 32>>> This is the <UTF8String> which has a size <<3>> and value represents a <<new line>> and 2 <<spaces>>. @@ -713,18 +712,18 @@ XML Data Model Example ** Text Node Pointable ({NodeID:13}) This child is contained in the parent Element Node ({{NodeID:1}}). - + Byte Array for Text Node NodeID:13 - + *** <<<104, 0, 0, 0, 13, 0, 1, 10>>> <<<104>>> is the value tag for the Text Node Pointable. - + Following are the bytes and contents of the Text Node Pointable. - <<<0, 0, 0, 13>>> This is the <Node Id> that uses 4 bytes and has value <<13>> - - <<<0, 1, 10>>> This is the <UTF8String> which has a size <<1>> + <<<0, 0, 0, 13>>> This is the <Node Id> that uses 4 bytes and has value <<13>> + + <<<0, 1, 10>>> This is the <UTF8String> which has a size <<1>> and value represents a <<new line>>. http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/src/site/apt/user_get_started_with_jsoniq.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user_get_started_with_jsoniq.apt b/src/site/apt/user_get_started_with_jsoniq.apt index 05fc056..1ff5612 100644 --- a/src/site/apt/user_get_started_with_jsoniq.apt +++ b/src/site/apt/user_get_started_with_jsoniq.apt @@ -22,7 +22,7 @@ JSONiq extension to XQuery This example uses the XML file (books.xml) from {{{http://www.w3schools.com/xsl/xquery_intro.asp}W3Schools XQuery Tutorial}} - [[i]] Query: + XML conversion to JSON ---------------------------------------- [ @@ -38,7 +38,7 @@ JSONiq extension to XQuery ] ---------------------------------------- - [[i]] Result: + Result: ---------------------------------------- [ @@ -79,7 +79,7 @@ JSONiq extension to XQuery ] ---------------------------------------- - [[i]] The following query uses the above result (given in a file books.json): + Book ordered titles whose price is greater than 30. ---------------------------------------- let $books := jn:json-doc("books.json") @@ -89,7 +89,7 @@ order by $x("title") return $x("title") ---------------------------------------- - [[i]] Result + Result ---------------------------------------- Learning XML http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/src/site/apt/user_indexing.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user_indexing.apt b/src/site/apt/user_indexing.apt index b572231..ce7fafa 100644 --- a/src/site/apt/user_indexing.apt +++ b/src/site/apt/user_indexing.apt @@ -15,188 +15,244 @@ How to use Indexing Features in VXQuery. -In VXQuery, all the indexes are created in user specified directory. In order to use indexing, -you will need to set this directory in your cluster configuration file. + In VXQuery, all the indexes are created in user specified directory. In order to use indexing, + you will need to set this directory in your cluster configuration file. -*** Configuring VXQuery to use indexing functions. -Add the following line to your cluster configuration (e.g. cluster.xml) +* Configuring VXQuery to use indexing functions. + + Add the following line to your cluster configuration (e.g. cluster.xml) -------- - <index_directory><path_to_index_directory></index_directory> +<index_directory><path_to_index_directory></index_directory> -------- -(You should create this index_directory) + (You should create this index_directory) + +* Using indexing queries. + + VXQuery offers following indexing functionality. -** Using indexing queries. + [[a]] Create an index for collection. + [[b]] Use the index in executing a query. + [[c]] Update the index. + [[d]] Delete the index. + [[e]] View existing indexes. -VXQuery offers following indexing functionality. +*Scenario I - When collection is a single directory. -[[a]] Create an index for collection. -[[b]] Use the index in executing a query. -[[c]] Update the index. -[[d]] Delete the index. -[[e]] View existing indexes. + In this scenario, all the XML files are stored in a single directory. (There can be sub directories) -*1. Scenario I - When collection is a single directory. -In this scenario, all the XML files are stored in a single directory. (There can be sub directories) +** Creating an index for collection -*** Creating an index for collection -If I need to create index for xml collection stored in <path_1>/collection1, + If I need to create index for xml collection stored in <path_to_collection_1>, + + Query structure: -Query structure: -------- -build-index-on-collection("collection") +build-index-on-collection("<path_to_collection_1>") -------- -You can see the index has created in a new sub-directory in the index_directory specified in local.xml + You can see the index has created in a new sub-directory in the index_directory specified in local.xml + + Example: -Example: -------- -build-index-on-collection("<path_1>/collection1") +build-index-on-collection("<path_to_collection_1>") -------- -This function takes the collection path as an argument. -*** Using index in query. -If we need to use the index and execute a query, use the following structure. + This function takes the collection path as an argument. + +** Using index in query. + + If we need to use the index and execute a query, use the following structure. ------ for $r in collection-from-index("<path1>/collection1", "/dataCollection/data")/data where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744 return $r ------ -Here the index access function is, + + Here the index access function is, ------ collection-from-index ------ -which takes two arguments, collection folder and the path element. + which takes two arguments, collection folder and the path element. -Result + Result: ------ -<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data> +<data> + <date>2001-01-01T00:00:00.000</date> + <dataType>AWND</dataType> + <station>GHCND:US000000001</station> + <value>1000</value> + <attributes> + <attribute/> + <attribute/> + <attribute>a</attribute> + </attributes> +</data> ------ -*** Updating the index. -A collection can be modified or changed by following ways. -[[1]] Inserting new XML files. -[[2]] Deleting files. -[[3]] Add/ remove or modify the content of XML files. +** Updating the index. + + A collection can be modified or changed by following ways. + [[1]] Inserting new XML files. + [[2]] Deleting files. + [[3]] Add/ remove or modify the content of XML files. -In this type of situation, the index corresponding to the modified collection must also be modified. To achieve this -the update-index function can be used. + In this type of situation, the index corresponding to the modified collection must also be modified. + To achieve this the update-index function can be used. + + Query structure: -Query structure: -------- -update-index("<path_to_collection>") +update-index("<path_to_collection_1>") -------- -Example: + Example: + ------- -update-index("<path_1>/collection1") +update-index("<path_to_collection_1>") ------- -This function takes the collection which was modified. + This function takes the collection which was modified. + +** Deleting the index. -*** Deleting the index. -If we want to delete the entire index created for a collection, the delete-index function can be used. -This function also takes the collection path of which the index is needed to be deleted. + If we want to delete the entire index created for a collection, the delete-index function can be used. + This function also takes the collection path of which the index is needed to be deleted. + + Query structure: -Query structure: -------- -delete-index("<path_to_collection>") +delete-index("<path_to_collection_1>") -------- -Example: + Example: + ------- -delete-index("<path_11>/collection1") +delete-index("<path_to_collection_1>") ------- -*2. Scenario II - When the collection is distributed. -In this scenario, the collection is distributed among several directories. We can distribute the queries among -partitions. +*Scenario II - When the collection is distributed. -*** Creating indexes for collections. + In this scenario, the collection is distributed among several directories. + We can distribute the queries among partitions. + +** Creating indexes for collections. + + Query structure: -Query structure: -------- -build-index-on-collection("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>") +build-index-on-collection("<path_to_collection_1>|<path_to_collection_2>|...|<path_to_collection_n>") -------- -In here the parameter contains the list of collection partitions separated by '|' character. + In here the parameter contains the list of collection partitions separated by '|' character. + + Example: + + Consider the collection has now distributed among four directories, path_to_collection_1, path_to_collection_2, + path_to_collection_3 and path_to_collection_4. -Example: -Consider the collection has now distributed among four directories, <path_1>/collection1, <path_2>/collection2, -<path_3>/collection3 and <path_4>/collection4. + To create indexes for all of the above collections, -To create indexes for all of the above collections, ------- -build-index-on-collection("<path_1>/collection1|<path_2>/collection2|<path_3>/collection3|<path_4>/collection4") +build-index-on-collection("path_to_collection_1|path_to_collection_2|path_to_collection_3|path_to_collection_4") ------- -In this case, all indexes will be created in separate sub-directories corresponding to each partition. Also note that - this query requires each node to have four partitions available + In this case, all indexes will be created in separate sub-directories corresponding to each partition. + Also note that this query requires each node to have four partitions available. -*** Using the indexes in query. -In this case, suppose you need to run a query on indexes of two collection partitions. +** Using the indexes in query. + + In this case, suppose you need to run a query on indexes of two collection partitions. + + Example: -Example: ----- -for $r in collection-from-index("<path_1>/collection1|<path-2>collection2", "/dataCollection/data")/data +for $r in collection-from-index("<path_to_collection_1>|<path_to_collection_2>", "/dataCollection/data")/data where $r/dataType eq "AWND" and xs:decimal($r/value) gt 491.744 return $r ----- -The result will be taken from the indexes of both collection1 and collection2. + The result will be taken from the indexes of both path_to_collection_1 and path_to_collection_2. + + Result: -Result: ------ -<data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute/><attribute/><attribute>a</attribute></attributes></data> +<data> + <date>2001-01-01T00:00:00.000</date> + <dataType>AWND</dataType> + <station>GHCND:US000000001</station> + <value>1000</value> + <attributes> + <attribute/> + <attribute/> + <attribute>a</attribute> + </attributes> +</data> ------ -*** Updating the indexes. -In cases of updating the collection files stored in several partitions, we can use this function to update the -indexes of those directories. +** Updating the indexes. + + In cases of updating the collection files stored in several partitions, we can use this function to update the + indexes of those directories. + + In this case, give a '|' separated list of collection directories. + + Query structure: -In this case, give a '|' separated list of collection directories. -Query structure: -------- -update-index("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>") +update-index("<path_to_collection_1>|<path_to_collection_2>|...|<path_to_collection_n>") -------- -Example: -Suppose that we need to update the indexes in partition1 and partition4 + Example: + + Suppose that we need to update the indexes in partition1 and partition4 + -------- -update-index("<path_1>/collection1|<path_4>/collection4") +update-index("<path_to_collection_1>|<path_to_collection_4>") -------- -*** Deleting the indexes. -If we want to delete indexes of collections in several partitions, we can use this function. -Query structure: +** Deleting the indexes. + + If we want to delete indexes of collections in several partitions, we can use this function. + + Query structure: + -------- -delete-index("<partition_1_path>|<partition_2_path>|<partition_3_path>|...|<partition_n_path>") +delete-index("<path_to_collection_1>|<path_to_collection_2>|...|<path_to_collection_n>") -------- -Example: -Suppose that we need to update the indexes in collection2 and collection3 + Example: + + Suppose that we need to update the indexes in collection2 and collection3 + -------- -delete-index("<path_2>/collection2|<path_3>/collection3") +delete-index("<path_to_collection_2>|<path_to_collection_3>") -------- ** Viewing Index information. -Suppose you need to check, what are the collections have indexes created. To do this, the show-index function can be -used. This function takes no arguments and returns a sequence of collection paths, which an index is already created. -If there are no indexes created for any collection, the result will be null. -Suppose we have two collections, <path_1/collection1>, <path_2/collection2> have indexes created. -Example: + Suppose you need to check, what are the collections have indexes created. + To do this, the show-index function can be used. + This function takes no arguments and returns a sequence of collection paths, which an index is already created. + If there are no indexes created for any collection, the result will be null. + + Suppose we have two collections, <path_to_collection_1>, <path_to_collection_2> have indexes created. + + Example: + ------ show-index() ------ -Result: + Result: + ------ -<path_1/collection1> -<path_2/collection2> +<path_to_collection_1> +<path_to_collection_2> ------ http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/src/site/site.xml ---------------------------------------------------------------------- diff --git a/src/site/site.xml b/src/site/site.xml index 4e6f189..c4dbfa5 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -72,6 +72,9 @@ limitations under the License. <item name="Running the Test Suite" href="user_running_tests.html" /> + <item + name="XML Indexing" + href="user_indexing.html" /> </menu> <menu name="For Developers"> <item http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java index eff7a6e..2e8a8ac 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java @@ -60,6 +60,8 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { * * @param opRef * Logical operator + * @param functions + * Functions identifiers * @return collection name */ protected String[] getFunctionalArguments(Mutable<ILogicalOperator> opRef, Set<FunctionIdentifier> functions) { http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java index 5691d73..0f3a44a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDateTimePointable.java @@ -36,18 +36,16 @@ import org.apache.hyracks.data.std.primitive.ShortPointable; * The datetime is split up into eight sections. Due to leap year, we have decided to keep the * storage simple by saving each datetime section separately. For calculations you can access * YearMonth (months) and DayTime (milliseconds) values. - * - * @author prestoncarman */ public class XSDateTimePointable extends AbstractPointable implements IDate, ITime, ITimezone { - public final static int YEAR_OFFSET = 0; - public final static int MONTH_OFFSET = 2; - public final static int DAY_OFFSET = 3; - public final static int HOUR_OFFSET = 4; - public final static int MINUTE_OFFSET = 5; - public final static int MILLISECOND_OFFSET = 6; - public final static int TIMEZONE_HOUR_OFFSET = 10; - public final static int TIMEZONE_MINUTE_OFFSET = 11; + public static final int YEAR_OFFSET = 0; + public static final int MONTH_OFFSET = 2; + public static final int DAY_OFFSET = 3; + public static final int HOUR_OFFSET = 4; + public static final int MINUTE_OFFSET = 5; + public static final int MILLISECOND_OFFSET = 6; + public static final int TIMEZONE_HOUR_OFFSET = 10; + public static final int TIMEZONE_MINUTE_OFFSET = 11; public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() { private static final long serialVersionUID = 1L; @@ -84,25 +82,12 @@ public class XSDateTimePointable extends AbstractPointable implements IDate, ITi int month = cal.get(Calendar.MONTH); int day = cal.get(Calendar.DAY_OF_MONTH); int ms = cal.get(Calendar.MILLISECOND); - int tzOffsetInMs = tz.getOffset( - cal.get(Calendar.ERA), - year, - month, - day, - cal.get(Calendar.DAY_OF_WEEK), - ms); + int tzOffsetInMs = tz.getOffset(cal.get(Calendar.ERA), year, month, day, cal.get(Calendar.DAY_OF_WEEK), ms); int tzOffsetInMin = tzOffsetInMs / 1000 / 60; - setDateTime( - year, - month + 1, - day, - cal.get(Calendar.HOUR_OF_DAY), - cal.get(Calendar.MINUTE), - cal.get(Calendar.SECOND) * 1000 + ms, - tzOffsetInMin / 60, - tzOffsetInMin % 60); - } - + setDateTime(year, month + 1L, day, cal.get(Calendar.HOUR_OF_DAY), cal.get(Calendar.MINUTE), + cal.get(Calendar.SECOND) * 1000L + ms, tzOffsetInMin / 60L, tzOffsetInMin % 60); + } + public void setDateTime(long year, long month, long day, long hour, long minute, long milliSecond, long timezoneHour, long timezoneMinute) { setDateTime(bytes, start, year, month, day, hour, minute, milliSecond, timezoneHour, timezoneMinute); @@ -198,7 +183,7 @@ public class XSDateTimePointable extends AbstractPointable implements IDate, ITi } public static long getTimezone(byte[] bytes, int start) { - return (getTimezoneHour(bytes, start) * 60 + getTimezoneMinute(bytes, start)); + return getTimezoneHour(bytes, start) * 60 + getTimezoneMinute(bytes, start); } @Override @@ -207,7 +192,7 @@ public class XSDateTimePointable extends AbstractPointable implements IDate, ITi } public static long getYearMonth(byte[] bytes, int start) { - return (getYear(bytes, start) * 12 + getMonth(bytes, start)); + return getYear(bytes, start) * 12 + getMonth(bytes, start); } @Override @@ -216,33 +201,26 @@ public class XSDateTimePointable extends AbstractPointable implements IDate, ITi } public static long getDayTime(byte[] bytes, int start) { - return (getDay(bytes, start) * DateTime.CHRONON_OF_DAY + getHour(bytes, start) * DateTime.CHRONON_OF_HOUR - + getMinute(bytes, start) * DateTime.CHRONON_OF_MINUTE + getMilliSecond(bytes, start)); + return getDay(bytes, start) * DateTime.CHRONON_OF_DAY + getHour(bytes, start) * DateTime.CHRONON_OF_HOUR + + getMinute(bytes, start) * DateTime.CHRONON_OF_MINUTE + getMilliSecond(bytes, start); } + @Override public String toString() { return toString(bytes, start); } public static String toString(byte[] bytes, int start) { final long millis = getMilliSecond(bytes, start); - return getYear(bytes, start) - + "-" - + getMonth(bytes, start) - + "-" - + getDay(bytes, start) - + "T" - + getHour(bytes, start) - + ":" - + getMinute(bytes, start) - + ":" - + millis / 1000 - + "." - + millis % 1000 - + (getTimezoneHour(bytes, start) != DateTime.TIMEZONE_HOUR_NULL - && getTimezoneMinute(bytes, start) != DateTime.TIMEZONE_MINUTE_NULL ? (getTimezoneHour(bytes, - start) < 0 || getTimezoneMinute(bytes, start) < 0 ? "" : "+") - + getTimezoneHour(bytes, start) + ":" + getTimezoneMinute(bytes, start) : ""); + String timezone = ""; + if (getTimezoneHour(bytes, start) != DateTime.TIMEZONE_HOUR_NULL + && getTimezoneMinute(bytes, start) != DateTime.TIMEZONE_MINUTE_NULL) { + timezone = (getTimezoneHour(bytes, start) < 0 || getTimezoneMinute(bytes, start) < 0 ? "" : "+") + + getTimezoneHour(bytes, start) + ":" + getTimezoneMinute(bytes, start); + } + return getYear(bytes, start) + "-" + getMonth(bytes, start) + "-" + getDay(bytes, start) + "T" + + getHour(bytes, start) + ":" + getMinute(bytes, start) + ":" + millis / 1000 + "." + millis % 1000 + + timezone; } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java index 3558e93..1d8bec4 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java @@ -58,15 +58,15 @@ public class HDFSFunctions { private Configuration conf; private FileSystem fs; - private String conf_path; + private String confPath; private Job job; private InputFormat inputFormat; private List<InputSplit> splits; private ArrayList<ArrayList<String>> nodes; private HashMap<Integer, String> schedule; - private final String TEMP = "java.io.tmpdir"; - private final String dfs_path = "vxquery_splits_schedule.txt"; - private final String filepath = System.getProperty(TEMP) + "splits_schedule.txt"; + private static final String TEMP = "java.io.tmpdir"; + private static final String DFS_PATH = "vxquery_splits_schedule.txt"; + private static final String FILEPATH = System.getProperty(TEMP) + "splits_schedule.txt"; protected static final Logger LOGGER = Logger.getLogger(HDFSFunctions.class.getName()); private final Map<String, NodeControllerInfo> nodeControllerInfos; @@ -75,12 +75,14 @@ public class HDFSFunctions { * Initialize an instance of HDFS FileSystem for this configuration. * * @param nodeControllerInfos + * Map of the node to its attributes * @param hdfsConf + * Hdfs path to config */ public HDFSFunctions(Map<String, NodeControllerInfo> nodeControllerInfos, String hdfsConf) { this.conf = new Configuration(); this.nodeControllerInfos = nodeControllerInfos; - this.conf_path = hdfsConf; + this.confPath = hdfsConf; } /** @@ -88,6 +90,9 @@ public class HDFSFunctions { * This method should run before the scheduleSplits method. * * @param filepath + * Path to config. + * @param tag + * Tag to read. */ @SuppressWarnings({ "deprecation", "unchecked" }) public void setJob(String filepath, String tag) { @@ -112,9 +117,11 @@ public class HDFSFunctions { * Searches in subdirectories of the home directory too. * * @param filename - * @return + * HDFS file path. + * @return boolean + * True if located in HDFS. * @throws IOException - * @throws IllegalArgumentException + * If searching for the filepath throws {@link IOException} */ public boolean isLocatedInHDFS(String filename) throws IllegalArgumentException, IOException { //search file path @@ -161,20 +168,22 @@ public class HDFSFunctions { * @return true if is successfully finds the Hadoop/HDFS home directory */ private boolean locateConf() { - if (this.conf_path == null) { + if (this.confPath == null) { //As a last resort, try getting the configuration from the system environment //Some systems won't have this set. - this.conf_path = System.getenv("HADOOP_CONF_DIR"); + this.confPath = System.getenv("HADOOP_CONF_DIR"); } - return this.conf_path != null; + return this.confPath != null; } /** * Upload a file/directory to HDFS.Filepath is the path in the local file system.dir is the destination path. * * @param filepath + * file to upload * @param dir - * @return + * HDFS directory to save the file + * @return boolean */ public boolean put(String filepath, String dir) { if (this.fs != null) { @@ -204,12 +213,12 @@ public class HDFSFunctions { * Get instance of the HDFSfile system if it is configured correctly. * Return null if there is no instance. * - * @return + * @return FileSystem */ public FileSystem getFileSystem() { if (locateConf()) { - conf.addResource(new Path(this.conf_path + "/core-site.xml")); - conf.addResource(new Path(this.conf_path + "/hdfs-site.xml")); + conf.addResource(new Path(this.confPath + "/core-site.xml")); + conf.addResource(new Path(this.confPath + "/hdfs-site.xml")); try { fs = FileSystem.get(conf); return this.fs; @@ -226,19 +235,13 @@ public class HDFSFunctions { return null; } - /** - * Create a HashMap that has as key the hostname and values the splits that belong to this hostname; - * - * @return - * @throws IOException - */ public HashMap<String, ArrayList<Integer>> getLocationsOfSplits() throws IOException { HashMap<String, ArrayList<Integer>> splitsMap = new HashMap<>(); ArrayList<Integer> temp; int i = 0; String hostname; for (InputSplit s : this.splits) { - SplitLocationInfo info[] = s.getLocationInfo(); + SplitLocationInfo[] info = s.getLocationInfo(); hostname = info[0].getLocation(); if (splitsMap.containsKey(hostname)) { temp = splitsMap.get(hostname); @@ -318,16 +321,18 @@ public class HDFSFunctions { * Writes the schedule to a temporary file, then uploads the file to the HDFS. * * @throws UnsupportedEncodingException + * The encoding of the file is not correct * @throws FileNotFoundException + * The file doesn't exist */ public void addScheduleToDistributedCache() throws FileNotFoundException, UnsupportedEncodingException { - PrintWriter writer = new PrintWriter(filepath, "UTF-8"); + PrintWriter writer = new PrintWriter(FILEPATH, "UTF-8"); for (int split : this.schedule.keySet()) { writer.write(split + "," + this.schedule.get(split)); } writer.close(); // Add file to HDFS - this.put(filepath, dfs_path); + this.put(FILEPATH, DFS_PATH); } public RecordReader getReader() { @@ -377,7 +382,8 @@ public class HDFSFunctions { * Return the splits belonging to this node for the existing schedule. * * @param node - * @return + * HDFS node + * @return List */ public ArrayList<Integer> getScheduleForNode(String node) { ArrayList<Integer> nodeSchedule = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CaseSensitiveAnalyzer.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CaseSensitiveAnalyzer.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CaseSensitiveAnalyzer.java index f832362..92b4985 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CaseSensitiveAnalyzer.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/CaseSensitiveAnalyzer.java @@ -38,7 +38,7 @@ import java.io.Reader; * Filters {@link StandardTokenizer} with {@link StandardFilter}, * and {@link StopFilter}, using a list of * English stop words. - * <a name="version"/> + * <a name="version"></a> * <p> * You must specify the required {@link Version} * compatibility when creating StandardAnalyzer: @@ -92,6 +92,7 @@ public final class CaseSensitiveAnalyzer extends StopwordAnalyzerBase { * @see WordlistLoader#getWordSet(Reader) * @param stopwords * Reader to read stop words from + * @throws IOException : If loading stop words generates {@link IOException} */ public CaseSensitiveAnalyzer(Reader stopwords) throws IOException { this(loadStopwordSet(stopwords)); @@ -102,6 +103,7 @@ public final class CaseSensitiveAnalyzer extends StopwordAnalyzerBase { * that exceeds this length then it is discarded. This * setting only takes effect the next time tokenStream or * tokenStream is called. + * @param length : maximum token length */ public void setMaxTokenLength(int length) { maxTokenLength = length; @@ -109,6 +111,7 @@ public final class CaseSensitiveAnalyzer extends StopwordAnalyzerBase { /** * @see #setMaxTokenLength + * @return maxTokenLength */ public int getMaxTokenLength() { return maxTokenLength; http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java index 2f22ade..4706496 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/IndexConstructorUtil.java @@ -140,9 +140,6 @@ public class IndexConstructorUtil { return (path.toLowerCase().endsWith(".xml") || path.toLowerCase().endsWith(".xml.gz")); } - /** - * Separated from create index method so that it could be used as a helper function in IndexUpdater - */ public IndexDocumentBuilder getIndexBuilder(File file, IndexWriter writer, TaggedValuePointable nodep, ArrayBackedValueStorage abvsFileNode, ITreeNodeIdProvider nodeIdProvider, ByteBufferInputStream bbis, DataInputStream di, String nodeId) throws IOException { http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java index 38dcc2a..51510d5 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/indexCentralizer/IndexCentralizerUtil.java @@ -41,7 +41,7 @@ import org.apache.vxquery.datamodel.values.ValueTag; /** * Class for maintaining the centralized index information file. * Index centralization procedure. - * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = <index_directory>. + * User can specify the collection directory in VXQuery.java, ncConfig.ioDevices = < index_directory > . * Then all the indexes will be created in that particular directory in sub-folders corresponding to collections. * There will be a single xml file, located in the directory specified in local.xml, which contains all information * about the existing indexes. @@ -83,6 +83,7 @@ public class IndexCentralizerUtil { * Index location is created by using the last 100 characters of collection. * * @param collection : Collection directory + * @return index */ public String putIndexForCollection(String collection) { int length = collection.replaceAll("/", "").length(); @@ -109,8 +110,8 @@ public class IndexCentralizerUtil { /** * Prints all collections which have an index created. - * - * @throws IOException + * @param sb : The output is stored in a sequence + * @throws IOException : If writing the dataOutput generates {@link IOException} */ public void getAllCollections(SequenceBuilder sb) throws IOException { for (String s : collections) { http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java index ba7cd88..d3b9fdf 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/IndexUpdater.java @@ -69,10 +69,9 @@ public class IndexUpdater { private SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); private IndexConstructorUtil indexConstructorUtil = new IndexConstructorUtil(); - public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, - ByteBufferInputStream bbis, DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, - ITreeNodeIdProvider nodeIdProvider, ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, - String nodeId) { + public IndexUpdater(String indexFolder, IPointable result, UTF8StringPointable stringp, ByteBufferInputStream bbis, + DataInputStream di, SequenceBuilder sb, ArrayBackedValueStorage abvs, ITreeNodeIdProvider nodeIdProvider, + ArrayBackedValueStorage abvsFileNode, TaggedValuePointable nodep, String nodeId) { this.indexFolder = indexFolder; this.result = result; this.bbis = bbis; @@ -88,9 +87,11 @@ public class IndexUpdater { /** * Perform the initial configuration for index update/ delete processes. - * + * * @throws SystemException + * : If getting the index folder generates {@link SystemException} * @throws IOException + * : If getting the index folder generates {@link IOException} */ public void setup() throws SystemException, IOException { @@ -107,14 +108,15 @@ public class IndexUpdater { sb.reset(abvs); Directory fsdir = FSDirectory.open(Paths.get(indexFolder)); - indexWriter = new IndexWriter(fsdir, new IndexWriterConfig(new CaseSensitiveAnalyzer()). - setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND)); + indexWriter = new IndexWriter(fsdir, new IndexWriterConfig(new CaseSensitiveAnalyzer()) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND)); } /** * Wrapper for update index function. - * + * * @throws IOException + * : If the directory doesn't exist */ public void updateIndex() throws IOException { File collectionDirectory = new File(collectionFolder); @@ -132,8 +134,9 @@ public class IndexUpdater { /** * Close opened IndexWriter and terminate the index update/ delete process. - * + * * @throws IOException + * : If exiting the index folder generates {@link IOException} */ public void exit() throws IOException { indexWriter.forceMerge(1); @@ -146,8 +149,9 @@ public class IndexUpdater { /** * Functional wrapper to update Metadata file. - * + * * @throws IOException + * : If updating metadata folder generates {@link IOException} */ public synchronized void updateMetadataFile() throws IOException { //Write the updated metadata to the file. @@ -159,7 +163,8 @@ public class IndexUpdater { * Check the collection for changes. * If changes are detected, update the index * - * @param collection : Collection folder path + * @param collection + * : Collection folder path */ private void updateIndex(File collection) throws IOException { @@ -185,9 +190,8 @@ public class IndexUpdater { //Update index corresponding to the xml file. indexWriter.deleteDocuments(new Term(Constants.FIELD_PATH, file.getCanonicalPath())); - indexDocumentBuilder = indexConstructorUtil - .getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, - nodeId); + indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, + abvsFileNode, nodeIdProvider, bbis, di, nodeId); indexDocumentBuilder.printStart(); if (LOGGER.isDebugEnabled()) { @@ -203,8 +207,8 @@ public class IndexUpdater { // In this case, the xml file has not added to the index. (It is a newly added file) // Therefore generate a new index for this file and add it to the existing index. - indexDocumentBuilder = indexConstructorUtil - .getIndexBuilder(file, indexWriter, nodep, abvsFileNode, nodeIdProvider, bbis, di, nodeId); + indexDocumentBuilder = indexConstructorUtil.getIndexBuilder(file, indexWriter, nodep, abvsFileNode, + nodeIdProvider, bbis, di, nodeId); indexDocumentBuilder.printStart(); if (LOGGER.isDebugEnabled()) { @@ -223,10 +227,13 @@ public class IndexUpdater { /** * Update the current XmlMetadata object related to the currently reading XML file. * - * @param file : XML file - * @param metadata : Existing metadata object + * @param file + * : XML file + * @param metadata + * : Existing metadata object * @return : XML metadata object with updated fields. * @throws IOException + * : If getting the file info generates {@link IOException} */ private XmlMetadata updateEntry(File file, XmlMetadata metadata) throws IOException { @@ -243,9 +250,12 @@ public class IndexUpdater { /** * Delete the index of deleted files. * - * @param pathsFromMap : Set of paths taken from metafile. - * @param pathsFromFileList : Set of paths taken from list of existing files. + * @param pathsFromMap + * : Set of paths taken from metafile. + * @param pathsFromFileList + * : Set of paths taken from list of existing files. * @throws IOException + * : If deleting Documents generates {@link IOException} */ private void deleteIndexOfDeletedFiles(Set<String> pathsFromMap, Set<String> pathsFromFileList) throws IOException { Set<String> sfm = new HashSet<>(pathsFromMap); @@ -276,6 +286,9 @@ public class IndexUpdater { * When deleting indexes, if any error occurred, the process will be rolled back and all the indexes will be * restored. * Otherwise the changes will be committed. + * + * @throws SystemException + * : An attempt to divide by zero */ public void deleteAllIndexes() throws SystemException { try { http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java index dd099b5..0dfb54a 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/index/updateIndex/MetaFileUtil.java @@ -144,7 +144,7 @@ public class MetaFileUtil { * * @param file : File which the checksum should be generated. * @return : Checksum String - * @throws IOException + * @throws IOException : The file is not available */ public String generateMD5(File file) throws IOException { try { http://git-wip-us.apache.org/repos/asf/vxquery/blob/233a4306/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index 4176b20..85f519e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -1356,9 +1356,9 @@ public class FunctionHelper { /** * Writes a number to the DataOutput with zeros as place holders if the number is too small to fill the padding. * - * @param value + * @param valueArg * value - * @param padding + * @param paddingArg * padding * @param dOut * data output
