lens git commit: LENS-751 : Remove DISCLAIMER file
Repository: lens Updated Branches: refs/heads/current-release-line fef971528 -> 9fa1cb515 LENS-751 : Remove DISCLAIMER file Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9fa1cb51 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9fa1cb51 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9fa1cb51 Branch: refs/heads/current-release-line Commit: 9fa1cb515e03d302cc375f43ef5668471d7d6c90 Parents: fef9715 Author: Rajat Khandelwal Authored: Mon Aug 24 13:56:42 2015 +0530 Committer: Rajat Khandelwal Committed: Mon Aug 24 14:10:05 2015 +0530 -- lens-dist/src/main/assembly/bin-dist.xml | 7 --- 1 file changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/9fa1cb51/lens-dist/src/main/assembly/bin-dist.xml -- diff --git a/lens-dist/src/main/assembly/bin-dist.xml b/lens-dist/src/main/assembly/bin-dist.xml index 9e2ce8f..8707af9 100644 --- a/lens-dist/src/main/assembly/bin-dist.xml +++ b/lens-dist/src/main/assembly/bin-dist.xml @@ -408,13 +408,6 @@ /server/webapp/ lens-server.war - - - ../DISCLAIMER - / - DISCLAIMER - -
svn commit: r1697435 - in /lens/site/publish: ./ admin/ apidocs/ apidocs/org/apache/lens/api/ apidocs/org/apache/lens/api/class-use/ apidocs/org/apache/lens/api/error/ apidocs/org/apache/lens/api/erro
Author: prongs Date: Mon Aug 24 15:16:28 2015 New Revision: 1697435 URL: http://svn.apache.org/r1697435 Log: Publish docs for 2.3.0-beta-SNAPSHOT [This commit notification would consist of 819 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.]
svn commit: r10293 - /release/lens/KEYS
Author: prongs Date: Tue Aug 25 07:06:07 2015 New Revision: 10293 Log: Add keys of Rajat Khandelwal Modified: release/lens/KEYS Modified: release/lens/KEYS == --- release/lens/KEYS (original) +++ release/lens/KEYS Tue Aug 25 07:06:07 2015 @@ -1273,3 +1273,73 @@ FtaaeFIn76B4BJj3++0xC4hypgQQqwrgZsKhcmHY JxFS13HrHTeduj36d+qYsgzSmE3afy7I083R454= =OeLd -END PGP PUBLIC KEY BLOCK- +pub 4096R/15A87037 2015-04-28 +uid [ultimate] Rajat Khandelwal +sig 315A87037 2015-08-25 Rajat Khandelwal +sig 7B8A5D79 2015-04-28 Amareshwari Sriramadasu (Works for Hadoop and ecosystem) +sub 4096R/E60934E4 2015-04-28 [expires: 2019-04-28] +sig 15A87037 2015-04-28 Rajat Khandelwal + +-BEGIN PGP PUBLIC KEY BLOCK- +Comment: GPGTools - https://gpgtools.org + +mQINBFU/NgABEADuIhQWR4D4WRCrFGLI/RvTCl/2HsYZ+23UJGyG7Ki+Nhg9e7oc +93/H/L+MiDsc6sDBlq6DolRXF0uegHToiGwDOjuv6NnIuDorjwO3lSGs2DIR8F6i +fvjPeA8q3y74cKv43elfgNv/cTWW9t2sgJbaPTtLQ2L44C5P4roCzwRDL/4Qw4yB +pisFPr3uWXEDIo2sKb7hkGAmtMZORqZfLOdKfYMfCQgU0oASaHzbylo174QHQmaA +bjl5nD17Dq+3rvFXvNSnloZzpTjzdzAcDqcZ/vuZAF8gA8lKJXKUk60alCuf5QCm +7SSygRFLWGqZhdPv9Y1AzH3v2HkJ8VOCw5aaf3UBllzkOFSql1bjnu6XK3jwLtPn +1mHrtORKUWqhEYbitIfxSHF0oR1Q+qyUELQFlNB7kFBS0U3zs2YWCXd1xpDZjojR +fxYCDJWksVCYu5EcoC+pD0AFVogMJy8PIm76fULn0sVRHpPu0OfkDmGpViDv6/vU +R71ZNlmOT9Sn42TTuQ+7l/qQtim5LQDWmtYPOAOkI3uuuZ0kiCdwba4p4XFh2nKe ++akQzvhyENGtLcnDbvVWibJsj5QruUfjGu4vx0dQ+IEX9GDoPUPWevPVWhmxpcSH +cyi8PMgOf183DxGuBtjqX1ntuHlr87DspbcNPgTPLYJDBIemN+UDvtqxpwARAQAB +tCRSYWphdCBLaGFuZGVsd2FsIDxwcm9uZ3NAYXBhY2hlLm9yZz6JAjcEEwEKACEC +GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlXcEiwACgkQ3zxSYxWocDcbIQ// +XPQEExSwuLEgFH2CBLPcDQzgnGLtoXZb5frjaOpVjEGbel9QQeY8ALho1ghjDUqd +yImM1JZePz202BEtnjCDiCx1sYoIYnpq5kZgXbmacRWOdM6fYzLH8fo0W8EwFiMh +L2UC6EM7A+kBahyBaDEcZQDZb7SbqCGMyZy5vw7sSHpeVIqJlwuleJI8fPw0jfjC +6nOd8cnP4rA9jW4YqE2sUkewZHh5pFLWXglSkSYwTwSy8sv6nY3IJWAiYNaTfCA0 +Enby5wvf7D2pHPLMuNwzBOo8UaZOpRDPEwKcjUCbxvC/VFVL1QdTqi/jsczXWk7/ ++/JJBu38bQdV9Key4jBJV3+3dahI3pIuazFjlhz9DgkIsi2+tHu/9JOVfR1CqT8R +N9hqxfPg/dFGHdWAJ6ONFfcy8jfBesueOxNt6FZYhNsYggkJN1eQgdGp+jcCGko1 +J7ujt5rXiEzNxJ9E+9AhLBYe5t4z9xKY1pJ3sB4oTbszFNDaxxn/xCnMSLT2vEuO +tSfm35dqprcnmzEOEF87Rnbr/jY/0fVKJHzYe3B/fggR/PBY6kDum12WooC4N4+4 +ICW3Q9ZjY5J+ZrIqOJwRYdTc9l9Y9snJBelVCI2UcMiIkuvKOhiZ44pNZ/T0G9Jy +ncJzUMotAvzmkPSDkCBzTIQoWqvVDYEJ9pqS1GNWYhyJAhwEEAEKAAYFAlU/Q8gA +CgkQcnPMwXuKXXk3JA//SRINp8uZObBOVEgTwzn7VCkQ4i9JY1c2ceNhWgTB1S2L +WjZYbRITeMHv0SYQkz1qw+kUt84bdxFgMmZLvGe9TtrlH8LTKGclXybAy4iwb1iI ++72aUhNUOve5eGTulsgZAA7vKQLJ204a5Uho3BqG+rzbQoAoOpgMHWtqMjCnb9H5 +q7DzNrTiFCVs8JxdQOi2+b4Uia46OKJdHrI7gAx+cQh6wYwY0LkAXTXpfOFAolA7 +r4iLp3ChzeamOq1hRTeyy+wuwzIe+ULDpbPueu4Itb9cKBjArK/2Zi2HPTyJ99MP +f2h/ZNvGiG0CNOICCkSmfdPtovTIHncVsmqspyJzQoHFqPLvXA5MCkBFcaFTkVKy +ZrxoevsKb/VNmX4cQWlowVGbMwq/Unt1L9OngFM2u/uj9RfzNHkRbv3GqSkJog0d +27Y11ibm+5d+PLIJTpN7uApP8Q+e6VPsA7B7S9rr7oKK1WrSvIi1embRHxsxdP7n +2ZQD/iC+uLco13FunV6mfchUI8aJw5p8lzKhh5qQnzO9v7kGdQL8z5fzKhCNTZYI +rZF4B2ZR+/vuCOrPSYBjQRgANWFV5ztYKORCZOItLUMFT8DPLahm3Ez4LNhpActr +JtzPQVqGOBg0f7Dl/tltrAiKKz2+EX99jJV+UCoefDYmwAFh9oCMfJhhlaDQdoy5 +Ag0EVT82AAEQAMo5Z6BUsBNP1hpjNVXHbDh64QCPrz+Sdljjs+nrRvvUDg5Q2y0H +XmrU23vHg2RAIlsGkzMfO5KthxGvMnLMmP+JCvgm1M5+aQyuuYSBH000hBZhvtFI +mKFhiElMH/mdPzmWyKERf04SqAZjeUkDovIgcdMAigFPKlo3RedPl7zAz7MhsFU/ +QgVSBa/rhThn584ksh1azjIJf/oVxjtkg/1T5IQIk3vil0BhKxFW4WZcQNftnYCo +Upoujz9Vhp/76aZcpuYGLWXCiztnEeDBOcmcCa8i7Y7Imq6SGMQRQ49gnWMBGAzt +v1aQ2IiV8pLT9eOyUslmBG3HjvRJqGvD3jtH/AW+y2mR2TTjz3OG9oWQLZEM9ZRm +ANkjyUw1bGUSGaVpQn9I+/txAmhf7yqMBJ8QV4sFAEyQpj5yEbDXpHxDmD0zM4lQ +9G+icv7+uhtdYl95LD/cVMd+DC+fodWZPzG4ieWWkVVnZ0AX8kWzxps2TDqVBUys +7oUKVek6IAO6HN4IdxTpQDMkL/JP+LC8IEv6RgCsESPBo3krgpjuzvADoL9yDDgr +5f6Sgzpn3kQjn59InHAsGjMoet+bvJ7r9Py20gG28sAJEL2h0YDNWfh7VfiHluz9 +utqiRgDDyOaTOpoEPdRfkCt5x/b5Rsl77n9OECSvGhZjY0eIp1KsZ7UnABEBAAGJ +AiUEGAEKAA8FAlU/NgACGwwFCQeGH4AACgkQ3zxSYxWocDcy+RAAjxSu1eWdZrn3 +UdAa5AeK2Lzlg2N2PhZZTqfjFfLeBXojmn/4q3bqHCbz6m2mK8k59dgxmGXWxVNV +8i01yKTsKkffTD0XDN2oVjWCf2P97asKcSNCuydwTgEUv18IuVzlvxtsK+/vkO4t +ACZpsXlDv42Wfs6jAma9h/Oi3ug51/o/9cRNfglDNeTr+KOtkfGkunw6eA/KHwlQ +X/HBG7Pw9dIpWgIjcdJo8hh+lZ1RWaFr0on4t9LCec+FgbzXK9tsCjmYozyijVXc +dbOKgO7X6Y4VakACwZUgD/4J4SNmz336yd/A9T8ueLvxtBpj7iGWAS3FHp+2OkG7 +YrSPNT+RwNNX7RfLq2Xr5wW2l/xsm2hkP+sFFxJ7sVwJYcdZa/vg6e3U+JlDfAyM +0SAN54cJc5tcF+lAWN3fVWJuRhBB6Pbd44/ILdu+8N/Bvklpkk57EWDFXxVtIGKF +hLPqoolw2tlnCbs/15gIV3MfmAAmPdfIsEJzAmFMjcX7abx8JXVD5JeE47ylspIn +4MaRCIFG/QeTAUQRacYPnTa+CvRYOYqF2bWKKIPS6stKxk28a7jv/4pi9vjUjkA0 +SR7hYmH3HUWGdp7M6a6UxUhiH5QoZqArz3ZF46COzwxIZUOiCurPE36UJ2+RFGm4 +fvv/Q4ov9uqeUx+HeADQIFzPKGFJnRc= +=/eBz +-END PGP PUBLIC KEY BLOCK-
lens git commit: LENS-726 : Update 2.3 release documentation
Repository: lens Updated Branches: refs/heads/current-release-line 095fdb0ca -> 8dcfcde9e LENS-726 : Update 2.3 release documentation Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8dcfcde9 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8dcfcde9 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8dcfcde9 Branch: refs/heads/current-release-line Commit: 8dcfcde9e18ac5a89aac1b3e07cd19faf9badd7b Parents: 095fdb0 Author: Rajat Khandelwal Authored: Tue Aug 25 16:00:05 2015 +0530 Committer: Rajat Khandelwal Committed: Tue Aug 25 16:12:29 2015 +0530 -- src/site/apt/releases/release-history.apt | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/8dcfcde9/src/site/apt/releases/release-history.apt -- diff --git a/src/site/apt/releases/release-history.apt b/src/site/apt/releases/release-history.apt index 16eef95..b683360 100644 --- a/src/site/apt/releases/release-history.apt +++ b/src/site/apt/releases/release-history.apt @@ -57,6 +57,10 @@ All Apache Lens releases * LENS-650 : Pre submit hook is separate from User Config Loader. User Config Loader is no longer an abstract class. + * LENS-651 : In /resultset api, file size is added. File size and the already existing field num-rows are only +populated if needed. Default value is no more -1, it's null(unpopulated). +Finished queries table has one more column now, so existing tables need to be updated. +The following sql should work: <<< ALTER TABLE finished_queries ADD filesize bigint AFTER rows>>> ** 2.2.x from 2.1.x
lens git commit: [maven-release-plugin] prepare release apache-lens-2.3.0-beta
Repository: lens Updated Branches: refs/heads/current-release-line 8dcfcde9e -> d3ba5a384 [maven-release-plugin] prepare release apache-lens-2.3.0-beta Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d3ba5a38 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d3ba5a38 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d3ba5a38 Branch: refs/heads/current-release-line Commit: d3ba5a3845fdd1a90596f04ac979ab54bc1e65ad Parents: 8dcfcde Author: Rajat Khandelwal Authored: Tue Aug 25 17:56:09 2015 +0530 Committer: Rajat Khandelwal Committed: Tue Aug 25 17:56:09 2015 +0530 -- checkstyle/pom.xml | 6 +- lens-api/pom.xml | 2 +- lens-cli/pom.xml | 2 +- lens-client/pom.xml | 2 +- lens-cube/pom.xml| 2 +- lens-dist/pom.xml| 2 +- lens-driver-es/pom.xml | 5 ++--- lens-driver-hive/pom.xml | 2 +- lens-driver-jdbc/pom.xml | 17 - lens-examples/pom.xml| 2 +- lens-ml-dist/pom.xml | 2 +- lens-ml-lib/pom.xml | 2 +- lens-query-lib/pom.xml | 2 +- lens-regression/pom.xml | 2 +- lens-server-api/pom.xml | 2 +- lens-server/pom.xml | 19 +-- lens-storage-db/pom.xml | 2 +- pom.xml | 13 ++--- 18 files changed, 43 insertions(+), 43 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/checkstyle/pom.xml -- diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index 99928c3..545941f 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -29,7 +29,7 @@ org.apache.lens checkstyle Lens Checkstyle Rules - 2.3.0-beta-SNAPSHOT + 2.3.0-beta @@ -49,4 +49,8 @@ + + +apache-lens-2.3.0-beta + http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-api/pom.xml -- diff --git a/lens-api/pom.xml b/lens-api/pom.xml index 6a9517f..b1b2103 100644 --- a/lens-api/pom.xml +++ b/lens-api/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-api http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-cli/pom.xml -- diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index 22bfa1f..783215b 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-client/pom.xml -- diff --git a/lens-client/pom.xml b/lens-client/pom.xml index e726ead..052dd54 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-client http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-cube/pom.xml -- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index 394acef..3a49b28 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-cube http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-dist/pom.xml -- diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml index 9b0cda6..8bb998e 100644 --- a/lens-dist/pom.xml +++ b/lens-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-dist http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-driver-es/pom.xml -- diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index 178aaca..d450b13 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -19,13 +19,12 @@ under the License. --> -http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> 4.0.0 org.apache.lens apache-lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-driver-es http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-driver-hive/pom.xml ---
Git Push Summary
Repository: lens Updated Tags: refs/tags/apache-lens-2.3.0-beta [created] e3bb345de
lens git commit: [maven-release-plugin] prepare for next development iteration
Repository: lens Updated Branches: refs/heads/current-release-line d3ba5a384 -> 1237842ab [maven-release-plugin] prepare for next development iteration Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1237842a Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1237842a Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1237842a Branch: refs/heads/current-release-line Commit: 1237842ab114b4fba7222492aa6e79b6e8d422b2 Parents: d3ba5a3 Author: Rajat Khandelwal Authored: Tue Aug 25 17:56:35 2015 +0530 Committer: Rajat Khandelwal Committed: Tue Aug 25 17:56:35 2015 +0530 -- checkstyle/pom.xml | 6 +- lens-api/pom.xml | 2 +- lens-cli/pom.xml | 2 +- lens-client/pom.xml | 2 +- lens-cube/pom.xml| 2 +- lens-dist/pom.xml| 2 +- lens-driver-es/pom.xml | 2 +- lens-driver-hive/pom.xml | 2 +- lens-driver-jdbc/pom.xml | 2 +- lens-examples/pom.xml| 2 +- lens-ml-dist/pom.xml | 2 +- lens-ml-lib/pom.xml | 2 +- lens-query-lib/pom.xml | 2 +- lens-regression/pom.xml | 2 +- lens-server-api/pom.xml | 2 +- lens-server/pom.xml | 2 +- lens-storage-db/pom.xml | 2 +- pom.xml | 8 18 files changed, 21 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/checkstyle/pom.xml -- diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index 545941f..bf15ca4 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -29,7 +29,7 @@ org.apache.lens checkstyle Lens Checkstyle Rules - 2.3.0-beta + 2.3.1-beta-SNAPSHOT @@ -49,8 +49,4 @@ - - -apache-lens-2.3.0-beta - http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-api/pom.xml -- diff --git a/lens-api/pom.xml b/lens-api/pom.xml index b1b2103..93bc1e7 100644 --- a/lens-api/pom.xml +++ b/lens-api/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-api http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-cli/pom.xml -- diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index 783215b..18c7af7 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-client/pom.xml -- diff --git a/lens-client/pom.xml b/lens-client/pom.xml index 052dd54..9c08a8a 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-client http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-cube/pom.xml -- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index 3a49b28..3fc784a 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-cube http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-dist/pom.xml -- diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml index 8bb998e..69a5fca 100644 --- a/lens-dist/pom.xml +++ b/lens-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-dist http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-driver-es/pom.xml -- diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index d450b13..f8af7af 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -24,7 +24,7 @@ org.apache.lens apache-lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-driver-es http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-driver-hive/pom.xml -- diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml index a1c87da..ee9d92e 100644 --- a/lens-driver-hive/pom.xml +++ b/lens-driver-hive/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-driver-jdbc/pom.xml -- diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jd
svn commit: r10296 - /dev/lens/apache-lens-2.3.0-beta-rc0/
Author: prongs Date: Tue Aug 25 13:45:17 2015 New Revision: 10296 Log: Adding release candidate apache-lens-2.3.0-beta-rc0 Added: dev/lens/apache-lens-2.3.0-beta-rc0/ dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz (with props) dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.asc dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.md5 dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.sha512 dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip (with props) dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip.asc dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip.md5 dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip.sha512 Added: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz == Binary file - no diff available. Propchange: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz -- svn:mime-type = application/octet-stream Added: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.asc == --- dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.asc (added) +++ dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.asc Tue Aug 25 13:45:17 2015 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Comment: GPGTools - https://gpgtools.org + +iQIcBAABCgAGBQJV3F6ZAAoJEN88UmMVqHA3C14P/jHs3TxkQk6oagbmPPcoNdK5 +Ctb8MiEAqtCCABR44D6D/qcdywDPktWP7wFUQIooBlQRozlT401ys/0OlPek+Kw5 +ydtZDSOLfSZxIO4OQssJ7btYzHNB1yyst8V9dHSnboDym6XoaYGAIOM1hRDy1ibN +Jp5Ff1mSF63HfkYNIu+hwdfTmrxYGqDaoUwYF2aCdKlFtfNLkpI2THbyyOMocBRD +upp2BPUpeY6ZaC7nUTX2rLmvltIkNCoO1tW8eN1M56ihVYcQNG/UZJtErYQ1W5al +A51d/3/2p7n18IyferruoNSz/9/6LOCwH+OPeYWFj/N3GTC8McQHSGEDezPvg36z +X/7rZ1YMFtLS+hl+HuI/iBeL5q29eE3GuJQVlRPalo0xSofomCE1TiSpbpZRTCFf +SxP/ZYlojYWG3dAhAVxj1ZwyUyhGHIyfWz8IqNmt9M8jdj/kv6ZNcj9nvqSnUqP7 +MKZqK1o+uzz+iuajOmeyuYH4iG92gpRMsKIN/rm727tmXJc8BsU3akV6biT2x408 +jIoL8aUdjBQeUNiR2G3BFTqf1+kI8SC+UEw5s7UYAjw7JEgWb1NKquF9Fb9XHxji +XZEKw8pNCcqFwUO102keh2D9zliqBllCZ/vk40kVrE8nDWxDawaXHEDbUn+t0tkN +RWqOMz9SToWIm6tzbcHP +=jHvy +-END PGP SIGNATURE- Added: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.md5 == --- dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.md5 (added) +++ dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.md5 Tue Aug 25 13:45:17 2015 @@ -0,0 +1,2 @@ +apache-lens-2.3.0-beta-bin.tar.gz: 7F C4 83 FB 19 BE 1D 0C 80 65 7A 14 DE 66 72 + 7A Added: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.sha512 == --- dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.sha512 (added) +++ dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-bin.tar.gz.sha512 Tue Aug 25 13:45:17 2015 @@ -0,0 +1,4 @@ +apache-lens-2.3.0-beta-bin.tar.gz: E675A966 13756E64 0D93F910 4E3D61BC 92C5E397 + A1151477 A8CCF384 D2B5EC70 C69F9AFD 4FE915A0 + 1E88B3E0 5023ABB5 F9F5D7CC 94EB6FDD F26B51F4 + 5927FD3C Added: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip == Binary file - no diff available. Propchange: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip.asc == --- dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip.asc (added) +++ dev/lens/apache-lens-2.3.0-beta-rc0/apache-lens-2.3.0-beta-source-release.zip.asc Tue Aug 25 13:45:17 2015 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Comment: GPGTools - https://gpgtools.org + +iQIcBAABCgAGBQJV3GaLAAoJEN88UmMVqHA3GjMP/3MOnYOiDi7NhSdKgnnG7645 +Qk23iTlIRl/ohve5QXd1+FSmp1e/6sPgTS4ZLejKu/N0y661NAPWceF5ah7cRRIS +uU50pVRdJS1jrW29gpBgKbsn4H075Av5q2QkL1n/jYT1r5PDVHranBRjjKcvT5Gj +kVdjdBKqM3pVWN/2hRS1LMYL4oogpScqnrdRuUwGPQxpqZ3vjB+20ZjnpJm8hzCJ +OiQN2SvBuig80tZn5OphEUN+pmqu7ts4Tav93n2cozKUuQqxQYW6R4mFUsiSA6e7 +4whvdmrWvEmqbigTo8p+jKXJ7ZOvrrVYiwBwo6mDQ8PC3dCh95wfmhBLjozHPXIA +VBgBQ20cYnsXMcxFQaCXmu2AXcZN8Z9iLYNn1Na3YUNEmfJ5RorbzJ+8Nq6q/R3t
lens git commit: LENS-757: Examples should continue to run in case submission failures
Repository: lens
Updated Branches:
refs/heads/master af3116639 -> fa8e5206f
LENS-757: Examples should continue to run in case submission failures
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fa8e5206
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fa8e5206
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fa8e5206
Branch: refs/heads/master
Commit: fa8e5206ffda1d4cdc6db13feac195d4b0caf9d0
Parents: af31166
Author: Amareshwari Sriramadasu
Authored: Wed Aug 26 20:17:57 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Aug 26 20:17:57 2015 +0530
--
.../org/apache/lens/examples/SampleQueries.java | 52 +++-
1 file changed, 29 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/fa8e5206/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
--
diff --git
a/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
b/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
index 6b662e7..805a282 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
@@ -26,6 +26,7 @@ import java.io.InputStreamReader;
import javax.xml.bind.JAXBException;
import org.apache.lens.api.query.*;
+import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientSingletonWrapper;
import org.apache.lens.client.LensMetadataClient;
import org.apache.lens.client.LensStatement;
@@ -83,9 +84,6 @@ public class SampleQueries {
}
}
queries.queryAll();
- if (queries.retCode != 0) {
-System.exit(queries.retCode);
- }
} finally {
if (queries != null) {
queries.close();
@@ -93,6 +91,9 @@ public class SampleQueries {
long end = System.currentTimeMillis();
System.out.println("Total time for running examples(in millis) :" +
(end-start));
}
+if (queries.retCode != 0) {
+ System.exit(queries.retCode);
+}
}
/**
@@ -132,29 +133,34 @@ public class SampleQueries {
}
total++;
System.out.println("Query:" + query);
- QueryHandle handle = queryClient.executeQuery(query, true,
null).getData();
- System.out.println("Status:" + queryClient.getQuery().getStatus());
- System.out.println("Total time in millis:"
-+ (queryClient.getQuery().getFinishTime() -
queryClient.getQuery().getSubmissionTime()));
- System.out.println("Driver run time in millis:"
-+ (queryClient.getQuery().getDriverFinishTime() -
queryClient.getQuery().getDriverStartTime()));
- if (queryClient.wasQuerySuccessful()) {
-success++;
-if (queryClient.getQuery().getStatus().isResultSetAvailable()) {
- System.out.println("Result:");
- QueryResult queryResult = queryClient.getResultSet();
- if (queryResult instanceof InMemoryQueryResult) {
-InMemoryQueryResult result = (InMemoryQueryResult) queryResult;
-for (ResultRow row : result.getRows()) {
- System.out.println(StringUtils.join(row.getValues(), "\t"));
+ try {
+QueryHandle handle = queryClient.executeQuery(query, true,
null).getData();
+System.out.println("Status:" + queryClient.getQuery().getStatus());
+System.out.println("Total time in millis:"
+ + (queryClient.getQuery().getFinishTime() -
queryClient.getQuery().getSubmissionTime()));
+System.out.println("Driver run time in millis:"
+ + (queryClient.getQuery().getDriverFinishTime() -
queryClient.getQuery().getDriverStartTime()));
+if (queryClient.wasQuerySuccessful()) {
+ success++;
+ if (queryClient.getQuery().getStatus().isResultSetAvailable()) {
+System.out.println("Result:");
+QueryResult queryResult = queryClient.getResultSet();
+if (queryResult instanceof InMemoryQueryResult) {
+ InMemoryQueryResult result = (InMemoryQueryResult) queryResult;
+ for (ResultRow row : result.getRows()) {
+System.out.println(StringUtils.join(row.getValues(), "\t"));
+ }
+} else if (queryResult instanceof PersistentQueryResult) {
+ PersistentQueryResult persistentQueryResult =
(PersistentQueryResult) queryResult;
+ System.out.println("Result stored at " +
persistentQueryResult.getPersistedURI());
}
- } else if (queryResult instanceof PersistentQueryResult) {
-PersistentQueryResult persistentQueryResult =
(PersistentQueryResult) queryResult;
-System.out.println("Result stored at " +
persistentQuery
lens git commit: LENS-753: Queue number for queries in submitted queue
Repository: lens
Updated Branches:
refs/heads/master fa8e5206f -> 2f22f60b2
LENS-753: Queue number for queries in submitted queue
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2f22f60b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2f22f60b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2f22f60b
Branch: refs/heads/master
Commit: 2f22f60b2d50c20ec9291483d73d4fd735eb8ef9
Parents: fa8e520
Author: Akshay Goyal
Authored: Thu Aug 27 20:52:52 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Aug 27 20:52:52 2015 +0530
--
.../org/apache/lens/api/query/QueryStatus.java | 11 +++
.../lens/cli/commands/LensQueryCommands.java| 3 ++
.../server/api/driver/DriverQueryStatus.java| 14 +++--
.../server/api/query/FinishedLensQuery.java | 2 +-
.../lens/server/api/query/QueryContext.java | 2 +-
.../query/collect/ImmutableQueryCollection.java | 6
.../server/query/QueryExecutionServiceImpl.java | 15 +-
.../DefaultEstimatedQueryCollection.java| 5
.../query/collect/DefaultQueryCollection.java | 20 +
.../query/collect/MutableQueryCollection.java | 1 +
.../ThreadSafeEstimatedQueryCollection.java | 5
.../collect/ThreadSafeQueryCollection.java | 5
.../collect/DefaultQueryCollectionTest.java | 23 +++
.../server/query/collect/QueryCollectUtil.java | 30
14 files changed, 130 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2f22f60b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index 9614caa..0d73c3f 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -39,6 +39,8 @@ import lombok.*;
*
* @param progress
* the progress
+ * @param queueNumber
+ * the queue number
* @param status
* the status
* @param statusMessage
@@ -121,6 +123,14 @@ public class QueryStatus implements Serializable {
private double progress;
/**
+ * The running queue number. A non zero value gives the queue number. Queue
number zero mean either the query is in
+ * waiting or completed state.
+ */
+ @XmlElement
+ @Getter
+ private int queueNumber;
+
+ /**
* The status.
*/
@XmlElement
@@ -168,6 +178,7 @@ public class QueryStatus implements Serializable {
StringBuilder str = new
StringBuilder(status.toString()).append(':').append(statusMessage);
if (status.equals(Status.RUNNING)) {
str.append(" -
Progress:").append(progress).append(":").append(progressMessage);
+ str.append(" - Queue number:").append(queueNumber);
}
if (status.equals(Status.SUCCESSFUL)) {
if (isResultSetAvailable) {
http://git-wip-us.apache.org/repos/asf/lens/blob/2f22f60b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
--
diff --git
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 7a5b177..5f90060 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -170,6 +170,9 @@ public class LensQueryCommands extends BaseLensCommand {
sb.append("Progress Message :
").append(status.getProgressMessage()).append("\n");
}
}
+if (status.getQueueNumber() != 0) {
+ sb.append("Queue Number :
").append(status.getQueueNumber()).append("\n");
+}
if (status.getErrorMessage() != null) {
sb.append("Error : ").append(status.getErrorMessage()).append("\n");
http://git-wip-us.apache.org/repos/asf/lens/blob/2f22f60b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
index f78b7c3..48a841b 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
@@ -89,6 +89,13 @@ public class DriverQueryStatus implements Serializable {
private double progress = 0.0f;
/**
+ * The running queue
svn commit: r10328 - in /release/lens: 2.2-beta-incubating/ 2.3-beta/
Author: prongs Date: Fri Aug 28 14:26:39 2015 New Revision: 10328 Log: Release 2.3 beta Added: release/lens/2.3-beta/ release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz (with props) release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.asc release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.md5 release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.sha512 release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip (with props) release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip.asc release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip.md5 release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip.sha512 Removed: release/lens/2.2-beta-incubating/ Added: release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz == Binary file - no diff available. Propchange: release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz -- svn:mime-type = application/octet-stream Added: release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.asc == --- release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.asc (added) +++ release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.asc Fri Aug 28 14:26:39 2015 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Comment: GPGTools - https://gpgtools.org + +iQIcBAABCgAGBQJV3F6ZAAoJEN88UmMVqHA3C14P/jHs3TxkQk6oagbmPPcoNdK5 +Ctb8MiEAqtCCABR44D6D/qcdywDPktWP7wFUQIooBlQRozlT401ys/0OlPek+Kw5 +ydtZDSOLfSZxIO4OQssJ7btYzHNB1yyst8V9dHSnboDym6XoaYGAIOM1hRDy1ibN +Jp5Ff1mSF63HfkYNIu+hwdfTmrxYGqDaoUwYF2aCdKlFtfNLkpI2THbyyOMocBRD +upp2BPUpeY6ZaC7nUTX2rLmvltIkNCoO1tW8eN1M56ihVYcQNG/UZJtErYQ1W5al +A51d/3/2p7n18IyferruoNSz/9/6LOCwH+OPeYWFj/N3GTC8McQHSGEDezPvg36z +X/7rZ1YMFtLS+hl+HuI/iBeL5q29eE3GuJQVlRPalo0xSofomCE1TiSpbpZRTCFf +SxP/ZYlojYWG3dAhAVxj1ZwyUyhGHIyfWz8IqNmt9M8jdj/kv6ZNcj9nvqSnUqP7 +MKZqK1o+uzz+iuajOmeyuYH4iG92gpRMsKIN/rm727tmXJc8BsU3akV6biT2x408 +jIoL8aUdjBQeUNiR2G3BFTqf1+kI8SC+UEw5s7UYAjw7JEgWb1NKquF9Fb9XHxji +XZEKw8pNCcqFwUO102keh2D9zliqBllCZ/vk40kVrE8nDWxDawaXHEDbUn+t0tkN +RWqOMz9SToWIm6tzbcHP +=jHvy +-END PGP SIGNATURE- Added: release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.md5 == --- release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.md5 (added) +++ release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.md5 Fri Aug 28 14:26:39 2015 @@ -0,0 +1,2 @@ +apache-lens-2.3.0-beta-bin.tar.gz: 7F C4 83 FB 19 BE 1D 0C 80 65 7A 14 DE 66 72 + 7A Added: release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.sha512 == --- release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.sha512 (added) +++ release/lens/2.3-beta/apache-lens-2.3.0-beta-bin.tar.gz.sha512 Fri Aug 28 14:26:39 2015 @@ -0,0 +1,4 @@ +apache-lens-2.3.0-beta-bin.tar.gz: E675A966 13756E64 0D93F910 4E3D61BC 92C5E397 + A1151477 A8CCF384 D2B5EC70 C69F9AFD 4FE915A0 + 1E88B3E0 5023ABB5 F9F5D7CC 94EB6FDD F26B51F4 + 5927FD3C Added: release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip == Binary file - no diff available. Propchange: release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip -- svn:mime-type = application/octet-stream Added: release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip.asc == --- release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip.asc (added) +++ release/lens/2.3-beta/apache-lens-2.3.0-beta-source-release.zip.asc Fri Aug 28 14:26:39 2015 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Comment: GPGTools - https://gpgtools.org + +iQIcBAABCgAGBQJV3GaLAAoJEN88UmMVqHA3GjMP/3MOnYOiDi7NhSdKgnnG7645 +Qk23iTlIRl/ohve5QXd1+FSmp1e/6sPgTS4ZLejKu/N0y661NAPWceF5ah7cRRIS +uU50pVRdJS1jrW29gpBgKbsn4H075Av5q2QkL1n/jYT1r5PDVHranBRjjKcvT5Gj +kVdjdBKqM3pVWN/2hRS1LMYL4oogpScqnrdRuUwGPQxpqZ3vjB+20ZjnpJm8hzCJ +OiQN2SvBuig80tZn5OphEUN+pmqu7ts4Tav93n2cozKUuQqxQYW6R4mFUsiSA6e7 +4whvdmrWvEmqbigTo8p+jKXJ7ZOvrrVYiwBwo6mDQ8PC3dCh95wfmhBLjozHPXIA +VBgBQ20cYnsXMcxFQaCXmu2AXcZN8Z9iLYNn1Na3YUNEmfJ5RorbzJ+8Nq6q/R3t +wW7rl8G6L6p1RUx/dP0yt7PznKHt0SBlxkhjLR92ZjhwjQT1aJvsxuSm8rS/gIpy +PX2OditJ7rLHl4yUxtUGalKG72Tw8zAEIFMX0xPF6gy4ORw3FaBtmouENx6iZ2v2 +f++s1asVkV/jX671cs+/6YCjE9n0xZInqPdKXatPwvNk7ScGkFdrKa9pmGxGfZf8 +0C/EeuuPgGfkDBG4FbYM1iXIfj1dJMUgH0011FDDY7VbZJNoKIvmpsnuG3hDgCbk +jE/pRXnbcoI4IExbCrxY +=l1Zc +-END PGP SIGNATURE- Added: release/lens/2.3-beta/apache
[10/13] lens git commit: LENS-726 : Update 2.3 release documentation
LENS-726 : Update 2.3 release documentation Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8dcfcde9 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8dcfcde9 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8dcfcde9 Branch: refs/heads/master Commit: 8dcfcde9e18ac5a89aac1b3e07cd19faf9badd7b Parents: 095fdb0 Author: Rajat Khandelwal Authored: Tue Aug 25 16:00:05 2015 +0530 Committer: Rajat Khandelwal Committed: Tue Aug 25 16:12:29 2015 +0530 -- src/site/apt/releases/release-history.apt | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/8dcfcde9/src/site/apt/releases/release-history.apt -- diff --git a/src/site/apt/releases/release-history.apt b/src/site/apt/releases/release-history.apt index 16eef95..b683360 100644 --- a/src/site/apt/releases/release-history.apt +++ b/src/site/apt/releases/release-history.apt @@ -57,6 +57,10 @@ All Apache Lens releases * LENS-650 : Pre submit hook is separate from User Config Loader. User Config Loader is no longer an abstract class. + * LENS-651 : In /resultset api, file size is added. File size and the already existing field num-rows are only +populated if needed. Default value is no more -1, it's null(unpopulated). +Finished queries table has one more column now, so existing tables need to be updated. +The following sql should work: <<< ALTER TABLE finished_queries ADD filesize bigint AFTER rows>>> ** 2.2.x from 2.1.x
[03/13] lens git commit: LENS-744 : Improve documentation for Server Component
LENS-744 : Improve documentation for Server Component
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a7506f93
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a7506f93
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a7506f93
Branch: refs/heads/master
Commit: a7506f933c146ca0449589d8e52b282daa37674a
Parents: 67d6969
Author: srikanth.sundarrajan
Authored: Thu Aug 20 09:23:45 2015 +0530
Committer: Raju Bairishetti
Committed: Thu Aug 20 09:23:45 2015 +0530
--
src/site/apt/admin/server-components.apt | 67 +--
1 file changed, 42 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/a7506f93/src/site/apt/admin/server-components.apt
--
diff --git a/src/site/apt/admin/server-components.apt
b/src/site/apt/admin/server-components.apt
index e718a48..98c536c 100644
--- a/src/site/apt/admin/server-components.apt
+++ b/src/site/apt/admin/server-components.apt
@@ -19,8 +19,10 @@
Lens server components
- Lens server comes up with embedded http server. It offers services for
- session management, metastore and query.
+ Lens server by default comes up with an embedded Jersey server. Lens server
is composed of multiple services
+ which are responsible for different functions. The key services inside Lens
are Session Management, Query life cycle management,
+ Schema management, Data availability management, Query schedule management
(in progress), Metrics Registry, Besides, it also offers
+ a simple UI for the users.
Here is the diagram describing all the components:
@@ -28,32 +30,47 @@ Lens server components
* Lens services
-** Session service
+** Session Management
+ All operations in Lens are performed in the context of a session. This
would essentially be the first
+ thing a client would need before any operation can be be performed in the
lens system. Session management
+ component facilitates creation, session specific configuration management,
session resource management
+ within a session and termination of a session. Session info is periodically
written out a file (could be a
+ local file or a file on HDFS) and the same is read during Lens restart.
Frequency of this flush to persistent storage is
+ configurable. Sessions are kept alive for a configurable period (24 hours
by default) since last activity on
+the session.
- Allows user to start a session, close a session,
- add/delete resources in session, set session level parameters.
+** Schema Management
+ This component allows for cubes, fact tables, dimension tables and storages
to be managed. While Storages can
+ be managed only by the administrator, other objects can be managed directly
by the lens users. Lens currently
+ extends and uses Hive Metastore for managing lens objects. Lens server has
to be configured appropriately to point
+ to the hive metastore end point. See
{{{../resource_MetastoreResource.html#path__metastore_storages.html}Storages
API}}
+ and
{{{../resource_MetastoreResource.html#path__metastore_storages_-storage-.html}Storage
API}} for details about
+ storage administration.
-** Metastore service
+** Data Availability Management
+ Once schema is registered with Lens through the Schema management API, data
can be made available in the form of partitions.
+ The Lens data availability management (logical component) manages the
partitions available for each fact and maintains an
+ active timeline of data. This can be consulted by the query execution and
managment components. Like the Schema management
+ component, Lens currently uses the Hive metastore for data availability
management as well.
- Allows user to do crud on cubes, fact tables, dimension tables, partitions.
- Administrator would be managing the storages available. So, all the crud for
- storages will be done by administrator. See
-
{{{../resource_MetastoreResource.html#path__metastore_storages.html}Storages
API}} and
-
{{{../resource_MetastoreResource.html#path__metastore_storages_-storage-.html}Storage
API}} for details.
-
- Metastore service talks to Hive metastore server.Hive metastore server
end point
- has to be configured for lens server.
+** Query life cycle management
+ Servicing user queries over cubes, fact, dimension or native tables across
different storages by choosing the best storage
+ and execution engine is the most critical capability of the lens. Lens
maintains state of queries submitted and tracks the progress
+ through the system and also maintains a history of queries submitted and
the its status. Unique query handle is issued to each
+ query submitted and this can be used to track status, retr
[01/13] lens git commit: LENS-729 : Add apache license header in es driver's json files
Repository: lens
Updated Branches:
refs/heads/master fe44e69fd -> 2901b4b33
LENS-729 : Add apache license header in es driver's json files
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0aad08f9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0aad08f9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0aad08f9
Branch: refs/heads/master
Commit: 0aad08f919571c45fa7f5bdc4365bc18109cc72a
Parents: 1ba567f
Author: Rajat Khandelwal
Authored: Mon Aug 17 13:50:03 2015 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 17 14:27:54 2015 +0530
--
.../lens/driver/es/QueryTranslationTest.java| 2 ++
.../src/test/resources/invalid-queries.data | 21 +++-
.../src/test/resources/valid-queries.data | 21 +++-
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/0aad08f9/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
--
diff --git
a/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
b/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
index be33764..3846b23 100644
---
a/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
+++
b/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
@@ -33,6 +33,7 @@ import org.testng.annotations.Test;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -85,6 +86,7 @@ public class QueryTranslationTest extends ESDriverTest {
private static final ImmutableList VALID_QUERIES;
private static final ImmutableList IN_VALID_QUERIES;
static {
+OBJECT_MAPPER.configure(JsonParser.Feature.ALLOW_COMMENTS, true); //
Jackson 1.2+
VALID_QUERIES =
ImmutableList.copyOf(loadResource(VALID_QUERIES_RESOURCE_PATH,
ValidQuery[].class));
IN_VALID_QUERIES =
ImmutableList.copyOf(loadResource(INVALID_QUERIES_RESOURCE_PATH,
InvalidQuery[].class));
}
http://git-wip-us.apache.org/repos/asf/lens/blob/0aad08f9/lens-driver-es/src/test/resources/invalid-queries.data
--
diff --git a/lens-driver-es/src/test/resources/invalid-queries.data
b/lens-driver-es/src/test/resources/invalid-queries.data
index 8756a51..dd6a57f 100644
--- a/lens-driver-es/src/test/resources/invalid-queries.data
+++ b/lens-driver-es/src/test/resources/invalid-queries.data
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
[
{
"name" : "Ambiguous column selection",
@@ -15,4 +34,4 @@
"name" : "Invalid columns in select and group by",
"hql" : "select a, b from a.x group by r"
}
-]
\ No newline at end of file
+]
http://git-wip-us.apache.org/repos/asf/lens/blob/0aad08f9/lens-driver-es/src/test/resources/valid-queries.data
--
diff --git a/lens-driver-es/src/test/resources/valid-queries.data
b/lens-driver-es/src/test/resources/valid-queries.data
index 982325c..46de030 100644
--- a/lens-driver-es/src/test/resources/valid-queries.data
+++ b/lens-driver-es/src/test/resources/valid-queries.data
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in
[13/13] lens git commit: Merge with current-release-line
Merge with current-release-line
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2901b4b3
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2901b4b3
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2901b4b3
Branch: refs/heads/master
Commit: 2901b4b33677f694b68adb22eee7994307bd77d4
Parents: fe44e69 1237842
Author: Rajat Khandelwal
Authored: Fri Aug 28 20:15:16 2015 +0530
Committer: Rajat Khandelwal
Committed: Fri Aug 28 20:15:16 2015 +0530
--
lens-driver-es/pom.xml | 3 +--
.../src/test/resources/valid-queries.data | 2 +-
lens-driver-jdbc/pom.xml | 15 +++
lens-server/pom.xml| 17 -
pom.xml| 5 ++---
5 files changed, 19 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2901b4b3/lens-driver-es/pom.xml
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2901b4b3/lens-driver-es/src/test/resources/valid-queries.data
--
diff --cc lens-driver-es/src/test/resources/valid-queries.data
index f4ad15b,46de030..ff3074e
--- a/lens-driver-es/src/test/resources/valid-queries.data
+++ b/lens-driver-es/src/test/resources/valid-queries.data
@@@ -82,20 -82,5 +82,20 @@@
"name" : "Group by with multiple aggregation and complex criteria, select
order jumbled",
"hql" : "select count(col1), col2, max(col3) from index.type where
!(col1=5 and col2='p' and col3='wer' or (col4='9')) group by col2",
"expectedJson" :
{"size":0,"timeout":1,"aggs":{"filter_wrapper":{"aggs":{"col2":{"terms":{"field":"col2","size":100},"aggs":{"col1":{"value_count":{"field":"col1"}},"col3":{"max":{"field":"col3"},"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}
+ },
+ {
+"name" : "Check in predicate",
+"hql" : "select col1 from index.type where col2 in ('val1', 'val2')",
+"expectedJson" :
{"from":0,"size":1,"fields":["col1"],"sort":[],"timeout":1,"filter":{"terms":{"col2":["val1","val2"]}}}
+ },
+ {
+"name" : "Check not in predicate",
+"hql" : "select col1 from index.type where col2 not in ('val1', 'val2')",
+"expectedJson" :
{"from":0,"size":1,"fields":["col1"],"sort":[],"timeout":1,"filter":{"not":{"terms":{"col2":["val1","val2"]
+ },
+ {
+"name" : "Check not equals predicate",
+"hql" : "select col1 from index.type where col2 != 'val1'",
+"expectedJson" :
{"from":0,"size":1,"fields":["col1"],"sort":[],"timeout":1,"filter":{"not":{"term":{"col2":"val1"
}
- ]
+ ]
http://git-wip-us.apache.org/repos/asf/lens/blob/2901b4b3/lens-driver-jdbc/pom.xml
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2901b4b3/lens-server/pom.xml
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2901b4b3/pom.xml
--
[09/13] lens git commit: LENS-754 : Remove DISCLAIMER file from install and run docs
LENS-754 : Remove DISCLAIMER file from install and run docs Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/095fdb0c Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/095fdb0c Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/095fdb0c Branch: refs/heads/master Commit: 095fdb0ca98d1405c49a59fb217384d134729f62 Parents: 7881765 Author: Rajat Khandelwal Authored: Tue Aug 25 13:55:56 2015 +0530 Committer: raju.bairishetti Committed: Tue Aug 25 13:55:56 2015 +0530 -- src/site/apt/lenshome/install-and-run.apt | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/095fdb0c/src/site/apt/lenshome/install-and-run.apt -- diff --git a/src/site/apt/lenshome/install-and-run.apt b/src/site/apt/lenshome/install-and-run.apt index 0c55c5a..491bbee 100644 --- a/src/site/apt/lenshome/install-and-run.apt +++ b/src/site/apt/lenshome/install-and-run.apt @@ -46,7 +46,6 @@ Installing and Running Lens âââ LICENSE âââ NOTICE âââ README - âââ DISCLAIMER âââ LICENSE* âââ server +---+
[07/13] lens git commit: LENS-751 : Remove DISCLAIMER file
LENS-751 : Remove DISCLAIMER file Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9fa1cb51 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9fa1cb51 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9fa1cb51 Branch: refs/heads/master Commit: 9fa1cb515e03d302cc375f43ef5668471d7d6c90 Parents: fef9715 Author: Rajat Khandelwal Authored: Mon Aug 24 13:56:42 2015 +0530 Committer: Rajat Khandelwal Committed: Mon Aug 24 14:10:05 2015 +0530 -- lens-dist/src/main/assembly/bin-dist.xml | 7 --- 1 file changed, 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/9fa1cb51/lens-dist/src/main/assembly/bin-dist.xml -- diff --git a/lens-dist/src/main/assembly/bin-dist.xml b/lens-dist/src/main/assembly/bin-dist.xml index 9e2ce8f..8707af9 100644 --- a/lens-dist/src/main/assembly/bin-dist.xml +++ b/lens-dist/src/main/assembly/bin-dist.xml @@ -408,13 +408,6 @@ /server/webapp/ lens-server.war - - - ../DISCLAIMER - / - DISCLAIMER - -
[12/13] lens git commit: [maven-release-plugin] prepare for next development iteration
[maven-release-plugin] prepare for next development iteration Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1237842a Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1237842a Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1237842a Branch: refs/heads/master Commit: 1237842ab114b4fba7222492aa6e79b6e8d422b2 Parents: d3ba5a3 Author: Rajat Khandelwal Authored: Tue Aug 25 17:56:35 2015 +0530 Committer: Rajat Khandelwal Committed: Tue Aug 25 17:56:35 2015 +0530 -- checkstyle/pom.xml | 6 +- lens-api/pom.xml | 2 +- lens-cli/pom.xml | 2 +- lens-client/pom.xml | 2 +- lens-cube/pom.xml| 2 +- lens-dist/pom.xml| 2 +- lens-driver-es/pom.xml | 2 +- lens-driver-hive/pom.xml | 2 +- lens-driver-jdbc/pom.xml | 2 +- lens-examples/pom.xml| 2 +- lens-ml-dist/pom.xml | 2 +- lens-ml-lib/pom.xml | 2 +- lens-query-lib/pom.xml | 2 +- lens-regression/pom.xml | 2 +- lens-server-api/pom.xml | 2 +- lens-server/pom.xml | 2 +- lens-storage-db/pom.xml | 2 +- pom.xml | 8 18 files changed, 21 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/checkstyle/pom.xml -- diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index 545941f..bf15ca4 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -29,7 +29,7 @@ org.apache.lens checkstyle Lens Checkstyle Rules - 2.3.0-beta + 2.3.1-beta-SNAPSHOT @@ -49,8 +49,4 @@ - - -apache-lens-2.3.0-beta - http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-api/pom.xml -- diff --git a/lens-api/pom.xml b/lens-api/pom.xml index b1b2103..93bc1e7 100644 --- a/lens-api/pom.xml +++ b/lens-api/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-api http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-cli/pom.xml -- diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index 783215b..18c7af7 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-client/pom.xml -- diff --git a/lens-client/pom.xml b/lens-client/pom.xml index 052dd54..9c08a8a 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-client http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-cube/pom.xml -- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index 3a49b28..3fc784a 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-cube http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-dist/pom.xml -- diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml index 8bb998e..69a5fca 100644 --- a/lens-dist/pom.xml +++ b/lens-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-dist http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-driver-es/pom.xml -- diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index d450b13..f8af7af 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -24,7 +24,7 @@ org.apache.lens apache-lens -2.3.0-beta +2.3.1-beta-SNAPSHOT lens-driver-es http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-driver-hive/pom.xml -- diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml index a1c87da..ee9d92e 100644 --- a/lens-driver-hive/pom.xml +++ b/lens-driver-hive/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta +2.3.1-beta-SNAPSHOT http://git-wip-us.apache.org/repos/asf/lens/blob/1237842a/lens-driver-jdbc/pom.xml -- diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml index db560be..f88d143 100644 --- a/lens-driver-jdbc/pom.xml +++ b/lens-driver-jdbc/pom.xml @@ -2
[11/13] lens git commit: [maven-release-plugin] prepare release apache-lens-2.3.0-beta
[maven-release-plugin] prepare release apache-lens-2.3.0-beta Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d3ba5a38 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d3ba5a38 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d3ba5a38 Branch: refs/heads/master Commit: d3ba5a3845fdd1a90596f04ac979ab54bc1e65ad Parents: 8dcfcde Author: Rajat Khandelwal Authored: Tue Aug 25 17:56:09 2015 +0530 Committer: Rajat Khandelwal Committed: Tue Aug 25 17:56:09 2015 +0530 -- checkstyle/pom.xml | 6 +- lens-api/pom.xml | 2 +- lens-cli/pom.xml | 2 +- lens-client/pom.xml | 2 +- lens-cube/pom.xml| 2 +- lens-dist/pom.xml| 2 +- lens-driver-es/pom.xml | 5 ++--- lens-driver-hive/pom.xml | 2 +- lens-driver-jdbc/pom.xml | 17 - lens-examples/pom.xml| 2 +- lens-ml-dist/pom.xml | 2 +- lens-ml-lib/pom.xml | 2 +- lens-query-lib/pom.xml | 2 +- lens-regression/pom.xml | 2 +- lens-server-api/pom.xml | 2 +- lens-server/pom.xml | 19 +-- lens-storage-db/pom.xml | 2 +- pom.xml | 13 ++--- 18 files changed, 43 insertions(+), 43 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/checkstyle/pom.xml -- diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index 99928c3..545941f 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -29,7 +29,7 @@ org.apache.lens checkstyle Lens Checkstyle Rules - 2.3.0-beta-SNAPSHOT + 2.3.0-beta @@ -49,4 +49,8 @@ + + +apache-lens-2.3.0-beta + http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-api/pom.xml -- diff --git a/lens-api/pom.xml b/lens-api/pom.xml index 6a9517f..b1b2103 100644 --- a/lens-api/pom.xml +++ b/lens-api/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-api http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-cli/pom.xml -- diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index 22bfa1f..783215b 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-client/pom.xml -- diff --git a/lens-client/pom.xml b/lens-client/pom.xml index e726ead..052dd54 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-client http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-cube/pom.xml -- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index 394acef..3a49b28 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-cube http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-dist/pom.xml -- diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml index 9b0cda6..8bb998e 100644 --- a/lens-dist/pom.xml +++ b/lens-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-dist http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-driver-es/pom.xml -- diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index 178aaca..d450b13 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -19,13 +19,12 @@ under the License. --> -http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> 4.0.0 org.apache.lens apache-lens -2.3.0-beta-SNAPSHOT +2.3.0-beta lens-driver-es http://git-wip-us.apache.org/repos/asf/lens/blob/d3ba5a38/lens-driver-hive/pom.xml -- diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml index 7fe1a7a..a1c8
[02/13] lens git commit: LENS-351 : Document db resources feature in user guide
LENS-351 : Document db resources feature in user guide
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/67d69697
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/67d69697
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/67d69697
Branch: refs/heads/master
Commit: 67d6969705cbe4592a5055759ad9f5b35fa265e6
Parents: 0aad08f
Author: Rajat Khandelwal
Authored: Wed Aug 19 10:38:37 2015 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Aug 19 10:39:50 2015 +0530
--
src/site/apt/user/index.apt | 94
1 file changed, 67 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/67d69697/src/site/apt/user/index.apt
--
diff --git a/src/site/apt/user/index.apt b/src/site/apt/user/index.apt
index d48c734..6a86b1b 100644
--- a/src/site/apt/user/index.apt
+++ b/src/site/apt/user/index.apt
@@ -25,29 +25,29 @@ Lens User Guide
Lens Server runs several services which can be used from their REST
endpoints.
This document covers some of the important services, their purpose and
key API calls.
-
- Lens server provides metastore service for managing metadata. Metadata
exposed by lens
+
+ Lens server provides metastore service for managing metadata. Metadata
exposed by lens
is inspired by OLAP data cubes. See
{{{./olap-cube.html#Metastore_model} Metastore Model}} for metastore constructs
that lens provides.
-
+
Lens server also provides query service for querying data exposed by
lens. See
{{{./olap-cube.html#Query_Language} Query Language}} for the grammar of
the query.
-
- To access any service on lens, user should be working in a session.
User can also pass
+
+ To access any service on lens, user should be working in a session.
User can also pass
various configuration parameters from client.
-
- The sections below give more details on each service and configuration.
+
+ The sections below give more details on each service and configuration.
* Configuration
Client configuration can be overridden in lens-client-site.xml.
See {{{./client-config.html}client configuration}} for all the configuration
parameters
available and their default values.
-
+
* Session Service
- To use any Lens service the user must first create a session. Each Lens
session is associated with a unique session handle, which must be passed when
making queries, or doing metadata operations in the same session. To check if
the session service is deployed in the Lens Server, user can send a GET request
to /session. An OK response means that the session service is deployed.
-
+ To use any Lens service the user must first create a session. Each Lens
session is associated with a unique session handle, which must be passed when
making queries, or doing metadata operations in the same session. To check if
the session service is deployed in the Lens Server, user can send a GET request
to /session. An OK response means that the session service is deployed.
+
Sessions also allow users to set configuration or resources which could
be shared across a group of queries. For example, if a group of queries need to
call a UDF available in some specific jar, then the jar file can be added as a
resource in the session. All queries started within the same session can make
use of the jar file.
@@ -56,18 +56,58 @@ Lens User Guide
and {{{./cli.html#Session_management} CLI}} for doing all session level
operations.
The important API calls exposed by the session resource are -
-
- * {{{../resource_SessionResource.html#path__session.html}/session}} -
Endpoint to create or delete a session.
+
+ * {{{../resource_SessionResource.html#path__session.html}/session}} -
Endpoint to create or delete a session.
*
{{{../resource_SessionResource.html#path__session_params.html}/session/params}}
- Endpoint to maintain session settings
*
{{{../resource_SessionResource.html#path__session_resources_add.html}/session/resources}}
- Adding or removing resources from the session
+
+ While adding resources through client, you have to provide a path to
the resource file and type. The type can either
+ be <<>> or <<>>. The path has to be a path accessible by the
lens server. E.g. It can be
+ a hdfs path or a local path on lens server's file system. But it can't
be a path only accessible on the client
+ machine. Lens client doesn't upload the jar to the server, just passes
the path through. The path can be a a path
+ to a file/jar, a path to a directory or a regex path.
[08/13] lens git commit: LENS-744 : Fix server component documentation format
LENS-744 : Fix server component documentation format
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/78817658
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/78817658
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/78817658
Branch: refs/heads/master
Commit: 7881765808e221cee120a764eb31c509a0d1082f
Parents: 9fa1cb5
Author: Raju Bairishetti
Authored: Tue Aug 25 13:36:35 2015 +0530
Committer: raju.bairishetti
Committed: Tue Aug 25 13:50:34 2015 +0530
--
src/site/apt/admin/server-components.apt | 9 -
1 file changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/78817658/src/site/apt/admin/server-components.apt
--
diff --git a/src/site/apt/admin/server-components.apt
b/src/site/apt/admin/server-components.apt
index 98c536c..b3f91bc 100644
--- a/src/site/apt/admin/server-components.apt
+++ b/src/site/apt/admin/server-components.apt
@@ -31,6 +31,7 @@ Lens server components
* Lens services
** Session Management
+
All operations in Lens are performed in the context of a session. This
would essentially be the first
thing a client would need before any operation can be be performed in the
lens system. Session management
component facilitates creation, session specific configuration management,
session resource management
@@ -40,6 +41,7 @@ Lens server components
the session.
** Schema Management
+
This component allows for cubes, fact tables, dimension tables and storages
to be managed. While Storages can
be managed only by the administrator, other objects can be managed directly
by the lens users. Lens currently
extends and uses Hive Metastore for managing lens objects. Lens server has
to be configured appropriately to point
@@ -48,12 +50,14 @@ Lens server components
storage administration.
** Data Availability Management
+
Once schema is registered with Lens through the Schema management API, data
can be made available in the form of partitions.
The Lens data availability management (logical component) manages the
partitions available for each fact and maintains an
active timeline of data. This can be consulted by the query execution and
managment components. Like the Schema management
component, Lens currently uses the Hive metastore for data availability
management as well.
** Query life cycle management
+
Servicing user queries over cubes, fact, dimension or native tables across
different storages by choosing the best storage
and execution engine is the most critical capability of the lens. Lens
maintains state of queries submitted and tracks the progress
through the system and also maintains a history of queries submitted and
the its status. Unique query handle is issued to each
@@ -61,16 +65,19 @@ Lens server components
used for performing various analysis on the actual usage.
** Query schedule management (in progress)
+
Lens should allow for users to schedule their queries with some
periodicity. This scheduling capability should allow users to
express the time schedule. Additionally Lens server would apply gating
criteria based on data availability. Also any throttling
that needs to be effected would be considered before a scheduled query
moves into running state.
** Metrics Registry
+
Lens today includes support for instrumenting various functions at a server
level and at a query level through gauges and metrics.
This can be used to understand the performance characteristics of the lens
server and a single query.
** Pluggable query drivers
+
Lens server allows pluggable execution drivers for running queries.
Available execution engines are Hive and JDBC.
For configuring Hive as an execution engine, administrator should configure
the HiveServer2 end point. More details on
configuring multiple drivers will be covered in {{{./config-server.html}
configuration guide}}
-
+
[06/13] lens git commit: LENS-749 : Remove incubating from version number
LENS-749 : Remove incubating from version number Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fef97152 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fef97152 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fef97152 Branch: refs/heads/master Commit: fef9715287a776436047187f0cc1287ea3a718a0 Parents: f42fc65 Author: Rajat Khandelwal Authored: Mon Aug 24 11:05:32 2015 +0530 Committer: Amareshwari Sriramadasu Committed: Mon Aug 24 11:05:32 2015 +0530 -- checkstyle/pom.xml | 2 +- lens-api/pom.xml | 2 +- lens-cli/pom.xml | 2 +- lens-client/pom.xml| 2 +- lens-cube/pom.xml | 2 +- lens-dist/pom.xml | 2 +- lens-docker/lens-test/Dockerfile | 2 +- lens-docker/lens-test/lens-bootstrap.sh| 2 +- lens-driver-es/pom.xml | 2 +- lens-driver-hive/pom.xml | 2 +- lens-driver-jdbc/pom.xml | 2 +- lens-examples/pom.xml | 2 +- lens-ml-dist/pom.xml | 2 +- lens-ml-lib/pom.xml| 2 +- lens-query-lib/pom.xml | 2 +- lens-regression/pom.xml| 2 +- lens-server-api/pom.xml| 2 +- lens-server/pom.xml| 2 +- lens-storage-db/pom.xml| 2 +- pom.xml| 6 +++--- tools/conf-pseudo-distr/server/hivedriver-site.xml | 2 +- 21 files changed, 23 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/checkstyle/pom.xml -- diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml index b45e5fd..99928c3 100644 --- a/checkstyle/pom.xml +++ b/checkstyle/pom.xml @@ -29,7 +29,7 @@ org.apache.lens checkstyle Lens Checkstyle Rules - 2.3.0-beta-incubating-SNAPSHOT + 2.3.0-beta-SNAPSHOT http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/lens-api/pom.xml -- diff --git a/lens-api/pom.xml b/lens-api/pom.xml index 7b917b3..6a9517f 100644 --- a/lens-api/pom.xml +++ b/lens-api/pom.xml @@ -25,7 +25,7 @@ apache-lens org.apache.lens -2.3.0-beta-incubating-SNAPSHOT +2.3.0-beta-SNAPSHOT lens-api http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/lens-cli/pom.xml -- diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index 8d41d8f..22bfa1f 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-incubating-SNAPSHOT +2.3.0-beta-SNAPSHOT http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/lens-client/pom.xml -- diff --git a/lens-client/pom.xml b/lens-client/pom.xml index c799ca9..e726ead 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -24,7 +24,7 @@ apache-lens org.apache.lens -2.3.0-beta-incubating-SNAPSHOT +2.3.0-beta-SNAPSHOT lens-client http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/lens-cube/pom.xml -- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index fe10917..394acef 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-incubating-SNAPSHOT +2.3.0-beta-SNAPSHOT lens-cube http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/lens-dist/pom.xml -- diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml index 6105978..9b0cda6 100644 --- a/lens-dist/pom.xml +++ b/lens-dist/pom.xml @@ -26,7 +26,7 @@ apache-lens org.apache.lens -2.3.0-beta-incubating-SNAPSHOT +2.3.0-beta-SNAPSHOT lens-dist http://git-wip-us.apache.org/repos/asf/lens/blob/fef97152/lens-docker/lens-test/Dockerfile -- diff --git a/lens-docker/lens-test/Dockerfile b/lens-docker/lens-test/Dockerfile index 46252f1..42d291c 100644 --- a/lens-docker/lens-test/Dockerfile +++ b/lens-docker/lens-test/Dockerfile @@ -24,7 +24,7 @@ RUN mv spark-1.3.0-bin-hadoop2.4 /usr/local ENV SPARK_HOME /usr/local/spark-1.3.0-bin-hadoop2.4 RUN rm spark-1.3.0-bin-hadoop2.4.tar -ENV
[05/13] lens git commit: LENS-747 : Update documentation wrt tlp migration
LENS-747 : Update documentation wrt tlp migration Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f42fc654 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f42fc654 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f42fc654 Branch: refs/heads/master Commit: f42fc65458b710be58157971bb0e627bcf898413 Parents: 69fdaf7 Author: Rajat Khandelwal Authored: Mon Aug 24 10:58:15 2015 +0530 Committer: Amareshwari Sriramadasu Committed: Mon Aug 24 11:00:26 2015 +0530 -- DISCLAIMER | 6 NOTICE | 2 +- README.md | 6 ++-- bin-dist-files/NOTICE | 2 +- bin-dist-files/README | 6 ++-- .../org/apache/lens/api/util/PathValidator.java | 2 +- .../src/deb/control/client-control/control | 8 +++--- .../src/deb/control/server-control/control | 8 +++--- lens-server/enunciate.xml | 2 +- pom.xml | 28 +-- src/site/apt/developer/commit.apt | 6 ++-- src/site/apt/developer/contribute.apt | 10 +++ src/site/apt/index.apt | 14 ++ .../apt/lenshome/pseudo-distributed-setup.apt | 2 +- src/site/apt/lenshome/quick-start.apt | 6 ++-- src/site/apt/releases/compatibility.apt | 4 +-- src/site/apt/releases/download.apt | 10 ++- src/site/apt/releases/release-history.apt | 22 ++- src/site/apt/user/olap-cube.apt | 2 +- src/site/resources/images/apache-feather-tm.gif | Bin 0 -> 6735 bytes .../resources/images/apache-incubator-logo.png | Bin 4234 -> 0 bytes src/site/site.xml | 10 +++ 22 files changed, 79 insertions(+), 77 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/f42fc654/DISCLAIMER -- diff --git a/DISCLAIMER b/DISCLAIMER deleted file mode 100644 index ead62f5..000 --- a/DISCLAIMER +++ /dev/null @@ -1,6 +0,0 @@ -Apache Lens is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by Apache Incubator. - -Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, -communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate -that the project has yet to be fully endorsed by the ASF. http://git-wip-us.apache.org/repos/asf/lens/blob/f42fc654/NOTICE -- diff --git a/NOTICE b/NOTICE index 3351512..5dfb3c2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ -Apache Lens (incubating) +Apache Lens Copyright 2014-2015 The Apache Software Foundation http://git-wip-us.apache.org/repos/asf/lens/blob/f42fc654/README.md -- diff --git a/README.md b/README.md index cc3b0ef..d8657c1 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Build the project : # mvn clean package -See [Development Environment Setup] (http://lens.incubator.apache.org/developer/contribute.html#Development_Environment_Setup) -and [Building from source] (http://lens.incubator.apache.org/developer/contribute.html#Building_from_source) docs for +See [Development Environment Setup] (http://lens.apache.org/developer/contribute.html#Development_Environment_Setup) +and [Building from source] (http://lens.apache.org/developer/contribute.html#Building_from_source) docs for more details. -[Detailed documentation for the project is available here](https://lens.incubator.apache.org) +[Detailed documentation for the project is available here](https://lens.apache.org) http://git-wip-us.apache.org/repos/asf/lens/blob/f42fc654/bin-dist-files/NOTICE -- diff --git a/bin-dist-files/NOTICE b/bin-dist-files/NOTICE index 67e8da7..d731a68 100644 --- a/bin-dist-files/NOTICE +++ b/bin-dist-files/NOTICE @@ -1,4 +1,4 @@ -Apache Lens (incubating) +Apache Lens Copyright 2014-2015 The Apache Software Foundation http://git-wip-us.apache.org/repos/asf/lens/blob/f42fc654/bin-dist-files/README -- diff --git a/bin-dist-files/README b/bin-dist-files/README index 6cf9ad4..84dcdf1 100644 --- a/bin-dist-files/README +++ b/bin-dist-files/README @@ -6,10 +6,10 @@ across multiple tiered data stores a
[04/13] lens git commit: LENS-490 : Document partition timeline in user guide
LENS-490 : Document partition timeline in user guide
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/69fdaf70
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/69fdaf70
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/69fdaf70
Branch: refs/heads/master
Commit: 69fdaf70aabbbc13f238ec2d396ee2d39cf0e631
Parents: a7506f9
Author: Rajat Khandelwal
Authored: Thu Aug 20 09:44:12 2015 +0530
Committer: Raju Bairishetti
Committed: Thu Aug 20 09:44:12 2015 +0530
--
src/site/apt/user/olap-cube.apt | 308 ++-
1 file changed, 235 insertions(+), 73 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/69fdaf70/src/site/apt/user/olap-cube.apt
--
diff --git a/src/site/apt/user/olap-cube.apt b/src/site/apt/user/olap-cube.apt
index 87fd659..0255834 100644
--- a/src/site/apt/user/olap-cube.apt
+++ b/src/site/apt/user/olap-cube.apt
@@ -23,109 +23,251 @@ OLAP Data cube concepts
* Introduction
- Typical data layout in a data ware house is to have fact data rolled up with
time and
+ Typical data layout in a data ware house is to have fact data rolled up with
time and
reduce dimensions at each level. Fact data will have dimension keys sothat
it can be
joined with actual dimension tables to get more information on dimension
attributes.
-
+
The typical data layout is depicted in the following diagram.
-
+
[/images/data-layout.png] Fact data and dimension data
-
+
Lens provides an abstraction to represent above layout, and allows user to
define schema
of the data at conceptual level and also query the same, without knowing
the physical storages and rollups, which is described in below sections.
* Metastore model
- Metastore model introduces constructs Storage, Cube, Fact table and
Dimension table, Partition.
-
-** Storage
+ Metastore model introduces constructs Storage, Cube, Dimension, Fact table
and Dimtable, Partition. Below we'll
+ provide a brief indroduction of the constructs. You're welcome to checkout
the
+ {{{../apidocs/org/apache/lens/api/metastore/package-summary.html}javadoc}}.
You'll find corresponding classes for
+ each of the construct. The entities can be defined by either creating
objects of these classes, or by writing
+ <<>>s according to their schema. The schema is also available in
javadoc.
+
+ We have followed a convention in naming classes for constructs, class for a
Storage is called XStorage and the xml
+ root tag is <<>>. If storage is part of a bigger xml where root
tag is some other construct, then the tag
+ is <<>>. So in all <<>>s for lens, all and only the outer most
tag are <<>> and other tags are not.
+
+** {{{../apidocs/org/apache/lens/api/metastore/XStorage.html}Storage}}
- Storage represents a physical storage. It can be Hadoop File system or a
data base. It is defined by
+ Storage represents a physical storage. It can be Hadoop File system or a
data base. It is defined by
name, endpoint and properties associated with.
-** Cube
+** {{{../apidocs/org/apache/lens/api/metastore/XField.html}Field}}
- A cube is a set of dimensions and measures in a particular subject, which
users can query.
- Cubes also have timed dimensions, which can be used query cube for a time
range.
+ A Field has a name, a display string and a description. Field has the
following sub types
-*** Measure
+
+*** {{{../apidocs/org/apache/lens/api/metastore/XMeasure.html}Measure}}
A measure is a quantity that you are interested in measuring.
-
- Measure will have name, type, default aggregator, a format string, start
time and end time. Measure can be
- simple column measure or an expression measure which is associated with an
expression.
-
-*** Dimension
+ Measure is a field having a default aggregator, a format string, unit, start
time and end time. Can also have min and
+ max value.
+
+*** {{{../apidocs/org/apache/lens/api/metastore/XDimAttribute.html}Dim
Attribute}}
+
+ Dim Attributes are not measured, they are more like properties of your data.
e.g. Location, user name etc.
+
+ * A dim attribute can be as simple as having name and type. // Base Dim
Attribute
+
+ * A dim attribute can also refer to a dimension // Referenced Dim Atrribute
+
+ * A dim attribute can be associated with an expression // Expression Column
+
+ * A dim attribute can be a hierarchy of dim attributes. // Hierarchical Dim
Attribute
+
+ * A dim attribute can define all the possible values it can take. // Inline
Dim Attribute
+
+*** {{{../apidocs/org/apache/lens/api/metastore/XExprColumn.html}Expression
Column}}
+
+ Expression column has one or many
{{{../apidocs/org/apache/lens/api/metastore/XExprSpec.html}expression sp
svn commit: r1698353 - in /lens/site/publish: ./ admin/ apidocs/ apidocs/org/apache/lens/api/ apidocs/org/apache/lens/api/class-use/ apidocs/org/apache/lens/api/error/ apidocs/org/apache/lens/api/erro
Author: prongs Date: Fri Aug 28 15:19:35 2015 New Revision: 1698353 URL: http://svn.apache.org/r1698353 Log: Publish docs for 2.3.0-beta [This commit notification would consist of 478 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.]
lens git commit: LENS-710 : Allow column name mapping for few/all columns in underlying storage tables
Repository: lens
Updated Branches:
refs/heads/master 3576207a1 -> 3563aacf7
LENS-710 : Allow column name mapping for few/all columns in underlying storage
tables
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3563aacf
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3563aacf
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3563aacf
Branch: refs/heads/master
Commit: 3563aacf7c41a257d4c306153555099333ed5a47
Parents: 3576207
Author: Amareshwari Sriramadasu
Authored: Tue Sep 8 21:49:39 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Sep 8 21:49:39 2015 +0530
--
lens-api/src/main/resources/cube-0.1.xsd| 9 +
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 179 ++-
.../driver/jdbc/TestColumnarSQLRewriter.java| 91 +-
.../lens/server/api/LensConfConstants.java | 5 +
4 files changed, 232 insertions(+), 52 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/3563aacf/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git a/lens-api/src/main/resources/cube-0.1.xsd
b/lens-api/src/main/resources/cube-0.1.xsd
index 0a981dd..58f68f5 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -811,6 +811,15 @@
Table properties.
+The following properties can be specified for DBStorage table :
+1. lens.metastore.native.db.name : The underlying databse name in
DB storage.
+2. lens.metastore.native.table.name : The underlying table name in
DB storage.
+3. lens.metastore.native.table.column.mapping : The column mapping
for columns of the table if they are
+different in underlying DB storage. The value is specified with
comma separated map entries specified with
+key-values separated by equalto. Example value: id=id1,name=name1
+The following properties can be specified for Elastic search
tables :
+1. lens.metastore.es.index.name : The underlying ES index name.
+2. lens.metastore.es.type.name : The underlying ES type name.
http://git-wip-us.apache.org/repos/asf/lens/blob/3563aacf/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 9ceb9f3..295b476 100644
---
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.antlr.runtime.CommonToken;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
@@ -147,24 +149,31 @@ public class ColumnarSQLRewriter implements QueryRewriter
{
private String fromTree;
/** The join ast. */
+ @Getter
private ASTNode joinAST;
/** The having ast. */
+ @Getter
private ASTNode havingAST;
/** The select ast. */
+ @Getter
private ASTNode selectAST;
/** The where ast. */
+ @Getter
private ASTNode whereAST;
/** The order by ast. */
+ @Getter
private ASTNode orderByAST;
/** The group by ast. */
+ @Getter
private ASTNode groupByAST;
/** The from ast. */
+ @Getter
protected ASTNode fromAST;
/**
@@ -944,7 +953,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
*/
public void buildQuery(Configuration conf, HiveConf hconf) throws
SemanticException {
analyzeInternal(conf, hconf);
-replaceWithUnderlyingStorage(hconf, fromAST);
+replaceWithUnderlyingStorage(hconf);
replaceAliasInAST();
getFilterInJoinCond(fromAST);
getAggregateColumns(selectAST, new MutableInt(0));
@@ -1187,67 +1196,143 @@ public class ColumnarSQLRewriter implements
QueryRewriter {
return queryReplacedUdf;
}
- // Replace Lens database names with storage's proper DB and table name based
- // on table properties.
+
+ @NoArgsConstructor
+ private static class NativeTableInfo {
+private Map columnMapping = new HashMap<>();
+NativeTableInfo(Table tbl) {
+ String columnMappingProp =
tbl.getProperty(LensConfConstants.NATIVE_TABLE_COLUMN_MAPPING);
+ if (StringUtils.isNotBlank(columnMappingProp)) {
+String[] columnMapArray = StringUtils.split(columnMappingProp, ",");
+for (String columnMapEntry : columnMapArray) {
+ String[] mapEntry = StringUtils.split(colu
lens git commit: LENS-753: Queue number for queries in submitted queue
Repository: lens
Updated Branches:
refs/heads/master 3563aacf7 -> 4addd7b62
LENS-753: Queue number for queries in submitted queue
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4addd7b6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4addd7b6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4addd7b6
Branch: refs/heads/master
Commit: 4addd7b62ed1db8fe1af498a6baf55ee35ad692a
Parents: 3563aac
Author: Akshay Goyal
Authored: Tue Sep 15 16:29:32 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Sep 15 16:29:32 2015 +0530
--
.../org/apache/lens/api/query/QueryStatus.java | 38 +++
.../lens/cli/commands/LensQueryCommands.java| 18 +
.../server/api/driver/DriverQueryStatus.java| 7 ++--
.../server/api/query/FinishedLensQuery.java | 2 +-
.../lens/server/api/query/QueryContext.java | 2 +-
.../query/collect/ImmutableQueryCollection.java | 6 +++
.../server/query/QueryExecutionServiceImpl.java | 26 -
.../DefaultEstimatedQueryCollection.java| 5 +++
.../query/collect/DefaultQueryCollection.java | 36 --
.../ThreadSafeEstimatedQueryCollection.java | 5 +++
.../collect/ThreadSafeQueryCollection.java | 5 +++
.../collect/DefaultQueryCollectionTest.java | 34 -
.../server/query/collect/QueryCollectUtil.java | 39 +++-
13 files changed, 178 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/4addd7b6/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index 9614caa..3c8531f 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -39,6 +39,8 @@ import lombok.*;
*
* @param progress
* the progress
+ * @param queueNumber
+ * the queue number
* @param status
* the status
* @param statusMessage
@@ -121,6 +123,13 @@ public class QueryStatus implements Serializable {
private double progress;
/**
+ * Queue number of a query when it is in waiting state.
+ */
+ @Getter
+ @Setter
+ private Integer queueNumber;
+
+ /**
* The status.
*/
@XmlElement
@@ -165,20 +174,33 @@ public class QueryStatus implements Serializable {
*/
@Override
public String toString() {
-StringBuilder str = new
StringBuilder(status.toString()).append(':').append(statusMessage);
-if (status.equals(Status.RUNNING)) {
- str.append(" -
Progress:").append(progress).append(":").append(progressMessage);
+
+StringBuilder str = new StringBuilder(" Status :
").append(status.toString()).append("\n");
+if (statusMessage != null) {
+ str.append(" Message : ").append(statusMessage).append("\n");
+}
+
+str.append(" Progress : ").append(progress).append("\n");
+if (progressMessage != null) {
+ str.append(" Progress Message : ").append(progressMessage).append("\n");
}
+
+if (queueNumber != null) {
+ str.append(" Position in queue : ").append(queueNumber).append("\n");
+}
+
+if (errorMessage != null) {
+ str.append(" Error : ").append(errorMessage).append("\n");
+}
+
if (status.equals(Status.SUCCESSFUL)) {
if (isResultSetAvailable) {
-str.append(" - Result Available");
+str.append(" Result Available");
} else {
-str.append(" - Result Not Available");
+str.append(" Result Not Available");
}
}
-if (status.equals(Status.FAILED)) {
- str.append(" - Cause:").append(errorMessage);
-}
+
return str.toString();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/4addd7b6/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
--
diff --git
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 44c0c62..006eaed 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -156,26 +156,10 @@ public class LensQueryCommands extends BaseLensCommand {
public String getStatus(
@CliOption(key = {"", "query_handle"}, mandatory = true, help =
"") String qh) {
QueryStatus status = getClient().getQueryStatus(new
QueryHandle(UUID.fromString(qh)));
-StringBuilder sb = new StringBuilder();
if (status == null) {
return "Unable
lens git commit: LENS-756: Example query on customer_city_name failing
Repository: lens Updated Branches: refs/heads/master 08135aa69 -> c8701a38d LENS-756: Example query on customer_city_name failing Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c8701a38 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c8701a38 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c8701a38 Branch: refs/heads/master Commit: c8701a38da380d771f40ae953b48221eb46aa2ee Parents: 08135aa Author: Amareshwari Sriramadasu Authored: Wed Sep 16 17:22:23 2015 +0530 Committer: Rajat Khandelwal Committed: Wed Sep 16 17:22:23 2015 +0530 -- lens-examples/src/main/resources/sales-cube.xml | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/c8701a38/lens-examples/src/main/resources/sales-cube.xml -- diff --git a/lens-examples/src/main/resources/sales-cube.xml b/lens-examples/src/main/resources/sales-cube.xml index 9cc2fe6..7ec2ec7 100644 --- a/lens-examples/src/main/resources/sales-cube.xml +++ b/lens-examples/src/main/resources/sales-cube.xml @@ -58,7 +58,6 @@ display_string="Customer City"> -
lens git commit: LENS-775 : NPE on closing HadoopFileFormatter
Repository: lens
Updated Branches:
refs/heads/master 3862c384f -> f4c6c11e9
LENS-775 : NPE on closing HadoopFileFormatter
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f4c6c11e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f4c6c11e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f4c6c11e
Branch: refs/heads/master
Commit: f4c6c11e9637107a63493b98e3bb7d50966e52c1
Parents: 3862c38
Author: Deepak Barr
Authored: Thu Sep 17 19:12:17 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Sep 17 19:12:17 2015 +0530
--
.../main/java/org/apache/lens/lib/query/HadoopFileFormatter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/f4c6c11e/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
--
diff --git
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
index 87aff69..d556fb0 100644
---
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
+++
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
@@ -135,7 +135,9 @@ public class HadoopFileFormatter extends
AbstractFileFormatter {
*/
@Override
public void close() throws IOException {
-rowWriter.close(Reporter.NULL);
+if (null != rowWriter) {
+ rowWriter.close(Reporter.NULL);
+}
}
@Override
lens git commit: LENS-777: Fix: Waiting query selector change from intersecting to union
Repository: lens
Updated Branches:
refs/heads/master 66f164b47 -> 7ad12ed73
LENS-777: Fix: Waiting query selector change from intersecting to union
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7ad12ed7
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7ad12ed7
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7ad12ed7
Branch: refs/heads/master
Commit: 7ad12ed738abbf14d0b5bfe29dd45de209d32167
Parents: 66f164b
Author: Akshay Goyal
Authored: Wed Sep 23 17:39:34 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Sep 23 17:39:34 2015 +0530
--
.../server/query/QueryExecutionServiceImpl.java | 2 +-
.../IntersectingWaitingQueriesSelector.java | 107 ---
.../collect/UnioningWaitingQueriesSelector.java | 93 ++
.../IntersectingWaitingQueriesSelectorTest.java | 185 ---
.../UnioningWaitingQueriesSelectorTest.java | 135 ++
5 files changed, 229 insertions(+), 293 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7ad12ed7/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 2ba54e0..3bc9a4a 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1026,7 +1026,7 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
ImmutableSet selectionPolicies =
getImplementations(
WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, hiveConf);
-this.waitingQueriesSelector = new
IntersectingWaitingQueriesSelector(selectionPolicies);
+this.waitingQueriesSelector = new
UnioningWaitingQueriesSelector(selectionPolicies);
try {
this.userQueryToCubeQueryRewriter = new
UserQueryToCubeQueryRewriter(conf);
http://git-wip-us.apache.org/repos/asf/lens/blob/7ad12ed7/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
b/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
deleted file mode 100644
index fee4120..000
---
a/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.query.collect;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.lens.server.api.query.FinishedLensQuery;
-import org.apache.lens.server.api.query.QueryContext;
-import
org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import lombok.NonNull;
-
-/**
- * Selects queries eligible by all {@link WaitingQueriesSelectionPolicy} to
move them out of waiting state.
- *
- */
-public class IntersectingWaitingQueriesSelector implements
WaitingQueriesSelector {
-
- private final ImmutableSet selectionPolicies;
-
- public IntersectingWaitingQueriesSelector(
- @NonNull final ImmutableSet
selectionPolicies) {
-this.selectionPolicies = selectionPolicies;
- }
-
- /**
- * Selects queries eligible by all {@link WaitingQueriesSelectionPolicy} to
move them out of waiting state.
- *
- * @see WaitingQueriesSelector#se
lens git commit: LENS-772 : Fix time and size based rolling for logs
Repository: lens
Updated Branches:
refs/heads/master 7ad12ed73 -> d524cc24e
LENS-772 : Fix time and size based rolling for logs
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d524cc24
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d524cc24
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d524cc24
Branch: refs/heads/master
Commit: d524cc24eb93b1ad44c8c148f3d9aabff0740b54
Parents: 7ad12ed
Author: Amareshwari Sriramadasu
Authored: Wed Sep 23 17:41:41 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Sep 23 17:41:41 2015 +0530
--
tools/conf-pseudo-distr/client/logback.xml | 10 ---
tools/conf-pseudo-distr/server/logback.xml | 37 +
tools/conf/client/logback.xml | 10 ---
tools/conf/server/logback.xml | 37 +
4 files changed, 50 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d524cc24/tools/conf-pseudo-distr/client/logback.xml
--
diff --git a/tools/conf-pseudo-distr/client/logback.xml
b/tools/conf-pseudo-distr/client/logback.xml
index d142348..8617e17 100644
--- a/tools/conf-pseudo-distr/client/logback.xml
+++ b/tools/conf-pseudo-distr/client/logback.xml
@@ -33,14 +33,16 @@
-
${lens.log.dir}/lensclient.log.%d{-MM-dd}
+
${lens.log.dir}/lensclient.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
-
- 100MB
-
http://git-wip-us.apache.org/repos/asf/lens/blob/d524cc24/tools/conf-pseudo-distr/server/logback.xml
--
diff --git a/tools/conf-pseudo-distr/server/logback.xml
b/tools/conf-pseudo-distr/server/logback.xml
index c27bea2..63ab23c 100644
--- a/tools/conf-pseudo-distr/server/logback.xml
+++ b/tools/conf-pseudo-distr/server/logback.xml
@@ -31,18 +31,18 @@
true
-
${lens.log.dir}/lensserver.log.%d{-MM-dd}
-
+
${lens.log.dir}/lensserver.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
-
%d{dd MMM HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p
%c - %m%n
-
- 100MB
-
@@ -50,18 +50,18 @@
true
-
${lens.log.dir}/lensserver-requests.log.%d{-MM-dd}
-
+
${lens.log.dir}/lensserver-requests.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
-
%d{dd MMM HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p
%c - %m%n
-
- 100MB
-
@@ -69,8 +69,12 @@
true
-
${lens.log.dir}/lensserver-query-status.log.%d{-MM-dd}
-
+
${lens.log.dir}/lensserver-query-status.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
@@ -78,9 +82,6 @@
%d{dd MMM HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p
%c - %m%n
-
- 100MB
-
@@ -132,4 +133,4 @@
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/lens/blob/d524cc24/tools/conf/client/logback.xml
--
diff --git a/tools/conf/client/logback.xml b/tools/conf/client/logback.xml
index d142348..8617e17 100644
--- a/tools/conf/client/logback.xml
+++ b/tools/conf/client/logback.xml
@@ -33,14 +33,16 @@
-
${lens.log.dir}/lensclient.log.%d{-MM-dd}
+
${lens.log.dir}/lensclient.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
-
- 100MB
-
http://git-wip-us.apache.org/repos/asf/lens/blob/d524cc24/tools/conf/server/logback.xml
--
diff --git a/tools/conf/server/logback.xml b/tools/conf/server/logback.xml
index c27bea2..63ab23c 100644
--- a/tools/conf/server/logback.xml
+++ b/tools/conf/server/logback.xml
@@ -31,18 +31,18 @@
true
-
${lens.log.dir}/lensserver.log.%d{-MM-dd}
-
+
${lens.log.dir}/lensserver.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
-
%d{dd MMM HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p
%c - %m%n
-
- 100MB
-
@@ -50,18 +50,18 @@
true
-
${lens.log.dir}/lensserver-requests.log.%d{-MM-dd}
-
+
${lens.log.dir}/lensserver-requests.log.%d{-MM-dd}.%i
+
+
+10MB
+
30
-
%d{dd MMM HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p
%c - %m%n
-
- 100MB
-
@@ -69,8 +69,12 @@
lens git commit: LENS-752: Support flattening of columns selected through bridge-tables(many-to-many relationships)
Repository: lens
Updated Branches:
refs/heads/master d524cc24e -> b2c8fc02c
LENS-752: Support flattening of columns selected through
bridge-tables(many-to-many relationships)
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b2c8fc02
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b2c8fc02
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b2c8fc02
Branch: refs/heads/master
Commit: b2c8fc02c8e194eb9b11151fc4a805b6b77d68cf
Parents: d524cc2
Author: Amareshwari Sriramadasu
Authored: Fri Sep 25 12:16:06 2015 +0530
Committer: Rajat Khandelwal
Committed: Fri Sep 25 12:16:06 2015 +0530
--
lens-api/src/main/resources/cube-0.1.xsd| 8 +
.../apache/lens/cli/TestLensCubeCommands.java | 2 +-
.../apache/lens/cube/metadata/JoinChain.java| 6 +-
.../lens/cube/metadata/MetastoreUtil.java | 1 +
.../apache/lens/cube/metadata/SchemaGraph.java | 45 ++--
.../lens/cube/metadata/TableReference.java | 73 ++-
.../lens/cube/parse/CubeQueryConfUtil.java | 4 +
.../apache/lens/cube/parse/JoinResolver.java| 147 +++--
.../src/main/resources/olap-query-conf.xml | 16 ++
.../apache/lens/cube/parse/CubeTestSetup.java | 210 ++-
.../lens/cube/parse/TestJoinResolver.java | 193 -
.../apache/lens/server/metastore/JAXBUtils.java | 9 +-
.../server/metastore/TestMetastoreService.java | 4 +-
src/site/apt/user/olap-cube.apt | 99 +
src/site/apt/user/olap-query-conf.apt | 44 ++--
15 files changed, 714 insertions(+), 147 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b2c8fc02/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git a/lens-api/src/main/resources/cube-0.1.xsd
b/lens-api/src/main/resources/cube-0.1.xsd
index 58f68f5..5d7630d 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -521,6 +521,14 @@
+
+
+
+ This boolean is used to specify if the destination reference can map
to many rows for single value of source
+ reference. Mainly signifies if there are bridge tables.
+
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/b2c8fc02/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
index 39441c9..b553739 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
@@ -103,7 +103,6 @@ public class TestLensCubeCommands extends
LensCliApplicationTest {
chain1.setName("testdetailchain");
chain1.getPaths().getPath().add(path);
chain1.setDestTable("test_detail");
-chains.getJoinChain().add(chain1);
XJoinChain chain2 = new XJoinChain();
chain2.setPaths(new XJoinPaths());
XJoinPath path2 = new XJoinPath();
@@ -122,6 +121,7 @@ public class TestLensCubeCommands extends
LensCliApplicationTest {
chain2.getPaths().getPath().add(path2);
chain2.setDestTable("test_dim");
chains.getJoinChain().add(chain2);
+chains.getJoinChain().add(chain1);
assertEquals(joinChains, new XJoinChainTable(chains).toString());
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b2c8fc02/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
index e394e20..6250905 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
@@ -178,7 +178,8 @@ public class JoinChain implements Named {
relationShip = new TableRelationship(from.getDestColumn(),
client.getDimension(from.getDestTable()),
to.getDestColumn(),
- client.getDimension(to.getDestTable()));
+ client.getDimension(to.getDestTable()),
+ to.isMapsToMany());
}
return relationShip;
}
@@ -203,7 +204,8 @@ public class JoinChain implements Named {
relationShip = new TableRelationship(from.getDestColumn(),
fromTable,
to.getDestColumn(),
-client.getDimension(to.getDestTable()));
+client.getDimension(to.getDestTable()),
+to.isMapsToMany());
}
}
ret
lens git commit: LENS-576 : Cannot retrieve query results on lens server restart
Repository: lens
Updated Branches:
refs/heads/master b2c8fc02c -> 5bae273ee
LENS-576 : Cannot retrieve query results on lens server restart
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5bae273e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5bae273e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5bae273e
Branch: refs/heads/master
Commit: 5bae273eef68396e42a3678b1182b529c6e42d69
Parents: b2c8fc0
Author: Deepak Barr
Authored: Fri Sep 25 13:39:44 2015 +0530
Committer: Rajat Khandelwal
Committed: Fri Sep 25 13:39:44 2015 +0530
--
.../lens/lib/query/AbstractFileFormatter.java | 18 +
.../lens/lib/query/WrappedFileFormatter.java| 13
.../lib/query/MockLensResultSetMetadata.java| 65 +
.../lib/query/TestAbstractFileFormatter.java| 73 +++-
.../api/driver/LensResultSetMetadata.java | 49 +
.../lens/server/api/query/QueryContext.java | 2 +-
.../server/api/query/QueryOutputFormatter.java | 3 +-
.../lens/server/query/LensPersistentResult.java | 6 +-
.../server/query/QueryExecutionServiceImpl.java | 39 +--
9 files changed, 193 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/5bae273e/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
--
diff --git
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
index ae5af03..8c06621 100644
---
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
+++
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
@@ -19,6 +19,8 @@
package org.apache.lens.lib.query;
import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.query.QueryContext;
@@ -90,4 +92,20 @@ public abstract class AbstractFileFormatter extends
AbstractOutputFormatter impl
public String getFinalOutputPath() {
return finalPath.toString();
}
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+out.writeUTF(metadata.toJson());
+out.writeUTF(finalPath.toString());
+out.writeInt(numRows);
+out.writeLong(fileSize);
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+metadata = LensResultSetMetadata.fromJson(in.readUTF());
+finalPath = new Path(in.readUTF());
+numRows = in.readInt();
+fileSize = in.readLong();
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5bae273e/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
--
diff --git
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
index e28c17b..5910709 100644
---
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
+++
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
@@ -19,6 +19,8 @@
package org.apache.lens.lib.query;
import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.query.QueryContext;
@@ -137,4 +139,15 @@ public abstract class WrappedFileFormatter extends
AbstractOutputFormatter {
public String getEncoding() {
return formatter.getEncoding();
}
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+out.writeObject(formatter);
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
+formatter = (AbstractFileFormatter) in.readObject();
+metadata = formatter.getMetadata();
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5bae273e/lens-query-lib/src/test/java/org/apache/lens/lib/query/MockLensResultSetMetadata.java
--
diff --git
a/lens-query-lib/src/test/java/org/apache/lens/lib/query/MockLensResultSetMetadata.java
b/lens-query-lib/src/test/java/org/apache/lens/lib/query/MockLensResultSetMetadata.java
new file mode 100644
index 000..3b6c0e4
--- /dev/null
+++
b/lens-query-lib/src/test/java/org/apache/lens/lib/query/MockLensResultSetMetadata.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.
lens git commit: LENS-224: FinishedLensQuery is not storing selectedDriver
Repository: lens
Updated Branches:
refs/heads/master 5bae273ee -> e5b793c83
LENS-224: FinishedLensQuery is not storing selectedDriver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e5b793c8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e5b793c8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e5b793c8
Branch: refs/heads/master
Commit: e5b793c83f2a627d049fac7de2ceee7a730574de
Parents: 5bae273
Author: Deepak Barr
Authored: Mon Sep 28 19:03:15 2015 +0530
Committer: Rajat Khandelwal
Committed: Mon Sep 28 19:03:15 2015 +0530
--
.../server/api/query/FinishedLensQuery.java | 34 ++--
.../apache/lens/server/query/LensServerDAO.java | 10 +++---
2 files changed, 37 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/e5b793c8/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 7a06c44..8308198 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -19,6 +19,7 @@
package org.apache.lens.server.api.query;
import java.util.Collection;
+import java.util.Iterator;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.QueryHandle;
@@ -164,6 +165,13 @@ public class FinishedLensQuery {
@Setter
private String queryName;
+ /**
+ * The selected driver class name.
+ */
+ @Getter
+ @Setter
+ private String driverClass;
+
@Getter
private LensDriver selectedDriver;
@@ -195,11 +203,21 @@ public class FinishedLensQuery {
this.queryName = ctx.getQueryName().toLowerCase();
}
this.selectedDriver = ctx.getSelectedDriver();
+if (null != ctx.getSelectedDriver()) {
+ this.driverClass = ctx.getSelectedDriver().getClass().getName();
+}
}
public QueryContext toQueryContext(Configuration conf,
Collection drivers) {
-QueryContext qctx = new QueryContext(userQuery, submitter, new LensConf(),
conf, drivers, null, submissionTime,
- false);
+
+if (null == selectedDriver && null != driverClass) {
+ selectedDriver = getDriverFromClassName(drivers);
+}
+
+QueryContext qctx =
+ new QueryContext(userQuery, submitter, new LensConf(), conf, drivers,
selectedDriver, submissionTime,
+false);
+
qctx.setQueryHandle(QueryHandle.fromString(handle));
qctx.setLaunchTime(this.startTime);
qctx.setEndTime(getEndTime());
@@ -212,6 +230,18 @@ public class FinishedLensQuery {
return qctx;
}
+ private LensDriver getDriverFromClassName(Collection drivers) {
+Iterator iterator = drivers.iterator();
+while (iterator.hasNext()) {
+ LensDriver driver = iterator.next();
+ if (driverClass.equals(driver.getClass().getName())) {
+//TODO : LENS-123 - Ability to load different instances of same driver
class
+return driver;
+ }
+}
+return null;
+ }
+
public ImmutableSet
getDriverSelectionPolicies() {
return this.selectedDriver.getWaitingQuerySelectionPolicies();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e5b793c8/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 59b1221..8a2ac19 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -85,8 +85,8 @@ public class LensServerDAO {
+ "userquery varchar(1) not null," + "submitter varchar(255) not
null," + "starttime bigint, "
+ "endtime bigint," + "result varchar(255)," + "status varchar(255), " +
"metadata varchar(10), "
+ "rows int, " + "filesize bigint, " + "errormessage varchar(1), " +
"driverstarttime bigint, "
- + "driverendtime bigint, " + "metadataclass varchar(1), " +
"queryname varchar(255), "
- + "submissiontime bigint" + ")";
+ + "driverendtime bigint, " + "metadataclass varchar(1), " +
"driverclass varchar(1), "
+ + "queryname varchar(255), " + "submissiontime bigint" + ")";
try {
QueryRunner runner = new QueryRunner(ds);
runner.update(sql);
@@ -109,15 +109,15 @@ public class LensServerDAO {
Connection conn = null;
Stri
lens git commit: LENS-113 : (Incremental) Document how to add feature documentation
Repository: lens
Updated Branches:
refs/heads/master e5b793c83 -> e18488105
LENS-113 : (Incremental) Document how to add feature documentation
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e1848810
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e1848810
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e1848810
Branch: refs/heads/master
Commit: e1848810591dd9c31e0993dbed977bcf0c627643
Parents: e5b793c
Author: Amareshwari Sriramadasu
Authored: Thu Oct 1 15:58:37 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Oct 1 15:58:37 2015 +0530
--
src/site/apt/developer/contribute.apt | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/e1848810/src/site/apt/developer/contribute.apt
--
diff --git a/src/site/apt/developer/contribute.apt
b/src/site/apt/developer/contribute.apt
index 6183b07..5b6fbaf 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -509,8 +509,14 @@ Developer Documentation : How to contribute to Apache Lens?
* Who are the users of feature and who are not, if any.
+ * Illustration with example would be very welcome
+
The design documentation related to the feature can go in developer
documentation or
- {{{https://cwiki.apache.org/confluence/display/LENS/DesignDocs} Design
docs}}.
+ {{{https://cwiki.apache.org/confluence/display/LENS/DesignDocs} Design
docs}}. It would be necessary to add in which
+ version the feature is available if the documentation is in confluence. Also
when behavior is modified or improved on
+ existing features, version tagging is quite useful. Any defaults (in terms
of config or behavior) assumed with the
+ feature should also be highlighted. Configuration descriptions should be
linked to config apt files, so that they
+ are always in sync with code.
** Confluence usage
lens git commit: LENS-790: UI Module failing if maven version is < 3.1
Repository: lens
Updated Branches:
refs/heads/master e18488105 -> ebb2c0366
LENS-790: UI Module failing if maven version is < 3.1
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ebb2c036
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ebb2c036
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ebb2c036
Branch: refs/heads/master
Commit: ebb2c036634b8257ffa007ddeb6c1e2178fc0ae2
Parents: e184881
Author: Ankeet Maini
Authored: Mon Oct 5 13:36:28 2015 +0530
Committer: Rajat Khandelwal
Committed: Mon Oct 5 13:36:28 2015 +0530
--
lens-ui/pom.xml | 36
pom.xml | 21 ++---
2 files changed, 22 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/ebb2c036/lens-ui/pom.xml
--
diff --git a/lens-ui/pom.xml b/lens-ui/pom.xml
index 69bcee5..b1ad565 100644
--- a/lens-ui/pom.xml
+++ b/lens-ui/pom.xml
@@ -35,42 +35,14 @@
-org.apache.maven.plugins
-maven-antrun-plugin
-${antrun.plugin.version}
-
-
-
-
-
-
-
-com.github.eirslett
-frontend-maven-plugin
-${frontend.maven.plugin}
+io.wcm.maven.plugins
+nodejs-maven-plugin
-install node and npm
-
- install-node-and-npm
-
-
- ${nodeVersion}
- ${npmVersion}
- https://nodejs.org/dist/
-
http://registry.npmjs.org/npm/-/
- node
-
-
-
-npm install
+compile
- npm
+ run
-
-
- install
-
http://git-wip-us.apache.org/repos/asf/lens/blob/ebb2c036/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dd26713..3ec5e5e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -108,9 +108,9 @@
2.7
-0.0.23
-v4.0.0
-2.7.6
+1.0.0
+0.10.32
+1.4.9
${project.build.directory}/debian
@@ -525,6 +525,21 @@
findbugs-maven-plugin
${findbugs.plugin.version}
+
+ io.wcm.maven.plugins
+ nodejs-maven-plugin
+ ${nodejs.plugin.version}
+
+${npmVersion}
+${nodeVersion}
+${project.basedir}/node
+
+
+${project.basedir}
+
+
+
+
lens git commit: LENS-795: Look ahead is failing with partition not found if any process time partitions are absent
Repository: lens
Updated Branches:
refs/heads/master ebb2c0366 -> aeaae6815
LENS-795: Look ahead is failing with partition not found if any process time
partitions are absent
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/aeaae681
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/aeaae681
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/aeaae681
Branch: refs/heads/master
Commit: aeaae681523989c798c059bb4345aed106bd5fe1
Parents: ebb2c03
Author: Rajat Khandelwal
Authored: Mon Oct 5 13:37:48 2015 +0530
Committer: Rajat Khandelwal
Committed: Mon Oct 5 13:37:48 2015 +0530
--
.../PartitionRangesForPartitionColumns.java | 6 --
.../lens/cube/parse/StorageTableResolver.java| 19 +++
2 files changed, 19 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/aeaae681/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
index b389a8f..fa3f6ef 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
@@ -41,10 +41,12 @@ public class PartitionRangesForPartitionColumns extends
HashMap toSet() {
+ public Set toSet(Set partColsQueried) {
Set ret = Sets.newHashSet();
for (Map.Entry entry : entrySet()) {
- ret.add(entry.getKey() + ":" + entry.getValue().getRanges());
+ if (partColsQueried.contains(entry.getKey())) {
+ret.add(entry.getKey() + ":" + entry.getValue().getRanges());
+ }
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/aeaae681/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 68ab5ab..f67fc26 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -363,7 +363,9 @@ class StorageTableResolver implements ContextRewriter {
PartitionRangesForPartitionColumns missingParts = new
PartitionRangesForPartitionColumns();
boolean noPartsForRange = false;
Set unsupportedTimeDims = Sets.newHashSet();
+ Set partColsQueried = Sets.newHashSet();
for (TimeRange range : cubeql.getTimeRanges()) {
+partColsQueried.add(range.getPartitionColumn());
StringBuilder extraWhereClause = new StringBuilder();
Set rangeParts = getPartitions(cfact.fact, range,
skipStorageCauses, missingParts);
// If no partitions were found, then we'll fallback.
@@ -389,6 +391,7 @@ class StorageTableResolver implements ContextRewriter {
if (fallBackRange == null) {
break;
}
+ partColsQueried.add(fallBackRange.getPartitionColumn());
rangeParts = getPartitions(cfact.fact, fallBackRange,
skipStorageCauses, missingParts);
extraWhereClause.append(sep)
.append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()),
timeDim));
@@ -433,7 +436,7 @@ class StorageTableResolver implements ContextRewriter {
i.remove();
continue;
}
- Set nonExistingParts = missingParts.toSet();
+ Set nonExistingParts = missingParts.toSet(partColsQueried);
if (!nonExistingParts.isEmpty()) {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
}
@@ -637,8 +640,10 @@ class StorageTableResolver implements ContextRewriter {
for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) {
FactPartition innerPart = new FactPartition(partCol, date,
pPart.getPeriod(), pPart,
partWhereClauseFormat);
-updateFactPartitionStorageTablesFrom(fact, innerPart,
pPart.getStorageTables());
-partitions.add(innerPart);
+updateFactPartitionStorageTablesFrom(fact, innerPart,
pPart);
+if (innerPart.isFound()) {
+ partitions.add(innerPart);
+}
}
log.debug("added all sub partitions blindly in pPart: {}",
pPart);
}
@@ -678,7 +683,7 @@ class StorageTableResolver implemen
lens git commit: LENS-796 : Slow response times for /metastore/nativetables API
Repository: lens
Updated Branches:
refs/heads/master aeaae6815 -> 5c9bae096
LENS-796 : Slow response times for /metastore/nativetables API
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5c9bae09
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5c9bae09
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5c9bae09
Branch: refs/heads/master
Commit: 5c9bae096351960f4790e11f76a5d7665d8e8d3d
Parents: aeaae68
Author: Deepak Barr
Authored: Tue Oct 6 12:52:29 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Oct 6 12:52:29 2015 +0530
--
.../metastore/CubeMetastoreServiceImpl.java | 60 ++--
1 file changed, 30 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/5c9bae09/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 37cebfe..4c8b525 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -38,14 +38,13 @@ import
org.apache.lens.server.api.metastore.CubeMetastoreService;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.thrift.TException;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
@@ -1198,28 +1197,37 @@ public class CubeMetastoreServiceImpl extends
BaseLensService implements CubeMet
}
}
- private List getTablesFromDB(LensSessionHandle sessionid,
-String dbName, boolean prependDbName)
-throws MetaException, UnknownDBException, HiveSQLException, TException,
LensException {
-List tables =
getSession(sessionid).getMetaStoreClient().getAllTables(
- dbName);
-List result = new ArrayList();
-if (tables != null && !tables.isEmpty()) {
- Iterator it = tables.iterator();
- while (it.hasNext()) {
-String tblName = it.next();
-org.apache.hadoop.hive.metastore.api.Table tbl =
- getSession(sessionid).getMetaStoreClient().getTable(dbName, tblName);
-if (tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY) ==
null) {
- if (prependDbName) {
-result.add(dbName + "." + tblName);
- } else {
-result.add(tblName);
+ private List getNativeTablesFromDB(LensSessionHandle sessionid,
String dbName, boolean prependDbName)
+throws LensException {
+IMetaStoreClient msc = null;
+try {
+ msc = getSession(sessionid).getMetaStoreClient();
+ List tables = msc.getAllTables(
+dbName);
+ List result = new ArrayList();
+ if (tables != null && !tables.isEmpty()) {
+List tblObjects =
+ msc.getTableObjectsByName(dbName, tables);
+Iterator it =
tblObjects.iterator();
+while (it.hasNext()) {
+ org.apache.hadoop.hive.metastore.api.Table tbl = it.next();
+ if (tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY) ==
null) {
+if (prependDbName) {
+ result.add(dbName + "." + tbl.getTableName());
+} else {
+ result.add(tbl.getTableName());
+}
}
}
}
+ return result;
+} catch (Exception e) {
+ throw new LensException("Error getting native tables from DB", e);
+} finally {
+ if (null != msc) {
+msc.close();
+ }
}
-return result;
}
@Override
@@ -1240,24 +1248,16 @@ public class CubeMetastoreServiceImpl extends
BaseLensService implements CubeMet
}
List tables;
if (!StringUtils.isBlank(dbName)) {
-tables = getTablesFromDB(sessionid, dbName, false);
+tables = getNativeTablesFromDB(sessionid, dbName, false);
} else {
log.info("Getting tables from all dbs");
List alldbs = getAllDatabases(sessionid);
tables = new ArrayList();
for (String db : alldbs) {
- tables.addAll(getTablesFromDB(sessionid, db, true));
+ tables.addAll(getNativeTablesFromDB(sessionid, db, true));
lens git commit: LENS-792: Improve StatusPoller logging to reduce noise
Repository: lens
Updated Branches:
refs/heads/master 5c9bae096 -> 2ecef112f
LENS-792: Improve StatusPoller logging to reduce noise
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2ecef112
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2ecef112
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2ecef112
Branch: refs/heads/master
Commit: 2ecef112f49aa45521a4b5a81802d265d0e34129
Parents: 5c9bae0
Author: Rajat Khandelwal
Authored: Tue Oct 6 15:35:59 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Oct 6 15:35:59 2015 +0530
--
.../main/java/org/apache/lens/server/api/query/QueryContext.java | 3 +++
.../org/apache/lens/server/query/QueryExecutionServiceImpl.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2ecef112/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 3a5b887..ac94207 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -43,11 +43,13 @@ import com.google.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class QueryContext.
*/
@ToString
+@Slf4j
public class QueryContext extends AbstractQueryContext {
/**
@@ -334,6 +336,7 @@ public class QueryContext extends AbstractQueryContext {
public synchronized void setStatus(final QueryStatus newStatus) throws
LensException {
validateTransition(newStatus);
+log.info("Updating status of {} from {} to {}", getQueryHandle(),
this.status, newStatus);
this.status = newStatus;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/2ecef112/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index ea19742..73179e1 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -670,7 +670,7 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
}
logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString());
-log.info("Polling status for {}", ctx.getQueryHandle());
+log.debug("Polling status for {}", ctx.getQueryHandle());
try {
// session is not required to update status of the query
// don't need to wrap this with acquire/release
@@ -770,7 +770,7 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
synchronized (ctx) {
QueryStatus before = ctx.getStatus();
if (!ctx.queued() && !ctx.finished() &&
!ctx.getDriverStatus().isFinished()) {
- log.info("Updating status for {}", ctx.getQueryHandle());
+ log.debug("Updating status for {}", ctx.getQueryHandle());
try {
ctx.getSelectedDriver().updateStatus(ctx);
ctx.setStatus(ctx.getDriverStatus().toQueryStatus());
lens git commit: LENS-739: (license fix) Generate udf jar for lens-regression
Repository: lens Updated Branches: refs/heads/master d4d4aa26a -> dd33a6058 LENS-739: (license fix) Generate udf jar for lens-regression Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dd33a605 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dd33a605 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dd33a605 Branch: refs/heads/master Commit: dd33a60589ca1a8efc71c2ceb73af22654b584f3 Parents: d4d4aa2 Author: Archana H Authored: Thu Oct 8 19:32:46 2015 +0530 Committer: Rajat Khandelwal Committed: Thu Oct 8 19:32:46 2015 +0530 -- .../src/additional/java/SampleUdf.java | 19 +++ 1 file changed, 19 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/dd33a605/lens-regression/src/additional/java/SampleUdf.java -- diff --git a/lens-regression/src/additional/java/SampleUdf.java b/lens-regression/src/additional/java/SampleUdf.java index 2c3e667..e340ffd 100644 --- a/lens-regression/src/additional/java/SampleUdf.java +++ b/lens-regression/src/additional/java/SampleUdf.java @@ -1,3 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.io.Text;
lens git commit: LENS-817 : No need to persist metadata class information in DB after LENS-576 fix
Repository: lens
Updated Branches:
refs/heads/master 9497c8367 -> f29c2f240
LENS-817 : No need to persist metadata class information in DB after LENS-576
fix
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f29c2f24
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f29c2f24
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f29c2f24
Branch: refs/heads/master
Commit: f29c2f2401238531d624a2438dd3e7911ab92014
Parents: 9497c83
Author: Deepak Barr
Authored: Fri Oct 16 18:18:30 2015 +0530
Committer: Rajat Khandelwal
Committed: Fri Oct 16 18:18:30 2015 +0530
--
.../org/apache/lens/server/api/query/FinishedLensQuery.java | 7 ---
.../java/org/apache/lens/server/query/LensServerDAO.java | 8
.../apache/lens/server/query/QueryExecutionServiceImpl.java | 1 -
.../test/java/org/apache/lens/server/query/TestLensDAO.java | 1 -
4 files changed, 4 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/f29c2f24/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 8308198..89053aa 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -152,13 +152,6 @@ public class FinishedLensQuery {
private long driverEndTime;
/**
- * The metadata class.
- */
- @Getter
- @Setter
- private String metadataClass;
-
- /**
* The query name.
*/
@Getter
http://git-wip-us.apache.org/repos/asf/lens/blob/f29c2f24/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 8a2ac19..05bee9e 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -85,7 +85,7 @@ public class LensServerDAO {
+ "userquery varchar(1) not null," + "submitter varchar(255) not
null," + "starttime bigint, "
+ "endtime bigint," + "result varchar(255)," + "status varchar(255), " +
"metadata varchar(10), "
+ "rows int, " + "filesize bigint, " + "errormessage varchar(1), " +
"driverstarttime bigint, "
- + "driverendtime bigint, " + "metadataclass varchar(1), " +
"driverclass varchar(1), "
+ + "driverendtime bigint, " + "driverclass varchar(1), "
+ "queryname varchar(255), " + "submissiontime bigint" + ")";
try {
QueryRunner runner = new QueryRunner(ds);
@@ -109,15 +109,15 @@ public class LensServerDAO {
Connection conn = null;
String sql = "insert into finished_queries (handle, userquery,submitter,"
+ "starttime,endtime,result,status,metadata,rows,filesize,"
-+ "errormessage,driverstarttime,driverendtime, metadataclass,
driverclass, queryname, submissiontime)"
-+ " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
++ "errormessage,driverstarttime,driverendtime, driverclass, queryname,
submissiontime)"
++ " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
conn = getConnection();
QueryRunner runner = new QueryRunner();
runner.update(conn, sql, query.getHandle(), query.getUserQuery(),
query.getSubmitter(), query.getStartTime(),
query.getEndTime(), query.getResult(), query.getStatus(),
query.getMetadata(), query.getRows(),
query.getFileSize(), query.getErrorMessage(),
query.getDriverStartTime(), query.getDriverEndTime(),
- query.getMetadataClass(), query.getDriverClass(),
query.getQueryName(), query.getSubmissionTime());
+ query.getDriverClass(), query.getQueryName(),
query.getSubmissionTime());
conn.commit();
} finally {
DbUtils.closeQuietly(conn);
http://git-wip-us.apache.org/repos/asf/lens/blob/f29c2f24/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index d57812f..deb8572 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceIm
lens git commit: LENS-292 : Query result retention policy
Repository: lens
Updated Branches:
refs/heads/master 13dc803c7 -> b54c4b999
LENS-292 : Query result retention policy
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b54c4b99
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b54c4b99
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b54c4b99
Branch: refs/heads/master
Commit: b54c4b9993bdfc5c79e6c9ab727e908d294a61dc
Parents: 13dc803
Author: Deepak Barr
Authored: Wed Oct 21 15:51:59 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Oct 21 15:51:59 2015 +0530
--
.../org/apache/lens/cube/parse/DateUtil.java| 4 +-
.../lens/server/api/LensConfConstants.java | 40 +
.../server/query/QueryExecutionServiceImpl.java | 22 +++
.../lens/server/query/QueryResultPurger.java| 177 +++
.../src/main/resources/lensserver-default.xml | 25 +++
.../server/query/TestQueryResultPurger.java | 100 +++
src/site/apt/admin/config.apt | 8 +
7 files changed, 374 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b54c4b99/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
index cbcc76c..67932da 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
@@ -432,7 +432,7 @@ public final class DateUtil {
}
@EqualsAndHashCode
- static class TimeDiff {
+ public static class TimeDiff {
int quantity;
UpdatePeriod updatePeriod;
@@ -441,7 +441,7 @@ public final class DateUtil {
this.updatePeriod = updatePeriod;
}
-static TimeDiff parseFrom(String diffStr) throws LensException {
+public static TimeDiff parseFrom(String diffStr) throws LensException {
// Get the relative diff part to get eventual date based on now.
Matcher qtyMatcher = P_QUANTITY.matcher(diffStr);
int qty = 1;
http://git-wip-us.apache.org/repos/asf/lens/blob/b54c4b99/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index f202603..7ee0749 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -942,4 +942,44 @@ public final class LensConfConstants {
*/
public static final int DEFAULT_FETCH_COUNT_SAVED_QUERY_LIST = 20;
+ /**
+ * The Constant RESULTSET_PURGE_ENABLED.
+ */
+ public static final String RESULTSET_PURGE_ENABLED = SERVER_PFX +
"resultset.purge.enabled";
+
+ /**
+ * The Constant DEFAULT_RESULTSET_PURGE_ENABLED
+ */
+ public static final boolean DEFAULT_RESULTSET_PURGE_ENABLED = false;
+
+ /**
+ * The Constant RESULTSET_PURGE_INTERVAL_IN_SECONDS.
+ */
+ public static final String RESULTSET_PURGE_INTERVAL_IN_SECONDS = SERVER_PFX
+ "resultsetpurger.sleep.interval.secs";
+
+ /*
+ * The Constant DEFAULT_RESULTSET_PURGE_INTERVAL_IN_SECONDS.
+ */
+ public static final int DEFAULT_RESULTSET_PURGE_INTERVAL_IN_SECONDS = 3600;
+
+ /**
+ * The Constant QUERY_RESULTSET_RETENTION.
+ */
+ public static final String QUERY_RESULTSET_RETENTION = SERVER_PFX +
"query.resultset.retention";
+
+ /**
+ * The Constant DEFAULT_QUERY_RESULTSET_RETENTION.
+ */
+ public static final String DEFAULT_QUERY_RESULTSET_RETENTION = "1 day";
+
+ /**
+ * The Constant HDFS_OUTPUT_RETENTION.
+ */
+ public static final String HDFS_OUTPUT_RETENTION = SERVER_PFX +
"hdfs.output.retention";
+
+ /**
+ * The Constant DEFAULT_HDFS_OUTPUT_RETENTION.
+ */
+ public static final String DEFAULT_HDFS_OUTPUT_RETENTION = "1 day";
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b54c4b99/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 7b15a3c..1a49250 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -200,6 +200,11 @@ public class QueryExecutionServiceImpl ext
lens git commit: LENS-847: Columnar JDBC Rewriter in incorrectly pushing filter against wrong alias when fact columns map to a single dimension
Repository: lens
Updated Branches:
refs/heads/master e274c2dfd -> ccf71332f
LENS-847: Columnar JDBC Rewriter in incorrectly pushing filter against wrong
alias when fact columns map to a single dimension
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ccf71332
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ccf71332
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ccf71332
Branch: refs/heads/master
Commit: ccf71332f90f7dd36eac0a84c3b32f9ff76d2812
Parents: e274c2d
Author: Srikanth Sundarrajan
Authored: Tue Oct 27 19:20:53 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Oct 27 19:20:53 2015 +0530
--
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 11 ---
.../driver/jdbc/TestColumnarSQLRewriter.java| 20
2 files changed, 28 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/ccf71332/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index ccb2b7f..825a039 100644
---
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -673,7 +673,10 @@ public class ColumnarSQLRewriter implements QueryRewriter {
.replaceAll("[(,)]", "");
String dimJoinKeys = HQLParser.getString(right).replaceAll("\\s+", "")
.replaceAll("[(,)]", "");
-String dimTableName = dimJoinKeys.substring(0,
dimJoinKeys.indexOf("__"));
+int dimTableDelimIndex = dimJoinKeys.indexOf("__");
+String dimTableName = dimJoinKeys.substring(0, dimTableDelimIndex);
+String dimAlias = dimJoinKeys.
+substring(dimTableDelimIndex + 3, dimJoinKeys.indexOf('.')).trim();
// Construct part of subquery by referring join condition
// fact.fact_key = dim_table.dim_key
@@ -691,14 +694,16 @@ public class ColumnarSQLRewriter implements QueryRewriter
{
// Check the occurrence of dimension table in the filter list and
// combine all filters of same dimension table with and .
// eg. "dim_table.key1 = 'abc' and dim_table.key2 = 'xyz'"
-if (setAllFilters.toString().matches("(.*)" + dimTableName + "(.*)")) {
+if (setAllFilters.toString().replaceAll("\\s+", "")
+.matches("(.*)" + dimAlias + "(.*)")) {
factFilters.delete(0, factFilters.length());
// All filters in where clause
for (int i = 0; i < setAllFilters.toArray().length; i++) {
-if (setAllFilters.toArray()[i].toString().matches("(.*)" +
dimTableName + ("(.*)"))) {
+if (setAllFilters.toArray()[i].toString().replaceAll("\\s+", "")
+.matches("(.*)" + dimAlias + ("(.*)"))) {
String filters2 = setAllFilters.toArray()[i].toString();
filters2 = filters2.replaceAll(
getTableOrAlias(filters2, "alias"),
http://git-wip-us.apache.org/repos/asf/lens/blob/ccf71332/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
--
diff --git
a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index d4d812f..3a00510 100644
---
a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++
b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -142,6 +142,7 @@ public class TestColumnarSQLRewriter {
factColumns.add(new FieldSchema("item_key", "int", ""));
factColumns.add(new FieldSchema("branch_key", "int", ""));
factColumns.add(new FieldSchema("location_key", "int", ""));
+factColumns.add(new FieldSchema("other_location_key", "int", ""));
factColumns.add(new FieldSchema("dollars_sold", "double", ""));
factColumns.add(new FieldSchema("units_sold", "int", ""));
@@ -233,6 +234,25 @@ public class TestColumnarSQLRewriter {
}
+ @Test
+ public void testPushDownFilterWithCommonDim() throws LensException {
+String query = "select fact.time_key, time_dim.day_of_week,
location_dim_a.location_name, "
+ + "other_location_dim.location_name, sum(fact.dollars_sold) from
sales_fact fact inner join "
+ + "time_dim time_dim on fact.time_key = time_dim.time_key inner join
location_dim location_dim_a "
+ + "on fact.location_key = locati
lens git commit: LENS-850: Queries with column name alias like "year" fails on some DBs
Repository: lens
Updated Branches:
refs/heads/master ccf71332f -> 6c4dba51b
LENS-850: Queries with column name alias like "year" fails on some DBs
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6c4dba51
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6c4dba51
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6c4dba51
Branch: refs/heads/master
Commit: 6c4dba51bfebba2bd019508d2e44850ef964d150
Parents: ccf7133
Author: Puneet Gupta
Authored: Tue Oct 27 20:39:49 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Oct 27 20:39:49 2015 +0530
--
.../org/apache/lens/cube/parse/HQLParser.java | 6 +-
.../lens/cube/parse/TestBaseCubeQueries.java| 97 ++--
.../lens/cube/parse/TestCubeRewriter.java | 28 +++---
.../lens/cube/parse/TestExpressionResolver.java | 12 +--
.../apache/lens/cube/parse/TestHQLParser.java | 26 +-
.../lens/cube/parse/TestJoinResolver.java | 33 ---
.../driver/jdbc/TestColumnarSQLRewriter.java| 12 +--
.../apache/lens/driver/jdbc/TestJDBCFinal.java | 12 +--
8 files changed, 120 insertions(+), 106 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/6c4dba51/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 1964e32..ea9badd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -345,9 +345,9 @@ public final class HQLParser {
buf.append(" true ");
} else if (KW_FALSE == rootType) {
buf.append(" false ");
- } else if (Identifier == rootType && TOK_SELEXPR == ((ASTNode)
root.getParent()).getToken().getType()
-&& hasSpaces(rootText)) {
-// If column alias contains spaces, enclose in back quotes
+ } else if (Identifier == rootType && TOK_SELEXPR == ((ASTNode)
root.getParent()).getToken().getType()) {
+// back quote column alias in all cases. This is required since some
alias values can match DB keywords
+// (example : year as alias) and in such case queries can fail on
certain DBs if the alias in not back quoted
buf.append(" as `").append(rootText).append("` ");
} else if (Identifier == rootType && TOK_FUNCTIONSTAR == ((ASTNode)
root.getParent()).getToken().getType()) {
// count(*) or count(someTab.*): Don't append space after the
identifier
http://git-wip-us.apache.org/repos/asf/lens/blob/6c4dba51/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index e6c3be0..548bf5c 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -167,10 +167,10 @@ public class TestBaseCubeQueries extends TestQueryRewrite
{
Assert.assertTrue(candidateFacts.contains("testfact2_base"));
String hqlQuery = ctx.toHQL();
String expected1 =
- getExpectedQuery(cubeName, "select sum(basecube.msr12) msr12 FROM ",
null,
+ getExpectedQuery(cubeName, "select sum(basecube.msr12) as `msr12` FROM
", null,
null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
String expected2 =
- getExpectedQuery(cubeName, "select round(sum(basecube.msr2)/1000)
roundedmsr2 FROM ", null,
+ getExpectedQuery(cubeName, "select round(sum(basecube.msr2)/1000) as
`roundedmsr2` FROM ", null,
null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
TestCubeRewriter.compareContains(expected2, hqlQuery);
@@ -185,10 +185,10 @@ public class TestBaseCubeQueries extends TestQueryRewrite
{
public void testMultiFactQueryWithSingleCommonDimension() throws Exception {
String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube"
+ " where " + TWO_DAYS_RANGE, conf);
String expected1 =
- getExpectedQuery(cubeName, "select basecube.dim1 dim1,
sum(basecube.msr12) msr12 FROM ", null,
+ getExpectedQuery(cubeName, "select basecube.dim1 as `dim1`,
sum(basecube.msr12) as `msr12` FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName,
"C1_testFact2_BASE"));
-String expected2 =
- getExpectedQuery(cubeName, "select basecube.dim1 dim1,
round(sum(basec
lens git commit: LENS-849: NPE with CandidateTableResolver with expression dimensions
Repository: lens
Updated Branches:
refs/heads/master 6c4dba51b -> 09419a577
LENS-849: NPE with CandidateTableResolver with expression dimensions
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/09419a57
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/09419a57
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/09419a57
Branch: refs/heads/master
Commit: 09419a577f6b4d4ac8890be10de7ddf256ef616b
Parents: 6c4dba5
Author: Amareshwari Sriramadasu
Authored: Tue Oct 27 21:07:29 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Oct 27 21:07:30 2015 +0530
--
.../apache/lens/cube/parse/CandidateTableResolver.java | 12 +++-
.../java/org/apache/lens/cube/parse/CubeTestSetup.java | 2 ++
.../apache/lens/cube/parse/TestExpressionResolver.java | 13 -
.../java/org/apache/lens/cube/parse/TestQuery.java | 12 +++-
.../lens/driver/jdbc/TestColumnarSQLRewriter.java | 2 +-
5 files changed, 29 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/09419a57/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index f81f3d5..0ad7610 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -590,11 +590,13 @@ class CandidateTableResolver implements ContextRewriter {
for (CandidateTable candidate : removedCandidates.get(dim)) {
// check if evaluable expressions of this candidate are no more
evaluable because dimension is not reachable
// if no evaluable expressions exist, then remove the candidate
-Iterator escIter =
ec.getEvaluableExpressions().get(candidate).iterator();
-while (escIter.hasNext()) {
- ExprSpecContext esc = escIter.next();
- if (esc.getExprDims().contains(dim)) {
-escIter.remove();
+if (ec.getEvaluableExpressions().get(candidate) != null) {
+ Iterator escIter =
ec.getEvaluableExpressions().get(candidate).iterator();
+ while (escIter.hasNext()) {
+ExprSpecContext esc = escIter.next();
+if (esc.getExprDims().contains(dim)) {
+ escIter.remove();
+}
}
}
if (cubeql.getExprCtx().isEvaluable(col.getExprCol(), candidate)) {
http://git-wip-us.apache.org/repos/asf/lens/blob/09419a57/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 705f19b..dc11b4c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -698,6 +698,8 @@ public class CubeTestSetup {
new ExprSpec("avg(roundedmsr2)", null, null), new
ExprSpec("avg(equalsums)", null, null),
new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when
substrexpr = 'abc' then avg(msr4)/100 end",
null, null)));
+exprs.add(new ExprColumn(new FieldSchema("msr2expr", "double", "nested
expr"), "Nested expr",
+ new ExprSpec("case when cityStateName = 'xyz' then msr2 else 0 end",
null, null)));
exprs.add(new ExprColumn(new FieldSchema("nestedExprWithTimes", "double",
"nested expr"), "Nested expr",
new ExprSpec("avg(roundedmsr2)", null, null), new
ExprSpec("avg(equalsums)", null, null),
new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when
substrexpr = 'abc' then avg(msr4)/100 end",
http://git-wip-us.apache.org/repos/asf/lens/blob/09419a57/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
index 1cb22eb..e90603c 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
@@ -316,6 +316,17 @@ public class TestExpressionResolver extends
TestQueryRewrite {
}
@Test
+ public void testExpressionFieldWithOtherFields() throws Exception {
+// sel
lens git commit: LENS-848: Allow JDBC driver to take connection properties in configuration
Repository: lens
Updated Branches:
refs/heads/master e264e53f3 -> 55b5223e9
LENS-848: Allow JDBC driver to take connection properties in configuration
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/55b5223e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/55b5223e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/55b5223e
Branch: refs/heads/master
Commit: 55b5223e9bfbb4904b46b2843f4410293cb4019b
Parents: e264e53
Author: Rajat Khandelwal
Authored: Wed Oct 28 14:10:52 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Oct 28 14:10:52 2015 +0530
--
.../org/apache/lens/api/util/CommonUtils.java | 58 +++
.../apache/lens/api/util/CommonUtilsTest.java | 54 +++
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 12 +-
.../jdbc/DataSourceConnectionProvider.java | 88 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 65
.../driver/jdbc/JDBCDriverConfConstants.java| 56 ---
.../src/main/resources/jdbcdriver-default.xml | 13 ++
.../jdbc/TestDataSourceConnectionProvider.java | 7 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 33 ++--
.../src/test/resources/jdbcdriver-site.xml | 25 +--
src/site/apt/admin/config.apt | 160 +--
src/site/apt/admin/jdbcdriver-config.apt| 54 ---
12 files changed, 389 insertions(+), 236 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/55b5223e/lens-api/src/main/java/org/apache/lens/api/util/CommonUtils.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/util/CommonUtils.java
b/lens-api/src/main/java/org/apache/lens/api/util/CommonUtils.java
new file mode 100644
index 000..38d58c7
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/util/CommonUtils.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.util;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class CommonUtils {
+ private CommonUtils() {
+
+ }
+
+ /**
+ * Splits given String str around non-escaped commas. Then parses each of
the split element
+ * as map entries in the format `key=value`. Constructs a map of such
entries.
+ * e.g. "a=b, c=d" parses to map{a:b, c:d} where the symbols are
self-explanatory.
+ *
+ * @param str The string to parse
+ * @return parsed map
+ */
+ public static Map parseMapFromString(String str) {
+Map map = new HashMap<>();
+if (str != null) {
+ for (String kv : str.split("(? 0) {
+key = kvArray[0].replaceAll(",", ",").trim();
+ }
+ if (kvArray.length > 1) {
+value = kvArray[1].replaceAll(",", ",").trim();
+ }
+ map.put(key, value);
+}
+ }
+}
+return map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/55b5223e/lens-api/src/test/java/org/apache/lens/api/util/CommonUtilsTest.java
--
diff --git
a/lens-api/src/test/java/org/apache/lens/api/util/CommonUtilsTest.java
b/lens-api/src/test/java/org/apache/lens/api/util/CommonUtilsTest.java
new file mode 100644
index 000..2c4b389
--- /dev/null
+++ b/lens-api/src/test/java/org/apache/lens/api/util/CommonUtilsTest.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR
lens git commit: LENS-862: Update HQLParser.printAST() to print Char Position In Line
Repository: lens
Updated Branches:
refs/heads/master ffc9987ec -> be36b6303
LENS-862: Update HQLParser.printAST() to print Char Position In Line
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/be36b630
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/be36b630
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/be36b630
Branch: refs/heads/master
Commit: be36b63034d8ba62ed491e92a53fa40fd0896e4e
Parents: ffc9987
Author: Puneet Gupta
Authored: Mon Nov 16 11:39:04 2015 +0530
Committer: Rajat Khandelwal
Committed: Mon Nov 16 11:39:04 2015 +0530
--
lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/be36b630/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index ea9badd..16e1aa3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -194,7 +194,7 @@ public final class HQLParser {
}
System.out.print(node.getText() + " [" +
tokenMapping.get(node.getToken().getType()) + "]");
-System.out.print(" (l" + level + "c" + child + ")");
+System.out.print(" (l" + level + "c" + child + "p" +
node.getCharPositionInLine() +")");
if (node.getChildCount() > 0) {
System.out.println(" {");
lens git commit: LENS-824 Fix all javadoc warnings/errors
Repository: lens
Updated Branches:
refs/heads/master be36b6303 -> 10dcebb92
LENS-824 Fix all javadoc warnings/errors
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/10dcebb9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/10dcebb9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/10dcebb9
Branch: refs/heads/master
Commit: 10dcebb9247f605769dfd90e25be82aff3f79dbc
Parents: be36b63
Author: Gabor Liptak
Authored: Mon Nov 16 11:40:06 2015 +0530
Committer: Rajat Khandelwal
Committed: Mon Nov 16 11:40:06 2015 +0530
--
.../lens/api/query/save/ParameterCollectionType.java| 2 +-
.../org/apache/lens/api/result/PrettyPrintable.java | 2 +-
.../java/org/apache/lens/api/util/PathValidator.java| 12 ++--
.../org/apache/lens/cli/commands/LensQueryCommands.java | 1 -
.../lens/server/metastore/CubeMetastoreServiceImpl.java | 2 +-
.../lens/server/query/QueryExecutionServiceImpl.java| 2 +-
.../apache/lens/server/query/save/SavedQueryDao.java| 4 ++--
.../org/apache/lens/server/quota/QuotaServiceImpl.java | 2 +-
.../server/scheduler/QuerySchedulerServiceImpl.java | 2 +-
.../apache/lens/server/session/HiveSessionService.java | 2 +-
pom.xml | 6 +++---
11 files changed, 18 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/10dcebb9/lens-api/src/main/java/org/apache/lens/api/query/save/ParameterCollectionType.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/query/save/ParameterCollectionType.java
b/lens-api/src/main/java/org/apache/lens/api/query/save/ParameterCollectionType.java
index 971e794..14637de 100644
---
a/lens-api/src/main/java/org/apache/lens/api/query/save/ParameterCollectionType.java
+++
b/lens-api/src/main/java/org/apache/lens/api/query/save/ParameterCollectionType.java
@@ -24,7 +24,7 @@ import javax.xml.bind.annotation.XmlRootElement;
* The enum ParameterCollectionType
* Collection type of a parameter has to be chosen based on its context.
* - If it is occurring next to an IN/NOT IN clause, its multiple
- * - If it is found with EQ/NEQ..>,<,>=,<=,like etc, its single
+ * - If it is found with EQ/NEQ..>,<,>=,<=,like etc, its single
*/
@XmlRootElement
public enum ParameterCollectionType {
http://git-wip-us.apache.org/repos/asf/lens/blob/10dcebb9/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
b/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
index e6152b8..5c45596 100644
--- a/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
+++ b/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
@@ -26,7 +26,7 @@ public interface PrettyPrintable {
/**
* Returns the contents of this object in the form of a pretty formatted
string.
*
- * @return
+ * @return contents as formatted String
*/
String toPrettyString();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/10dcebb9/lens-api/src/main/java/org/apache/lens/api/util/PathValidator.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/util/PathValidator.java
b/lens-api/src/main/java/org/apache/lens/api/util/PathValidator.java
index 92a936a..45397d7 100644
--- a/lens-api/src/main/java/org/apache/lens/api/util/PathValidator.java
+++ b/lens-api/src/main/java/org/apache/lens/api/util/PathValidator.java
@@ -52,10 +52,10 @@ public class PathValidator {
* Also removes erroneously appended prefix for URI's.
* Takes additional properties for special URI handling.
*
- * @param path
- * @param shouldBeDirectory
- * @param shouldExist
- * @return
+ * @param path input path
+ * @param shouldBeDirectory should be a directory
+ * @param shouldExist should exist
+ * @return converted path
*/
public String getValidPath(File path, boolean shouldBeDirectory,
boolean shouldExist) {
@@ -89,8 +89,8 @@ public class PathValidator {
*
* Any new URI's have to be handled appropriately
*
- * @param path
- * @return
+ * @param path input path
+ * @return cleaned up path
*/
public String removePrefixBeforeURI(String path) {
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/10dcebb9/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
--
diff --git
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.ja
lens git commit: LENS-719: Allow fact start time to be specified for a storage and a update period
Repository: lens
Updated Branches:
refs/heads/master 10dcebb92 -> 3ed191aca
LENS-719: Allow fact start time to be specified for a storage and a update
period
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3ed191ac
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3ed191ac
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3ed191ac
Branch: refs/heads/master
Commit: 3ed191aca6c18be7c53afefdea28a6d08ae1fd07
Parents: 10dcebb
Author: Rajat Khandelwal
Authored: Mon Nov 16 11:41:05 2015 +0530
Committer: Rajat Khandelwal
Committed: Mon Nov 16 11:41:05 2015 +0530
--
lens-api/src/main/resources/cube-0.1.xsd| 9
.../lens/cube/metadata/MetastoreUtil.java | 6 +++
.../cube/parse/CandidateTablePruneCause.java| 2 +
.../org/apache/lens/cube/parse/DateUtil.java| 4 +-
.../lens/cube/parse/StorageTableResolver.java | 51 +---
.../apache/lens/cube/parse/CubeTestSetup.java | 13 +++--
.../lens/cube/parse/TestCubeRewriter.java | 5 +-
7 files changed, 76 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/3ed191ac/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git a/lens-api/src/main/resources/cube-0.1.xsd
b/lens-api/src/main/resources/cube-0.1.xsd
index 5d7630d..4092133 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -828,6 +828,15 @@
The following properties can be specified for Elastic search
tables :
1. lens.metastore.es.index.name : The underlying ES index name.
2. lens.metastore.es.type.name : The underlying ES type name.
+Start and End times for storage table:
+1. cube.storagetable.start.times: Comma separated list of start
times for this table.
+ Start times can be relative times(e.g. now.day - 1 month) or
absolute times(e.g. 2014-02)
+ The max of the start times will be considered as the final
start time. This storagetable will
+ not be candidate for answering time ranges completely before
its start time.
+2. cube.storagetable.end.times: Comma separated list of end times
for this table.
+ End times can be relative times(e.g. now.day - 1 month) or
absolute times(e.g. 2014-02)
+ The min of the end times will be considered as the final end
time. This storagetable will not be
+ candidate for answering time ranges completely after its end
time.
http://git-wip-us.apache.org/repos/asf/lens/blob/3ed191ac/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 2796cd9..e5cf468 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -150,6 +150,12 @@ public class MetastoreUtil {
public static String getCubeColEndTimePropertyKey(String colName) {
return getColumnKeyPrefix(colName) + END_TIME_SFX;
}
+ public static String getStoragetableStartTimesKey(){
+return STORAGE_PFX + "start.times";
+ }
+ public static String getStoragetableEndTimesKey(){
+return STORAGE_PFX + "end.times";
+ }
public static String getCubeColCostPropertyKey(String colName) {
return getColumnKeyPrefix(colName) + COST_SFX;
http://git-wip-us.apache.org/repos/asf/lens/blob/3ed191ac/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 9ea43bb..9c8b5b9 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -180,6 +180,8 @@ public class CandidateTablePruneCause {
NO_PARTITIONS,
// partition column does not exist
PART_COL_DOES_NOT_EXIST,
+// Range is not supported by this storage table
+RANGE_NOT_ANSWERABLE,
// storage is not supported by execution engine
UNSUPPORTED
}
http://git-wip-us.apache.org/repos/asf/lens/blob/3ed191ac/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
--
diff --git a/lens-cube/src/main/j
lens git commit: LENS-864: LENS - 864
Repository: lens
Updated Branches:
refs/heads/master 7b5f4a09a -> 2870be7c8
LENS-864: LENS - 864
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2870be7c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2870be7c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2870be7c
Branch: refs/heads/master
Commit: 2870be7c8c2dbef92c237878c5faba048d3a139d
Parents: 7b5f4a0
Author: Ankeet Maini
Authored: Wed Nov 18 17:58:45 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Nov 18 17:58:45 2015 +0530
--
lens-ui/app/components/QueryBoxComponent.js | 122 +++--
.../app/components/QueryParamRowComponent.js| 176 +--
lens-ui/app/components/QueryParamsComponent.js | 72 +---
.../components/SavedQueryPreviewComponent.js| 26 +--
4 files changed, 219 insertions(+), 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2870be7c/lens-ui/app/components/QueryBoxComponent.js
--
diff --git a/lens-ui/app/components/QueryBoxComponent.js
b/lens-ui/app/components/QueryBoxComponent.js
index 6f4eeb7..ec6a06b 100644
--- a/lens-ui/app/components/QueryBoxComponent.js
+++ b/lens-ui/app/components/QueryBoxComponent.js
@@ -21,6 +21,7 @@ import React from 'react';
import ClassNames from 'classnames';
import CodeMirror from 'codemirror';
import assign from 'object-assign';
+import _ from 'lodash';
import 'codemirror/lib/codemirror.css';
import 'codemirror/addon/edit/matchbrackets.js';
import 'codemirror/addon/hint/sql-hint.js';
@@ -74,6 +75,20 @@ function setCode (code) {
}
}
+function getEmptyState () {
+ return {
+clientMessage: null, // to give user instant ack
+isRunQueryDisabled: true,
+serverMessage: null, // type (success or error), text as keys
+isCollapsed: false,
+params: null,
+isModeEdit: false,
+savedQueryId: null,
+runImmediately: false,
+description: ''
+ };
+}
+
// used to populate the query box when user wants to edit a query
// TODO improve this.
// this takes in the query handle and writes the query
@@ -114,16 +129,7 @@ class QueryBox extends React.Component {
this.saveOrUpdate = this.saveOrUpdate.bind(this);
this.runSavedQuery = this.runSavedQuery.bind(this);
-this.state = {
- clientMessage: null, // to give user instant ack
- isRunQueryDisabled: true,
- serverMessage: null, // type (success or error), text as keys
- isCollapsed: false,
- params: null,
- isModeEdit: false,
- savedQueryId: null,
- runImmediately: false
-};
+this.state = getEmptyState();
}
componentDidMount () {
@@ -184,6 +190,7 @@ class QueryBox extends React.Component {
this.setState({
params: savedQuery.parameters,
savedQueryId: savedQuery.id,
+ description: savedQuery.description,
isModeEdit: true
});
}
@@ -257,7 +264,7 @@ class QueryBox extends React.Component {
{ this.state.params && !!this.state.params.length &&
+saveParams={this.saveParams}
description={this.state.description}/>
}
{ this.state.serverMessage &&
@@ -304,17 +311,40 @@ class QueryBox extends React.Component {
updateQuery (params) {
let query = this._getSavedQueryDetails(params);
if (!query) return;
+
+var options = {
+ parameters: query.parameters,
+ description: query.description,
+ name: query.name
+};
+
AdhocQueryActions
- .updateSavedQuery(query.secretToken, query.user, query.query,
query.params, this.state.savedQueryId);
-this.setState({ clientMessage: clientMessages.updateQuery });
+ .updateSavedQuery(query.secretToken, query.user, query.query,
+options, this.state.savedQueryId);
+
+this.setState({
+ clientMessage: clientMessages.updateQuery,
+ runImmediately: params && params.runImmediately
+});
}
saveQuery (params) {
let query = this._getSavedQueryDetails(params);
if (!query) return;
+
+var options = {
+ parameters: query.parameters,
+ description: query.description,
+ name: query.name
+};
+
AdhocQueryActions
- .saveQuery(query.secretToken, query.user, query.query, query.params);
-this.setState({ clientMessage: clientMessages.saveQuery });
+ .saveQuery(query.secretToken, query.user, query.query, options);
+
+this.setState({
+ clientMessage: clientMessages.saveQuery,
+ runImmediately: params && params.runImmediately
+});
}
// internal which is called during save saved query & edit saved query
@@ -329,14 +359,13 @@ class QueryBox extends React.Component {
let user = UserStore.getUserDetails().email;
lens git commit: LENS-870 : Expressions in multi fact query is not rewritten properly
Repository: lens
Updated Branches:
refs/heads/master d820c32ab -> 09baa1288
LENS-870 : Expressions in multi fact query is not rewritten properly
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/09baa128
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/09baa128
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/09baa128
Branch: refs/heads/master
Commit: 09baa12880ba27c00323bbf881e9c69acb580117
Parents: d820c32
Author: Sushil Mohanty
Authored: Tue Nov 24 12:05:05 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Nov 24 12:05:05 2015 +0530
--
.../lens/cube/parse/CandidateTableResolver.java | 3 ++-
.../lens/cube/parse/ExpressionResolver.java | 6 ++---
.../apache/lens/cube/parse/CubeTestSetup.java | 2 ++
.../lens/cube/parse/TestBaseCubeQueries.java| 27
4 files changed, 34 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/09baa128/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 0ad7610..38ff5a4 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -272,7 +272,8 @@ class CandidateTableResolver implements ContextRewriter {
if (!checkForColumnExists(cfact, queriedMsrs)
&& (cubeql.getQueriedExprsWithMeasures().isEmpty()
||
cubeql.getExprCtx().allNotEvaluable(cubeql.getQueriedExprsWithMeasures(),
cfact))) {
- log.info("Not considering fact table:{} as columns {} is not
available", cfact, queriedMsrs);
+ log.info("Not considering fact table:{} as columns {},{} is not
available", cfact, queriedMsrs,
+ cubeql.getQueriedExprsWithMeasures());
cubeql.addFactPruningMsgs(cfact.fact,
CandidateTablePruneCause.columnNotFound(queriedMsrs,
cubeql.getQueriedExprsWithMeasures()));
toRemove = true;
http://git-wip-us.apache.org/repos/asf/lens/blob/09baa128/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 1a347b2..200a48c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -405,11 +405,11 @@ class ExpressionResolver implements ContextRewriter {
*/
public boolean allNotEvaluable(Set exprs, CandidateTable cTable) {
for (String expr : exprs) {
-if (!isEvaluable(expr, cTable)) {
- return true;
+if (isEvaluable(expr, cTable)) {
+ return false;
}
}
- return false;
+ return true;
}
public Collection coveringExpressions(Set exprs,
CandidateTable cTable) {
http://git-wip-us.apache.org/repos/asf/lens/blob/09baa128/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index aa15a2c..999faa0 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -732,6 +732,8 @@ public class CubeTestSetup {
"round(msr1/1000)"));
exprs.add(new ExprColumn(new FieldSchema("roundedmsr2", "double", "rounded
measure2"), "Rounded msr2",
"round(msr2/1000)"));
+exprs.add(new ExprColumn(new FieldSchema("flooredmsr12", "double",
"floored measure12"), "Floored msr12",
+"floor(msr12)"));
exprs.add(new ExprColumn(new FieldSchema("nestedexpr", "double", "nested
expr"), "Nested expr",
new ExprSpec("avg(roundedmsr2)", null, null), new
ExprSpec("avg(equalsums)", null, null),
new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when
substrexpr = 'abc' then avg(msr4)/100 end",
http://git-wip-us.apache.org/repos/asf/lens/blob/09baa128/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
b/lens-cube/src/test/java/org
[1/2] lens git commit: LENS-865: Add/delete partition throws NPE when a part col doesn't exist in the table
Repository: lens
Updated Branches:
refs/heads/master 09baa1288 -> e5691d8d6
http://git-wip-us.apache.org/repos/asf/lens/blob/e5691d8d/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 0748d5b..fb937c6 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -26,13 +26,17 @@ import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import javax.xml.bind.JAXBElement;
-import org.apache.lens.api.*;
+import org.apache.lens.api.APIResult;
import org.apache.lens.api.APIResult.*;
+import org.apache.lens.api.DateTime;
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.StringList;
import org.apache.lens.api.metastore.*;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metastore.CubeMetastoreService;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -52,11 +56,11 @@ import lombok.extern.slf4j.Slf4j;
public class MetastoreResource {
public static final ObjectFactory X_CUBE_OBJECT_FACTORY = new
ObjectFactory();
- public CubeMetastoreService getSvc() {
+ public static CubeMetastoreService getSvc() {
return LensServices.get().getService(CubeMetastoreService.NAME);
}
- private void checkSessionId(LensSessionHandle sessionHandle) {
+ private static void checkSessionId(LensSessionHandle sessionHandle) {
if (sessionHandle == null) {
throw new BadRequestException("Invalid session handle");
}
@@ -75,6 +79,154 @@ public class MetastoreResource {
checkNonNullArgs("One partition is null",
partitions.getPartition().toArray());
}
+ private static LensException processLensException(LensException exc) {
+if (exc != null) {
+ exc.buildLensErrorTO(LensServices.get().getErrorCollection());
+}
+return exc;
+ }
+
+ public enum Entity {
+DATABASE {
+ @Override
+ public List doGetAll(LensSessionHandle sessionHandle) throws
LensException {
+return getSvc().getAllDatabases(sessionHandle);
+ }
+
+ @Override
+ public void doDelete(LensSessionHandle sessionid, String entityName,
Boolean cascade) throws LensException {
+if (cascade == null) {
+ throw new NotImplementedException();
+} else {
+ getSvc().dropDatabase(sessionid, entityName, cascade);
+}
+ }
+
+}, STORAGE {
+ @Override
+ public List doGetAll(LensSessionHandle sessionid) throws
LensException {
+return getSvc().getAllStorageNames(sessionid);
+ }
+
+ @Override
+ public void doDelete(LensSessionHandle sessionid, String entityName,
Boolean cascade) throws LensException {
+if (cascade == null) {
+ getSvc().dropStorage(sessionid, entityName);
+} else {
+ throw new NotImplementedException();
+}
+ }
+}, CUBE {
+ @Override
+ public List doGetAll(LensSessionHandle sessionHandle) throws
LensException {
+return getSvc().getAllCubeNames(sessionHandle);
+ }
+
+ @Override
+ public void doDelete(LensSessionHandle sessionid, String entityName,
Boolean cascade) throws LensException {
+if (cascade == null) {
+ getSvc().dropCube(sessionid, entityName);
+} else {
+ throw new NotImplementedException();
+}
+ }
+}, FACT {
+ @Override
+ public List doGetAll(LensSessionHandle sessionid) throws
LensException {
+return getSvc().getAllFactNames(sessionid, null);
+ }
+
+ @Override
+ public void doDelete(LensSessionHandle sessionid, String entityName,
Boolean cascade) throws LensException {
+if (cascade == null) {
+ throw new NotImplementedException();
+} else {
+ getSvc().dropFactTable(sessionid, entityName, cascade);
+}
+ }
+}, DIMENSION {
+ @Override
+ public List doGetAll(LensSessionHandle sessionid) throws
LensException {
+return getSvc().getAllDimensionNames(sessionid);
+ }
+
+ @Override
+ public void doDelete(LensSessionHandle sessionid, String entityName,
Boolean cascade) throws LensException {
+if (cascade == null) {
+ getSvc().dropDimension(sessionid, entityName);
+} else {
+ throw new NotImplementedException();
+}
+ }
+}, DIMTABLE {
+ @Override
+ public List doGetAll(LensSessionHandle sessionid) throws
LensException {
+return getSvc().getAllDimT
[2/2] lens git commit: LENS-865: Add/delete partition throws NPE when a part col doesn't exist in the table
LENS-865: Add/delete partition throws NPE when a part col doesn't exist in the
table
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e5691d8d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e5691d8d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e5691d8d
Branch: refs/heads/master
Commit: e5691d8d655c94cdd64e0d83028ec59735d73edc
Parents: 09baa12
Author: Rajat Khandelwal
Authored: Tue Nov 24 12:07:17 2015 +0530
Committer: Rajat Khandelwal
Committed: Tue Nov 24 12:07:17 2015 +0530
--
.../java/org/apache/lens/api/APIResult.java | 10 +-
lens-api/src/main/resources/lens-errors.conf| 6 +
.../lens/cube/error/LensCubeErrorCode.java | 5 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 42 +-
.../cube/metadata/TestCubeMetastoreClient.java | 404 ++-
.../lens/server/api/error/LensException.java| 8 +-
.../server/metastore/MetastoreResource.java | 384 ++
.../server/metastore/TestMetastoreService.java | 40 +-
8 files changed, 498 insertions(+), 401 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/e5691d8d/lens-api/src/main/java/org/apache/lens/api/APIResult.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
index 06c608a..0cdee0d 100644
--- a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
@@ -152,11 +152,13 @@ public class APIResult {
}
private static String extractCause(Throwable e) {
-String cause = null;
-while ((cause == null || cause.isEmpty()) && e != null) {
- cause = e.getMessage();
+StringBuilder cause = new StringBuilder();
+String sep = "";
+while (e != null) {
+ cause.append(sep).append(e.getMessage());
e = e.getCause();
+ sep = ": ";
}
-return cause;
+return cause.toString();
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e5691d8d/lens-api/src/main/resources/lens-errors.conf
--
diff --git a/lens-api/src/main/resources/lens-errors.conf
b/lens-api/src/main/resources/lens-errors.conf
index f50433a..ca8562f 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -296,6 +296,12 @@ lensCubeErrorsForMetastore = [
httpStatusCode = ${BAD_REQUEST}
errorMsg = "Problem in submitting entity: %s"
}
+
+ {
+ errorCode = 3102
+ httpStatusCode = ${BAD_REQUEST}
+ errorMsg = "No timeline found for fact=%s, storage=%s, update period=%s,
partition column=%s."
+ }
]
lensCubeErrors = ${lensCubeErrorsForQuery}${lensCubeErrorsForMetastore}
http://git-wip-us.apache.org/repos/asf/lens/blob/e5691d8d/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
index 24fb80b..6c5dc2f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.error;
import org.apache.lens.server.api.LensErrorInfo;
public enum LensCubeErrorCode {
+ // Error codes less than 3100 are errors encountered while submitting a query
// Error codes same for drivers
SYNTAX_ERROR(3001, 0),
FIELDS_CANNOT_BE_QUERIED_TOGETHER(3002, 0),
@@ -54,7 +55,9 @@ public enum LensCubeErrorCode {
NO_CANDIDATE_DIM_STORAGE_TABLES(3029, 1300),
NO_STORAGE_TABLE_AVAIABLE(3030, 1400),
STORAGE_UNION_DISABLED(3031, 100),
- ERROR_IN_ENTITY_DEFINITION(3101, 100);
+ // Error codes greater than 3100 are errors while doing a metastore
operation.
+ ERROR_IN_ENTITY_DEFINITION(3101, 100),
+ TIMELINE_ABSENT(3102, 100);
public LensErrorInfo getLensErrorInfo() {
return this.errorInfo;
http://git-wip-us.apache.org/repos/asf/lens/blob/e5691d8d/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index f945e0f..1f13617 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -25,6 +25,7 @@ import java.text.P
[1/2] lens git commit: LENS-878 : Refactor inner classes in JoinResolver
Repository: lens
Updated Branches:
refs/heads/master 7a3a1734e -> f7ab827e9
http://git-wip-us.apache.org/repos/asf/lens/blob/f7ab827e/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinTree.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinTree.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinTree.java
new file mode 100644
index 000..5a294af
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinTree.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import java.util.*;
+
+import org.apache.lens.cube.metadata.AbstractCubeTable;
+import org.apache.lens.cube.metadata.SchemaGraph;
+
+import org.apache.hadoop.hive.ql.parse.JoinType;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+@Data
+@ToString(exclude = "parent")
+@EqualsAndHashCode(exclude = "parent")
+public class JoinTree {
+ //parent of the node
+ JoinTree parent;
+ // current table is parentRelationship.destTable;
+ SchemaGraph.TableRelationship parentRelationship;
+ // Alias for the join clause
+ String alias;
+ private Map subtrees =
+new LinkedHashMap();
+ // Number of nodes from root to this node. depth of root is 0. Unused for
now.
+ private int depthFromRoot;
+ // join type of the current table.
+ JoinType joinType;
+
+ public static JoinTree createRoot() {
+return new JoinTree(null, null, 0);
+ }
+
+ public JoinTree(JoinTree parent, SchemaGraph.TableRelationship
tableRelationship,
+ int depthFromRoot) {
+this.parent = parent;
+this.parentRelationship = tableRelationship;
+this.depthFromRoot = depthFromRoot;
+ }
+
+ public JoinTree addChild(SchemaGraph.TableRelationship tableRelationship,
+ CubeQueryContext cubeql, Map
aliasUsage) {
+if (getSubtrees().get(tableRelationship) == null) {
+ JoinTree current = new JoinTree(this, tableRelationship,
+this.depthFromRoot + 1);
+ // Set alias. Need to compute only when new node is being created.
+ // The following code ensures that For intermediate tables, aliases are
given
+ // in the order citydim, citydim_0, citydim_1, ...
+ // And for destination tables, an alias will be decided from here but
might be
+ // overridden outside this function.
+ AbstractCubeTable destTable = tableRelationship.getToTable();
+ current.setAlias(cubeql.getAliasForTableName(destTable.getName()));
+ if (aliasUsage.get(current.getAlias()) == null) {
+aliasUsage.put(current.getAlias(), 0);
+ } else {
+aliasUsage.put(current.getAlias(), aliasUsage.get(current.getAlias())
+ 1);
+current.setAlias(current.getAlias() + "_" +
(aliasUsage.get(current.getAlias()) - 1));
+ }
+ getSubtrees().put(tableRelationship, current);
+}
+return getSubtrees().get(tableRelationship);
+ }
+
+ // Recursive computation of number of edges.
+ public int getNumEdges() {
+int ret = 0;
+for (JoinTree tree : getSubtrees().values()) {
+ ret += 1;
+ ret += tree.getNumEdges();
+}
+return ret;
+ }
+
+ public boolean isLeaf() {
+return getSubtrees().isEmpty();
+ }
+
+ // Breadth First Traversal. Unused currently.
+ public Iterator bft() {
+return new Iterator() {
+ List remaining = new ArrayList() {
+{
+ addAll(getSubtrees().values());
+}
+ };
+
+ @Override
+ public boolean hasNext() {
+return remaining.isEmpty();
+ }
+
+ @Override
+ public JoinTree next() {
+JoinTree retval = remaining.remove(0);
+remaining.addAll(retval.getSubtrees().values());
+return retval;
+ }
+
+ @Override
+ public void remove() {
+throw new RuntimeException("Not implemented");
+ }
+};
+ }
+
+ // Depth first traversal of the tree. Used in forming join string.
+ public Iterator dft() {
+return new Iterator() {
+ Stack joinTreeStack = new Stack() {
+{
+ addAll(getSubtrees().values());
+}
+ };
+
+ @Overr
[2/2] lens git commit: LENS-878 : Refactor inner classes in JoinResolver
LENS-878 : Refactor inner classes in JoinResolver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f7ab827e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f7ab827e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f7ab827e
Branch: refs/heads/master
Commit: f7ab827e967e8a6b44cd8d540e293dbd01ff8d9b
Parents: 7a3a173
Author: Amareshwari Sriramadasu
Authored: Wed Nov 25 14:22:37 2015 +0530
Committer: Rajat Khandelwal
Committed: Wed Nov 25 14:22:37 2015 +0530
--
.../apache/lens/cube/parse/AutoJoinContext.java | 760 ++
.../lens/cube/parse/CubeQueryContext.java | 2 +-
.../org/apache/lens/cube/parse/JoinClause.java | 144 +++
.../apache/lens/cube/parse/JoinResolver.java| 982 +--
.../org/apache/lens/cube/parse/JoinTree.java| 164
.../lens/cube/parse/TimerangeResolver.java | 2 +-
6 files changed, 1076 insertions(+), 978 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/f7ab827e/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
new file mode 100644
index 000..9472506
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
@@ -0,0 +1,760 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import java.util.*;
+
+import org.apache.lens.cube.error.LensCubeErrorCode;
+import org.apache.lens.cube.metadata.*;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.parse.JoinType;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Store join chain information resolved by join resolver
+ */
+@Slf4j
+public class AutoJoinContext {
+ // Map of a joined table to list of all possible paths from that table to
+ // the target
+ private final Map, List> allPaths;
+ @Getter
+ // User supplied partial join conditions
+ private final Map partialJoinConditions;
+ // True if the query contains user supplied partial join conditions
+ @Getter
+ private final boolean partialJoinChains;
+ @Getter
+ // Map of joined table to the join type (if provided by user)
+ private final Map tableJoinTypeMap;
+
+ // True if joins were resolved automatically
+ private boolean joinsResolved;
+ // Target table for the auto join resolver
+ private final AbstractCubeTable autoJoinTarget;
+ // Configuration string to control join type
+ private String joinTypeCfg;
+
+ // Map of a joined table to its columns which are part of any of the join
+ // paths. This is used in candidate table resolver
+ @Getter
+ private Map>>
joinPathFromColumns =
+new HashMap>>();
+
+ @Getter
+ private Map>>
joinPathToColumns =
+new HashMap>>();
+
+ // there can be separate join clause for each fact incase of multi fact
queries
+ @Getter
+ Map factClauses = new HashMap();
+ @Getter
+ @Setter
+ JoinClause minCostClause;
+ private final boolean flattenBridgeTables;
+ private final String bridgeTableFieldAggr;
+
+ public AutoJoinContext(Map, List>
allPaths,
+ Map
optionalDimensions,
+ Map partialJoinConditions,
+ boolean partialJoinChains, Map tableJoinTypeMap,
+ AbstractCubeTable autoJoinTarget, String joinTypeCfg,
boolean joinsResolved,
+ boolean flattenBridgeTables, String
bridgeTableFieldAggr) {
+this.allPaths = allPaths;
+initJoinPathColumns();
+this.partialJoinConditions = partialJoinConditions;
+this.partialJoinChains = partialJoinChains;
+this.tableJoinTypeMap = tableJoinTypeMap;
+this.autoJoinTarget = autoJoinTarget;
+this.joinTypeCfg = joinTypeCfg;
+this.joinsResolved = joi
lens git commit: LENS-883 : httpresultset api should return 404 when resultset file does not exist
Repository: lens
Updated Branches:
refs/heads/master 5d79ecfdb -> 942f071f9
LENS-883 : httpresultset api should return 404 when resultset file does not
exist
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/942f071f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/942f071f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/942f071f
Branch: refs/heads/master
Commit: 942f071f9925b1265a67a1f7b2ab437127485a09
Parents: 5d79ecf
Author: Deepak Barr
Authored: Thu Nov 26 13:51:14 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Nov 26 13:51:14 2015 +0530
--
.../apache/lens/server/query/QueryExecutionServiceImpl.java | 8
1 file changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/942f071f/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 941b1e5..7201e0d 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -2485,6 +2485,14 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
throw new NotFoundException("http result not available");
}
final Path resultPath = new Path(resultSet.getOutputPath());
+try {
+ FileSystem fs = resultPath.getFileSystem(conf);
+ if (!fs.exists(resultPath)) {
+throw new NotFoundException("Result file does not exist!");
+ }
+} catch (IOException e) {
+ throw new LensException(e);
+}
final QueryContext ctx = getQueryContext(sessionHandle, queryHandle);
String resultFSReadUrl = conf.get(RESULT_FS_READ_URL);
if (resultFSReadUrl != null) {
lens git commit: LENS-882: Provide option from CLI to list queries by driver
Repository: lens
Updated Branches:
refs/heads/master 22e202208 -> 0e4c18cb9
LENS-882: Provide option from CLI to list queries by driver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0e4c18cb
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0e4c18cb
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0e4c18cb
Branch: refs/heads/master
Commit: 0e4c18cb917015b469ca43d30d7e2196d8f64789
Parents: 22e2022
Author: Deepak Barr
Authored: Thu Dec 10 12:16:11 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Dec 10 12:16:11 2015 +0530
--
.../lens/cli/commands/LensQueryCommands.java| 4 ++-
.../apache/lens/cli/TestLensQueryCommands.java | 26 +---
.../java/org/apache/lens/client/LensClient.java | 5 ++--
.../org/apache/lens/client/LensStatement.java | 6 +++--
src/site/apt/user/cli.apt | 2 +-
5 files changed, 28 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/0e4c18cb/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
--
diff --git
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index fe9b84d..e3c08ff 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -218,6 +218,7 @@ public class LensQueryCommands extends BaseLensCommand {
* @param state the state
* @param queryName the query name
* @param user the user
+ * @param driverthe driver name
* @param fromDate the from date
* @param toDatethe to date
* @return the all queries
@@ -229,11 +230,12 @@ public class LensQueryCommands extends BaseLensCommand {
@CliOption(key = {"state"}, mandatory = false, help = "")
String state,
@CliOption(key = {"name"}, mandatory = false, help = "")
String queryName,
@CliOption(key = {"user"}, mandatory = false, help =
"") String user,
+@CliOption(key = {"driver"}, mandatory = false, help =
"") String driver,
@CliOption(key = {"fromDate"}, mandatory = false, unspecifiedDefaultValue
= "-1", help
= "") long fromDate,
@CliOption(key = {"toDate"}, mandatory = false, unspecifiedDefaultValue =
"" + Long.MAX_VALUE, help
= "") long toDate) {
-List handles = getClient().getQueries(state, queryName, user,
fromDate, toDate);
+List handles = getClient().getQueries(state, queryName, user,
driver, fromDate, toDate);
if (handles != null && !handles.isEmpty()) {
return
Joiner.on("\n").skipNulls().join(handles).concat("\n").concat("Total number of
queries: "
+ handles.size());
http://git-wip-us.apache.org/repos/asf/lens/blob/0e4c18cb/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index 48b3ebb..6ebfff7 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -230,7 +230,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
String qh = qCom.executeQuery(sql, true, "testQuery1");
String user = qCom.getClient().getLensStatement(new
QueryHandle(UUID.fromString(qh)))
.getQuery().getSubmittedUser();
-String result = qCom.getAllQueries("", "testQuery1", user, -1,
Long.MAX_VALUE);
+String result = qCom.getAllQueries("", "testQuery1", user, "", -1,
Long.MAX_VALUE);
// this is because previous query has run two query handle will be there
assertTrue(result.contains(qh), result);
assertTrue(result.contains("Total number of queries"));
@@ -249,7 +249,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
}
// Check that query name searching is 'ilike'
-String result2 = qCom.getAllQueries("", "query", "all", -1,
Long.MAX_VALUE);
+String result2 = qCom.getAllQueries("", "query", "all", "", -1,
Long.MAX_VALUE);
assertTrue(result2.contains(qh), result2);
assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
@@ -265,10 +265,10 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
// Kill query is not tested as there is no deterministic way of killing a
query
-result = qCom.getAllQueries("SUCCESSFUL", "", "all", -1, Long.MAX_VALUE);
+result = qCom.getAllQueries("SUCCESSFUL", "", "all", "", -1,
Long.MAX_VALUE);
as
[1/4] lens git commit: LENS-885: Cleanup of Cube test cases
Repository: lens
Updated Branches:
refs/heads/master 7e9e47ea6 -> 7c7c86dae
http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index d16ea4c..36c1dba 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -19,6 +19,7 @@
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.metadata.DateFactory.*;
import static org.apache.lens.cube.parse.CubeTestSetup.*;
import java.util.*;
@@ -55,18 +56,16 @@ public class TestDenormalizationResolver extends
TestQueryRewrite {
@Test
public void testDenormsAsDirectFields() throws ParseException,
LensException, HiveException {
// denorm fields directly available
-String twoDaysITRange =
- "time_range_in(it, '" + CubeTestSetup.getDateUptoHours(TWODAYS_BACK) +
"','"
-+ CubeTestSetup.getDateUptoHours(NOW) + "')";
-String hqlQuery = rewrite("select dim2big1, max(msr3)," + " msr2 from
testCube" + " where " + twoDaysITRange, conf);
+String hqlQuery = rewrite("select dim2big1, max(msr3)," + " msr2 from
testCube" + " where " + TWO_DAYS_RANGE_IT,
+ conf);
String expecteddim2big1 =
getExpectedQuery(cubeName, "select testcube.dim2big1," + "
max(testcube.msr3), sum(testcube.msr2) FROM ", null,
" group by testcube.dim2big1",
getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"),
null);
TestCubeRewriter.compareQueries(hqlQuery, expecteddim2big1);
// with another table
-hqlQuery = rewrite("select dim2big1, citydim.name, max(msr3)," + " msr2
from testCube" + " where " + twoDaysITRange,
- conf);
+hqlQuery = rewrite("select dim2big1, citydim.name, max(msr3)," + " msr2
from testCube" + " where "
+ + TWO_DAYS_RANGE_IT, conf);
String expecteddim2big1WithAnotherTable = getExpectedQuery(cubeName,
"select testcube.dim2big1, citydim.name, max(testcube.msr3),
sum(testcube.msr2) FROM ", " JOIN "
+ getDbName() + "c1_citytable citydim " + "on testcube.cityid =
citydim.id and citydim.dt = 'latest' ", null,
@@ -75,7 +74,7 @@ public class TestDenormalizationResolver extends
TestQueryRewrite {
null);
TestCubeRewriter.compareQueries(hqlQuery,
expecteddim2big1WithAnotherTable);
-hqlQuery = rewrite("select dim2big2, max(msr3)," + " msr2 from testCube" +
" where " + twoDaysITRange, conf);
+hqlQuery = rewrite("select dim2big2, max(msr3)," + " msr2 from testCube" +
" where " + TWO_DAYS_RANGE_IT, conf);
String expecteddim2big2 =
getExpectedQuery(cubeName, "select testcube.dim2big2,
max(testcube.msr3), sum(testcube.msr2) FROM ", null,
" group by testcube.dim2big2",
getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary4"),
@@ -84,8 +83,8 @@ public class TestDenormalizationResolver extends
TestQueryRewrite {
Configuration conf2 = new Configuration(conf);
conf2.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2");
-hqlQuery =
- rewrite("select testdim3.name, dim2big1, max(msr3)," + " msr2 from
testCube" + " where " + twoDaysITRange, conf2);
+hqlQuery = rewrite("select testdim3.name, dim2big1, max(msr3)," + " msr2
from testCube" + " where "
+ + TWO_DAYS_RANGE_IT, conf2);
String expected =
getExpectedQuery(cubeName,
"select testdim3.name, testcube.dim2big1, max(testcube.msr3),
sum(testcube.msr2) FROM ", " JOIN "
@@ -96,9 +95,9 @@ public class TestDenormalizationResolver extends
TestQueryRewrite {
null);
TestCubeRewriter.compareQueries(hqlQuery, expected);
-hqlQuery = rewrite("select dim2big1, max(msr3)," + " msr2 from testCube" +
" where " + twoDaysITRange, conf2);
+hqlQuery = rewrite("select dim2big1, max(msr3)," + " msr2 from testCube" +
" where " + TWO_DAYS_RANGE_IT, conf2);
TestCubeRewriter.compareQueries(hqlQuery, expecteddim2big1);
-hqlQuery = rewrite("select dim2big2, max(msr3)," + " msr2 from testCube" +
" where " + twoDaysITRange, conf2);
+hqlQuery = rewrite("select dim2big2, max(msr3)," + " msr2 from testCube" +
" where " + TWO_DAYS_RANGE_IT, conf2);
TestCubeRewriter.compareQueries(hqlQuery, expecteddim2big2);
}
@@ -190,11 +189,8 @@ public class TestDenormalizationResolver extends
TestQueryRewrite {
@Test
public void
testCubeQueryWithExpressionHavingDenormColumnComingAsDirectColumn() throws
Exception {
-String twoDaysITRange =
- "time_range_in(it, '" + CubeTestSetup.getDateUptoHours(TWODAYS_BACK) +
"','"
-+ CubeTestSetup.getDateUptoHours(NOW) + "')";
-Str
[4/4] lens git commit: LENS-885: Cleanup of Cube test cases
LENS-885: Cleanup of Cube test cases Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7c7c86da Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7c7c86da Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7c7c86da Branch: refs/heads/master Commit: 7c7c86daed2e9907bda92f5ed29e73ed99a9a726 Parents: 7e9e47e Author: Rajat Khandelwal Authored: Fri Dec 11 18:40:59 2015 +0530 Committer: Rajat Khandelwal Committed: Fri Dec 11 18:41:00 2015 +0530 -- .../apache/lens/cube/metadata/CubeColumn.java | 1 - .../lens/cube/metadata/CubeFactTable.java | 1 - .../org/apache/lens/cube/metadata/DateUtil.java | 396 .../lens/cube/metadata/TimePartitionRange.java | 1 - .../apache/lens/cube/metadata/TimeRange.java| 219 + .../apache/lens/cube/metadata/UpdatePeriod.java | 84 +++- .../timeline/EndsAndHolesPartitionTimeline.java | 2 +- .../apache/lens/cube/parse/CandidateFact.java | 5 +- .../cube/parse/CandidateTablePruneCause.java| 2 + .../org/apache/lens/cube/parse/DateUtil.java| 456 -- .../lens/cube/parse/ExpressionResolver.java | 11 +- .../lens/cube/parse/SingleFactHQLContext.java | 2 +- .../lens/cube/parse/StorageTableResolver.java | 2 +- .../org/apache/lens/cube/parse/TimeRange.java | 220 - .../lens/cube/parse/TimerangeResolver.java | 5 +- .../lens/cube/metadata/CubeFactTableTest.java | 1 - .../apache/lens/cube/metadata/DateFactory.java | 196 .../cube/metadata/TestCubeMetastoreClient.java | 115 ++--- .../apache/lens/cube/metadata/TestDateUtil.java | 297 .../apache/lens/cube/parse/CubeTestSetup.java | 191 ++-- .../FieldsCannotBeQueriedTogetherTest.java | 8 +- .../lens/cube/parse/TestAggregateResolver.java | 1 + .../lens/cube/parse/TestBaseCubeQueries.java| 5 +- .../cube/parse/TestBetweenTimeRangeWriter.java | 25 +- .../lens/cube/parse/TestCubeRewriter.java | 459 +-- .../apache/lens/cube/parse/TestDateUtil.java| 299 .../cube/parse/TestDenormalizationResolver.java | 28 +- .../lens/cube/parse/TestExpressionContext.java | 4 +- .../lens/cube/parse/TestExpressionResolver.java | 1 + .../lens/cube/parse/TestJoinResolver.java | 1 + .../lens/cube/parse/TestORTimeRangeWriter.java | 40 +- .../lens/cube/parse/TestQueryMetrics.java | 2 +- .../lens/cube/parse/TestRewriterPlan.java | 2 +- .../apache/lens/cube/parse/TestStorageUtil.java | 98 ++-- .../lens/cube/parse/TestTimeRangeExtractor.java | 33 +- .../lens/cube/parse/TestTimeRangeResolver.java | 2 +- .../lens/cube/parse/TestTimeRangeWriter.java| 48 +- .../parse/TestTimeRangeWriterWithQuery.java | 134 +++--- .../lens/server/query/QueryResultPurger.java| 2 +- 39 files changed, 1693 insertions(+), 1706 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java index a2a00d2..b04532f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java @@ -24,7 +24,6 @@ import java.util.Date; import java.util.Map; import java.util.TimeZone; -import org.apache.lens.cube.parse.TimeRange; import com.google.common.base.Optional; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index d6bfb79..dd0adb7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.metadata; import java.util.*; import org.apache.lens.cube.metadata.UpdatePeriod.UpdatePeriodComparator; -import org.apache.lens.cube.parse.DateUtil; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metad
[3/4] lens git commit: LENS-885: Cleanup of Cube test cases
http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index e5dbde7..c6ce6ad 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -19,9 +19,9 @@
package org.apache.lens.cube.metadata;
+import static org.apache.lens.cube.metadata.DateFactory.*;
import static org.apache.lens.cube.metadata.MetastoreUtil.*;
import static org.apache.lens.cube.metadata.UpdatePeriod.*;
-import static org.apache.lens.cube.parse.CubeTestSetup.DateOffsetProvider;
import static org.apache.lens.server.api.util.LensUtil.getHashMap;
import static org.testng.Assert.*;
@@ -35,7 +35,6 @@ import
org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.TestPartitionTimelines;
-import org.apache.lens.cube.parse.TimeRange;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.util.LensUtil;
@@ -100,11 +99,7 @@ public class TestCubeMetastoreClient {
private static Set cubeExpressions = new HashSet<>();
private static Set joinChains = new HashSet<>();
private static Set dimExpressions = new HashSet<>();
- private static DateOffsetProvider dateOffsetProvider = new
DateOffsetProvider(HOURLY);
- public static Date getDateWithOffset(int i) {
-return dateOffsetProvider.get(i);
- }
/**
* Get the date partition as field schema
@@ -127,7 +122,7 @@ public class TestCubeMetastoreClient {
private static HashMap getTimePartitionByOffsets(Object...
args) {
for (int i = 1; i < args.length; i += 2) {
if (args[i] instanceof Integer) {
-args[i] = getDateWithOffset((Integer) args[i]);
+args[i] = getDateWithOffset(HOURLY, (Integer) args[i]);
}
}
return getHashMap(args);
@@ -169,13 +164,13 @@ public class TestCubeMetastoreClient {
new FieldSchema("msr4", "bigint", "fourth measure"), "Measure4", null,
"COUNT", null));
cubeMeasures.add(new ColumnMeasure(
new FieldSchema("msrstarttime", "int", "measure with start time"),
- "Measure With Starttime", null, null, null, getDateWithOffset(0), null,
null, 0.0, 99.0));
+ "Measure With Starttime", null, null, null, NOW, null, null, 0.0,
99.0));
cubeMeasures.add(new ColumnMeasure(
new FieldSchema("msrendtime", "float", "measure with end time"),
- "Measure With Endtime", null, "SUM", "RS", getDateWithOffset(0),
getDateWithOffset(0), null));
+ "Measure With Endtime", null, "SUM", "RS", NOW, NOW, null));
cubeMeasures.add(new ColumnMeasure(
new FieldSchema("msrcost", "double", "measure with cost"), "Measure With
cost",
- null, "MAX", null, getDateWithOffset(0), getDateWithOffset(0), 100.0));
+ null, "MAX", null, NOW, NOW, 100.0));
cubeMeasures.add(new ColumnMeasure(
new FieldSchema("msrcost2", "bigint", "measure with cost"),
"Measure With cost2", null, "MAX", null, null, null, 100.0, 0.0,
999.0));
@@ -234,12 +229,12 @@ public class TestCubeMetastoreClient {
List locationHierarchyWithStartTime = new ArrayList<>();
locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new
FieldSchema("zipcode2", "int", "zip"),
- "Zip refer2", new TableReference("zipdim", "zipcode"),
getDateWithOffset(0), getDateWithOffset(0),
+ "Zip refer2", new TableReference("zipdim", "zipcode"), NOW, NOW,
100.0, true, 1000L));
locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new
FieldSchema("cityid2", "int", "city"),
- "City refer2", new TableReference("citydim", "id"),
getDateWithOffset(0), null, null));
+ "City refer2", new TableReference("citydim", "id"), NOW, null, null));
locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new
FieldSchema("stateid2", "int", "state"),
- "state refer2", new TableReference("statedim", "id"),
getDateWithOffset(0), null, 100.0));
+ "state refer2", new TableReference("statedim", "id"), NOW, null, 100.0));
locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new
FieldSchema("countryid2", "int", "country"),
"Country refer2", new TableReference("countrydim", "id"), null, null,
null));
locationHierarchyWithStartTime.add(new BaseDimAttribute(new
FieldSchema("regionname2", "string", "region"),
@@ -248,9 +243,10 @@ public class TestCubeMetastoreClient {
cubeDimensions
.add(new Hierarc
[2/4] lens git commit: LENS-885: Cleanup of Cube test cases
http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index fea70b7..3be9406 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -19,12 +19,11 @@
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.metadata.DateFactory.*;
import static org.apache.lens.cube.metadata.UpdatePeriod.*;
import static
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
-import static
org.apache.lens.cube.parse.CubeQueryConfUtil.getValidStorageTablesKey;
-import static
org.apache.lens.cube.parse.CubeQueryConfUtil.getValidUpdatePeriodsKey;
+import static org.apache.lens.cube.parse.CubeQueryConfUtil.*;
import static org.apache.lens.cube.parse.CubeTestSetup.*;
-import static
org.apache.lens.cube.parse.CubeTestSetup.getWhereForMonthlyDailyAndHourly2monthsUnionQuery;
import static org.testng.Assert.*;
@@ -36,6 +35,7 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause;
import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode;
+import org.apache.lens.server.api.LensServerAPITestUtil;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.time.DateUtils;
@@ -61,18 +61,16 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class TestCubeRewriter extends TestQueryRewrite {
- private final String cubeName = CubeTestSetup.TEST_CUBE_NAME;
-
private Configuration conf;
@BeforeTest
public void setupDriver() throws Exception {
-conf = new Configuration();
-conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C0,C1,C2");
-conf.setBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS, true);
-conf.setBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, true);
-conf.setBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, true);
-conf.setBoolean(CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER, false);
+conf = LensServerAPITestUtil.getConfiguration(
+ DRIVER_SUPPORTED_STORAGES, "C0,C1,C2",
+ DISABLE_AUTO_JOINS, true,
+ ENABLE_SELECT_TO_GROUPBY, true,
+ ENABLE_GROUP_BY_TO_SELECT, true,
+ DISABLE_AGGREGATE_RESOLVER, false);
}
@Override
@@ -83,7 +81,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
@Test
public void testQueryWithNow() throws Exception {
LensException e = getLensExceptionInRewrite(
- "select SUM(msr2) from testCube where" + " time_range_in(d_time, 'NOW -
2DAYS', 'NOW')", getConf());
+ "select SUM(msr2) from testCube where " + getTimeRangeString("NOW -
2DAYS", "NOW"), getConf());
assertEquals(e.getErrorCode(),
LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode());
}
@@ -93,19 +91,12 @@ public class TestCubeRewriter extends TestQueryRewrite {
conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS,
BetweenTimeRangeWriter.class, TimeRangeWriter.class);
DateFormat qFmt = new SimpleDateFormat("-MM-dd-HH:mm:ss");
-Calendar qCal = Calendar.getInstance();
-Date toDate = qCal.getTime();
-String qTo = qFmt.format(toDate);
-qCal.setTime(TWODAYS_BACK);
-Date from2DaysBackDate = qCal.getTime();
-String qFrom = qFmt.format(from2DaysBackDate);
-
-CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr15) from
testCube where"
- + " time_range_in(d_time, '" + qFrom + "', '" + qTo + "')", conf);
+String timeRangeString;
+timeRangeString = getTimeRangeString(DAILY, -2, 0, qFmt);
+CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr15) from
testCube where " + timeRangeString, conf);
-DateFormat fmt = UpdatePeriod.CONTINUOUS.format();
-String to = fmt.format(toDate);
-String from = fmt.format(from2DaysBackDate);
+String to = getDateStringWithOffset(DAILY, 0, CONTINUOUS);
+String from = getDateStringWithOffset(DAILY, -2, CONTINUOUS);
String expected = "select SUM((testCube.msr15)) from
TestQueryRewrite.c0_testFact_CONTINUOUS testcube"
+ " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "'
))";
@@ -114,20 +105,17 @@ public class TestCubeRewriter extends TestQueryRewrite {
compareQueries(rewrittenQuery.toHQL(), expected);
//test with msr2 on different fact
-rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where" + "
time_range_in(d_time, '"
- + qFrom + "', '" + qTo + "')", conf);
+rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where " +
timeRangeString, conf);
expected = "select SUM((tes
lens git commit: LENS-552: Union support across storage tables in multi fact query
Repository: lens
Updated Branches:
refs/heads/master bf1053b4a -> 04f5a8223
LENS-552: Union support across storage tables in multi fact query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/04f5a822
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/04f5a822
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/04f5a822
Branch: refs/heads/master
Commit: 04f5a8223f652baa5cfbebec7f8f9a2886df5076
Parents: bf1053b
Author: Rajat Khandelwal
Authored: Thu Dec 24 12:39:33 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Dec 24 12:39:33 2015 +0530
--
.../lens/cube/parse/AggregateResolver.java | 12 +-
.../apache/lens/cube/parse/AliasReplacer.java | 5 +-
.../apache/lens/cube/parse/AutoJoinContext.java | 2 +-
.../apache/lens/cube/parse/CandidateFact.java | 130 ---
.../apache/lens/cube/parse/ColumnResolver.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 69 +-
.../lens/cube/parse/CubeQueryRewriter.java | 2 +-
.../apache/lens/cube/parse/DefaultQueryAST.java | 74 +++
.../cube/parse/DenormalizationResolver.java | 18 +--
.../apache/lens/cube/parse/DimHQLContext.java | 6 +-
.../lens/cube/parse/DimOnlyHQLContext.java | 15 +--
.../lens/cube/parse/ExpressionResolver.java | 24 ++--
.../apache/lens/cube/parse/FactHQLContext.java | 65 --
.../apache/lens/cube/parse/GroupbyResolver.java | 2 -
.../org/apache/lens/cube/parse/HQLParser.java | 1 -
.../apache/lens/cube/parse/JoinResolver.java| 17 +--
.../lens/cube/parse/MultiFactHQLContext.java| 46 +++
.../org/apache/lens/cube/parse/QueryAST.java| 86
.../lens/cube/parse/SingleFactHQLContext.java | 96 --
.../parse/SingleFactMultiStorageHQLContext.java | 52 +---
.../SingleFactSingleStorageHQLContext.java | 85
.../lens/cube/parse/StorageTableResolver.java | 43 +++---
.../apache/lens/cube/metadata/DateFactory.java | 11 ++
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +-
.../lens/cube/parse/TestBaseCubeQueries.java| 24 ++--
.../lens/cube/parse/TestCubeRewriter.java | 53 ++--
26 files changed, 481 insertions(+), 461 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index 39bd1cc..fd7036a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -47,8 +47,6 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
class AggregateResolver implements ContextRewriter {
- public AggregateResolver(Configuration conf) {
- }
@Override
public void rewriteContext(CubeQueryContext cubeql) throws LensException {
@@ -166,7 +164,7 @@ class AggregateResolver implements ContextRewriter {
String colname;
if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) {
- colname = ((ASTNode) node.getChild(0)).getText();
+ colname = node.getChild(0).getText();
} else {
// node in 'alias.column' format
ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL,
Identifier);
@@ -193,15 +191,9 @@ class AggregateResolver implements ContextRewriter {
throw new
LensException(LensCubeErrorCode.NO_DEFAULT_AGGREGATE.getLensErrorInfo(),
colname);
}
ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION));
-fnroot.setParent(node.getParent());
-
ASTNode fnIdentNode = new ASTNode(new
CommonToken(HiveParser.Identifier, aggregateFn));
-fnIdentNode.setParent(fnroot);
fnroot.addChild(fnIdentNode);
-
-node.setParent(fnroot);
fnroot.addChild(node);
-
return fnroot;
}
} else {
@@ -224,7 +216,7 @@ class AggregateResolver implements ContextRewriter {
String colname;
if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) {
-colname = ((ASTNode) node.getChild(0)).getText();
+colname = node.getChild(0).getText();
} else {
// node in 'alias.column' format
ASTNode colIdent = (ASTNode) node.getChild(1);
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
b/lens-cube/src/main/java/org/apache/
lens git commit: LENS-836: Query commands in CLI should take default value for query handle as the last executed query
Repository: lens
Updated Branches:
refs/heads/master 04f5a8223 -> 7a89db13e
LENS-836: Query commands in CLI should take default value for query handle as
the last executed query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7a89db13
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7a89db13
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7a89db13
Branch: refs/heads/master
Commit: 7a89db13e74984de81d840dc015c4ba59471d785
Parents: 04f5a82
Author: Rajat Khandelwal
Authored: Thu Dec 24 13:19:12 2015 +0530
Committer: Rajat Khandelwal
Committed: Thu Dec 24 13:19:12 2015 +0530
--
.../lens/cli/commands/LensQueryCommands.java| 76
.../apache/lens/cli/TestLensQueryCommands.java | 7 +-
.../java/org/apache/lens/client/LensClient.java | 9 +--
.../org/apache/lens/client/LensStatement.java | 12 +---
src/site/apt/user/cli.apt | 10 +--
5 files changed, 62 insertions(+), 52 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7a89db13/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
--
diff --git
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index e3c08ff..a29600d 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -61,6 +61,8 @@ import com.google.common.base.Joiner;
+ " <<>>,\n"
+ " will be parsed as <<>>")
public class LensQueryCommands extends BaseLensCommand {
+ private static final String DEFAULT_QUERY_HANDLE_DESCRIPTION =
+"If not provided, takes last query handle interacted with.";
/**
* Execute query.
@@ -110,7 +112,6 @@ public class LensQueryCommands extends BaseLensCommand {
*/
private String formatResultSet(LensClient.LensClientResultSetWithStats rs) {
StringBuilder b = new StringBuilder();
-int numRows = 0;
if (rs.getResultSet() != null) {
QueryResultSetMetadata resultSetMetadata =
rs.getResultSet().getResultSetMetadata();
for (ResultColumn column : resultSetMetadata.getColumns()) {
@@ -125,7 +126,7 @@ public class LensQueryCommands extends BaseLensCommand {
PersistentQueryResult temp = (PersistentQueryResult) r;
b.append("Results of query stored at :
").append(temp.getPersistedURI()).append(" ");
if (null != temp.getNumRows()) {
- b.append(temp.getNumRows() + " rows ");
+ b.append(temp.getNumRows()).append(" rows ");
}
}
}
@@ -139,20 +140,33 @@ public class LensQueryCommands extends BaseLensCommand {
return b.toString();
}
+ public String getOrDefaultQueryHandleString(String queryHandleString) {
+if (queryHandleString != null) {
+ return queryHandleString;
+}
+if (getClient().getStatement().getQuery() != null) {
+ return getClient().getStatement().getQueryHandleString();
+}
+throw new IllegalArgumentException("Query handle not provided and no
queries interacted with in the session.");
+ }
+
/**
* Gets the status.
*
* @param qh the qh
* @return the status
*/
- @CliCommand(value = "query status", help = "Fetch status of executed query
having query handle ")
+ @CliCommand(value = "query status",
+help = "Fetch status of executed query having query handle .
" + DEFAULT_QUERY_HANDLE_DESCRIPTION)
public String getStatus(
-@CliOption(key = {"", "query_handle"}, mandatory = true, help =
"") String qh) {
-QueryStatus status = getClient().getQueryStatus(new
QueryHandle(UUID.fromString(qh)));
+@CliOption(key = {"", "query_handle"}, mandatory = false, help =
"") String qh) {
+qh = getOrDefaultQueryHandleString(qh);
+QueryHandle handle = QueryHandle.fromString(qh);
+QueryStatus status = getClient().getQueryStatus(handle);
if (status == null) {
- return "Unable to find status for " + qh;
+ return "Unable to find status for " + handle;
}
-return status.toString();
+return "Query Handle: " + qh + "\n" + status.toString();
}
/**
@@ -161,15 +175,15 @@ public class LensQueryCommands extends BaseLensCommand {
* @param qh the qh
* @return the query
*/
- @CliCommand(value = "query details", help = "Get query details of query with
handle ")
+ @CliCommand(value = "query details",
+help = "Get query details of query with handle ." +
DEFAULT_QUERY_HANDLE_DESCRIPTION)
public String getDetails(
-@CliOption(key = {"", "query_handle"}, mandatory = true, help
- = "") String qh) {
+@CliOption(key = {"", "query_handle"}, mandator
lens git commit: LENS-913: Seeing DateUtil exception in all cube tests
Repository: lens
Updated Branches:
refs/heads/master 71cf9ffd0 -> c73d5844b
LENS-913: Seeing DateUtil exception in all cube tests
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c73d5844
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c73d5844
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c73d5844
Branch: refs/heads/master
Commit: c73d5844b6e74eb3184b70aea283bf26b1bd267c
Parents: 71cf9ff
Author: Rajat Khandelwal
Authored: Fri Jan 8 17:23:19 2016 +0530
Committer: Rajat Khandelwal
Committed: Fri Jan 8 17:23:19 2016 +0530
--
.../lens/cube/metadata/CubeFactTable.java | 45 +++-
.../org/apache/lens/cube/metadata/DateUtil.java | 3 +-
.../lens/cube/metadata/CubeFactTableTest.java | 4 ++
3 files changed, 30 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/c73d5844/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index dd0adb7..b1fec8c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata;
import java.util.*;
import org.apache.lens.cube.metadata.UpdatePeriod.UpdatePeriodComparator;
+import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -28,7 +29,9 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class CubeFactTable extends AbstractCubeTable {
private String cubeName;
private final Map> storageUpdatePeriods;
@@ -80,7 +83,7 @@ public class CubeFactTable extends AbstractCubeTable {
}
private static Map> getUpdatePeriods(String name,
Map props) {
-Map> storageUpdatePeriods = new HashMap>();
+Map> storageUpdatePeriods = new HashMap<>();
String storagesStr = props.get(MetastoreUtil.getFactStorageListKey(name));
if (!StringUtils.isBlank(storagesStr)) {
String[] storages = storagesStr.split(",");
@@ -88,7 +91,7 @@ public class CubeFactTable extends AbstractCubeTable {
String updatePeriodStr =
props.get(MetastoreUtil.getFactUpdatePeriodKey(name, storage));
if (StringUtils.isNotBlank(updatePeriodStr)) {
String[] periods = updatePeriodStr.split(",");
- Set updatePeriods = new TreeSet();
+ Set updatePeriods = new TreeSet<>();
for (String period : periods) {
updatePeriods.add(UpdatePeriod.valueOf(period));
}
@@ -321,20 +324,28 @@ public class CubeFactTable extends AbstractCubeTable {
getProperties().put(MetastoreConstants.FACT_AGGREGATED_PROPERTY,
Boolean.toString(isAggregated));
}
- public Date getAbsoluteStartTime() {
+ public Date getDateFromProperty(String propKey, boolean relative, boolean
start) {
+String prop = getProperties().get(propKey);
try {
- return
DateUtil.resolveAbsoluteDate(getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME));
-} catch (Exception e) {
- return new Date(Long.MIN_VALUE);
+ if (StringUtils.isNotBlank(prop)) {
+if (relative) {
+ return DateUtil.resolveRelativeDate(prop, now());
+} else {
+ return DateUtil.resolveAbsoluteDate(prop);
+}
+ }
+} catch (LensException e) {
+ log.error("unable to parse {} {} date: {}", relative ? "relative" :
"absolute", start ? "start" : "end", prop);
}
+return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE;
+ }
+
+ public Date getAbsoluteStartTime() {
+return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_START_TIME,
false, true);
}
public Date getRelativeStartTime() {
-try {
- return
DateUtil.resolveRelativeDate(getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME),
now());
-} catch (Exception e) {
- return new Date(Long.MIN_VALUE);
-}
+return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_START_TIME,
true, true);
}
public Date getStartTime() {
@@ -342,19 +353,11 @@ public class CubeFactTable extends AbstractCubeTable {
}
public Date getAbsoluteEndTime() {
-try {
- return
DateUtil.resolveAbsoluteDate(getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME));
-} catch (Exception e) {
- return new Date(Long.MAX_VALUE);
-}
+return getDateFromPrope
[2/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
index d9e442d..2cf92b9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.metadata.DateFactory.*;
import static org.apache.lens.cube.parse.CubeTestSetup.*;
+import static org.apache.lens.cube.parse.TestCubeRewriter.compareQueries;
import static org.testng.Assert.*;
@@ -28,7 +29,6 @@ import java.util.*;
import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.metadata.SchemaGraph.TableRelationship;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
@@ -45,7 +45,6 @@ import org.testng.annotations.Test;
public class TestJoinResolver extends TestQueryRewrite {
private static HiveConf hconf = new HiveConf(TestJoinResolver.class);
- private CubeMetastoreClient metastore;
@BeforeTest
public void setupInstance() throws Exception {
@@ -53,124 +52,25 @@ public class TestJoinResolver extends TestQueryRewrite {
hconf.setBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS, false);
hconf.setBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, true);
hconf.setBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, true);
+hconf.setBoolean(CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER, false);
hconf.setBoolean(CubeQueryConfUtil.ENABLE_FLATTENING_FOR_BRIDGETABLES,
true);
-this.metastore = CubeMetastoreClient.getInstance(hconf);
}
@AfterTest
public void closeInstance() throws Exception {
}
- // testBuildGraph - graph correctness
- @Test
- public void testBuildGraph() throws Exception {
-SchemaGraph schemaGraph = metastore.getSchemaGraph();
-CubeInterface cube = metastore.getCube(CubeTestSetup.TEST_CUBE_NAME);
-Map> graph =
schemaGraph.getCubeGraph(cube);
-printGraph(graph);
-Assert.assertNotNull(graph);
-
-// Let's do some lookups
-Set dim4Edges =
graph.get(metastore.getDimension("testdim4"));
-Assert.assertNull(dim4Edges);
-dim4Edges = graph.get(metastore.getDimension("testdim3"));
-Assert.assertNotNull(dim4Edges);
-Assert.assertEquals(1, dim4Edges.size());
-
-List edges = new
ArrayList(dim4Edges);
-TableRelationship dim4edge = edges.get(0);
-Assert.assertEquals("id", dim4edge.getToColumn());
-Assert.assertEquals(metastore.getDimension("testdim4"),
dim4edge.getToTable());
-Assert.assertEquals("testdim4id", dim4edge.getFromColumn());
-Assert.assertEquals(metastore.getDimension("testdim3"),
dim4edge.getFromTable());
- }
-
- private void searchPaths(AbstractCubeTable source, AbstractCubeTable target,
SchemaGraph graph) {
-SchemaGraph.GraphSearch search = new SchemaGraph.GraphSearch(source,
target, graph);
-List joinPaths = search.findAllPathsToTarget();
-
-System.out.println("@@ " + source + " ==> " + target + " paths =");
-int i = 0;
-for (SchemaGraph.JoinPath jp : joinPaths) {
- Assert.assertEquals(jp.getEdges().get(0).getToTable(), source);
- Assert.assertEquals(jp.getEdges().get(jp.getEdges().size() -
1).getFromTable(), target);
- Collections.reverse(jp.getEdges());
- System.out.println(++i + " " + jp.getEdges());
-}
- }
-
- @Test
- public void testFindChain() throws Exception {
-SchemaGraph schemaGraph = metastore.getSchemaGraph();
-schemaGraph.print();
-
-// Search For all cubes and all dims to make sure that search terminates
-for (CubeInterface cube : metastore.getAllCubes()) {
- for (Dimension dim : metastore.getAllDimensions()) {
-searchPaths(dim, (AbstractCubeTable) cube, schemaGraph);
- }
-}
-
-for (Dimension dim : metastore.getAllDimensions()) {
- for (Dimension otherDim : metastore.getAllDimensions()) {
-if (otherDim != dim) {
- searchPaths(dim, otherDim, schemaGraph);
-}
- }
-}
-
-// Assert for testcube
-CubeInterface testCube = metastore.getCube("testcube");
-Dimension zipDim = metastore.getDimension("zipdim");
-Dimension cityDim = metastore.getDimension("citydim");
-Dimension testDim2 = metastore.getDimension("testDim2");
-
-SchemaGraph.GraphSearch search = new SchemaGraph.GraphSearch(zipDim,
(AbstractCubeTable) testCube, schemaGraph);
-
-List paths = search.findAllPathsToTarget();
-Assert.assertEquals(6, paths.size());
-validatePath(paths.get(0), zipDim, (AbstractCubeTable) testCube);
-validatePath(paths.get(1), zipDim, cityDim, (A
[5/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
index 1a83d09..33ec9d9 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java
@@ -28,7 +28,9 @@ import org.apache.lens.cube.error.ColUnAvailableInTimeRange;
import org.apache.lens.cube.error.ColUnAvailableInTimeRangeException;
import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.JoinPath;
import
org.apache.lens.cube.parse.DenormalizationResolver.ReferencedQueriedColumn;
+import org.apache.lens.cube.parse.join.AutoJoinContext;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
@@ -197,14 +199,14 @@ class TimerangeResolver implements ContextRewriter {
if (!column.isColumnAvailableInTimeRange(range)) {
log.info("Timerange queried is not in column life for {}, Removing
join paths containing the column", column);
// Remove join paths containing this column
- Map, List> allPaths =
joinContext.getAllPaths();
+ Map, List> allPaths =
joinContext.getAllPaths();
for (Aliased dimension : allPaths.keySet()) {
-List joinPaths = allPaths.get(dimension);
-Iterator joinPathIterator =
joinPaths.iterator();
+List joinPaths = allPaths.get(dimension);
+Iterator joinPathIterator = joinPaths.iterator();
while (joinPathIterator.hasNext()) {
- SchemaGraph.JoinPath path = joinPathIterator.next();
+ JoinPath path = joinPathIterator.next();
if (path.containsColumnOfTable(col, (AbstractCubeTable)
cubeql.getCube())) {
log.info("Removing join path: {} as columns :{} is not
available in the range", path, col);
joinPathIterator.remove();
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
new file mode 100644
index 000..993955a
--- /dev/null
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
@@ -0,0 +1,719 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse.join;
+
+import java.util.*;
+
+import org.apache.lens.cube.error.LensCubeErrorCode;
+import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.JoinPath;
+import org.apache.lens.cube.metadata.join.TableRelationship;
+import org.apache.lens.cube.parse.*;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.parse.JoinType;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Store join chain information resolved by join resolver
+ */
+@Slf4j
+public class AutoJoinContext {
+ // Map of a joined table to list of all possible paths from that table to
+ // the target
+ private final Map, List> allPaths;
+ private Set requiredDimensions;
+ @Getter
+ // Map of joined table to the join type (if provided by user)
+ private final Map tableJoinTypeMap;
+
+ // True if joins were resolved automatically
+ private boolean joinsResolved;
+ // Target table for the auto join resolver
+ private final AbstractCubeTable autoJoinTarget;
+ // Configuration string to control join type
+ private String joinTypeCfg;
+
+ // Map of a joined table to its columns which are part of any of the join
+ // paths. This is used in candidate table resolver
+ @Getter
+ private Map>>
joinPathFromC
[3/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 61fb73c..4810559 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -135,7 +135,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
@Test
public void testCubeQuery() throws Exception {
CubeQueryContext rewrittenQuery =
- rewriteCtx("cube select" + " SUM(msr2) from testCube where " +
TWO_DAYS_RANGE, getConfWithStorages("C2"));
+ rewriteCtx("cube select SUM(msr2) from testCube where " +
TWO_DAYS_RANGE, getConfWithStorages("C2"));
String expected =
getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ",
null, null,
getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact"));
@@ -151,7 +151,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
conf.set(DRIVER_SUPPORTED_STORAGES, "C1,C2,C4");
CubeQueryContext cubeQueryContext =
- rewriteCtx("cube select" + " SUM(msr2) from testCube where " +
THIS_YEAR_RANGE, conf);
+ rewriteCtx("cube select SUM(msr2) from testCube where " +
THIS_YEAR_RANGE, conf);
PruneCauses pruneCause =
cubeQueryContext.getFactPruningMsgs();
int lessDataCauses = 0;
for (Map.Entry> entry :
pruneCause.entrySet()) {
@@ -167,7 +167,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
@Test
public void testLightestFactFirst() throws Exception {
// testFact is lighter than testFact2.
-String hqlQuery = rewrite("cube select" + " SUM(msr2) from testCube where
" + TWO_DAYS_RANGE, getConfWithStorages(
+String hqlQuery = rewrite("cube select SUM(msr2) from testCube where " +
TWO_DAYS_RANGE, getConfWithStorages(
"C2"));
String expected =
getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ",
null, null,
@@ -202,7 +202,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
@Test
public void testDerivedCube() throws ParseException, LensException,
HiveException, ClassNotFoundException {
CubeQueryContext rewrittenQuery =
- rewriteCtx("cube select" + " SUM(msr2) from derivedCube where " +
TWO_DAYS_RANGE, getConfWithStorages("C2"));
+ rewriteCtx("cube select SUM(msr2) from derivedCube where " +
TWO_DAYS_RANGE, getConfWithStorages("C2"));
String expected =
getExpectedQuery(DERIVED_CUBE_NAME, "select sum(derivedCube.msr2) FROM
", null, null,
getWhereForDailyAndHourly2days(DERIVED_CUBE_NAME, "C2_testfact"));
@@ -211,7 +211,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
assertNotNull(rewrittenQuery.getNonExistingParts());
LensException th = getLensExceptionInRewrite(
- "select SUM(msr4) from derivedCube" + " where " + TWO_DAYS_RANGE,
getConf());
+ "select SUM(msr4) from derivedCube where " + TWO_DAYS_RANGE, getConf());
assertEquals(th.getErrorCode(),
LensCubeErrorCode.COLUMN_NOT_FOUND.getLensErrorInfo().getErrorCode());
// test join
@@ -219,27 +219,28 @@ public class TestCubeRewriter extends TestQueryRewrite {
conf.setBoolean(DISABLE_AUTO_JOINS, false);
String hqlQuery;
-hqlQuery = rewrite("cube select" + " testdim2.name, SUM(msr2) from
derivedCube where " + TWO_DAYS_RANGE, conf);
+/*
+Accessing join chains from derived cubes are not supported yet.
+hqlQuery = rewrite("cube select dim2chain.name, SUM(msr2) from derivedCube
where " + TWO_DAYS_RANGE, conf);
expected =
- getExpectedQuery(DERIVED_CUBE_NAME, "select testdim2.name,
sum(derivedCube.msr2) FROM ", " JOIN "
- + getDbName() + "c1_testdim2tbl testdim2 ON derivedCube.dim2 = "
- + " testdim2.id and (testdim2.dt = 'latest') ", null, "group by
(testdim2.name)", null,
+ getExpectedQuery(DERIVED_CUBE_NAME, "select dim2chain.name,
sum(derivedCube.msr2) FROM ", " JOIN "
+ + getDbName() + "c1_testdim2tbl dim2chain ON derivedCube.dim2 = "
+ + " dim2chain.id and (dim2chain.dt = 'latest') ", null, "group by
(dim2chain.name)", null,
getWhereForDailyAndHourly2days(DERIVED_CUBE_NAME, "c1_summary2"));
compareQueries(hqlQuery, expected);
// Test that explicit join query passes with join resolver disabled
conf.setBoolean(DISABLE_AUTO_JOINS, true);
-List joinWhereConds = new ArrayList();
-joinWhereConds.add(StorageUtil.getWherePartClause("dt", "testdim2",
StorageConstants.getPartitionsForLatest()));
hqlQuery =
- rewrite("cube select" + " testdim2.name, SUM(msr2) from derivedCube "
-+ " inner join testdim2 on der
[7/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/908530f5 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/908530f5 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/908530f5 Branch: refs/heads/master Commit: 908530f5883ae8844c6a16cb5564c926cc10bf19 Parents: c73d584 Author: Amareshwari Sriramadasu Authored: Mon Jan 11 13:24:32 2016 +0530 Committer: Rajat Khandelwal Committed: Mon Jan 11 13:24:33 2016 +0530 -- lens-api/src/main/resources/cube-0.1.xsd| 36 +- lens-api/src/main/resources/lens-errors.conf| 11 +- .../lens/cli/TestLensDimensionCommands.java | 28 +- .../resources/cube_with_no_weight_facts.xml | 8 +- lens-cli/src/test/resources/sample-cube.xml | 14 +- lens-cli/src/test/resources/test-dimension.xml | 22 +- .../lens/cube/error/LensCubeErrorCode.java | 6 +- .../lens/cube/metadata/AbstractBaseTable.java | 53 +- .../lens/cube/metadata/AbstractCubeTable.java | 26 +- .../org/apache/lens/cube/metadata/Cube.java | 111 ++- .../apache/lens/cube/metadata/CubeColumn.java | 1 - .../lens/cube/metadata/CubeDimensionTable.java | 21 +- .../lens/cube/metadata/CubeFactTable.java | 15 +- .../lens/cube/metadata/CubeMetastoreClient.java | 11 +- .../apache/lens/cube/metadata/DerivedCube.java | 31 +- .../apache/lens/cube/metadata/Dimension.java| 20 +- .../apache/lens/cube/metadata/JoinChain.java| 16 +- .../cube/metadata/ReferencedDimAtrribute.java | 195 - .../cube/metadata/ReferencedDimAttribute.java | 115 +++ .../apache/lens/cube/metadata/SchemaGraph.java | 377 - .../lens/cube/metadata/join/JoinPath.java | 101 +++ .../cube/metadata/join/TableRelationship.java | 46 + .../apache/lens/cube/parse/AutoJoinContext.java | 760 - .../apache/lens/cube/parse/CandidateDim.java| 16 +- .../lens/cube/parse/CubeQueryContext.java | 94 +-- .../cube/parse/DenormalizationResolver.java | 74 +- .../apache/lens/cube/parse/DimHQLContext.java | 4 +- .../apache/lens/cube/parse/FieldValidator.java | 9 +- .../org/apache/lens/cube/parse/HQLParser.java | 2 +- .../org/apache/lens/cube/parse/JoinClause.java | 144 .../apache/lens/cube/parse/JoinResolver.java| 262 +- .../org/apache/lens/cube/parse/JoinTree.java| 164 .../org/apache/lens/cube/parse/StorageUtil.java | 8 +- .../lens/cube/parse/TimerangeResolver.java | 10 +- .../lens/cube/parse/join/AutoJoinContext.java | 719 .../apache/lens/cube/parse/join/JoinClause.java | 139 +++ .../apache/lens/cube/parse/join/JoinTree.java | 164 .../apache/lens/cube/parse/join/JoinUtils.java | 49 ++ .../cube/metadata/TestCubeMetastoreClient.java | 284 +-- .../apache/lens/cube/parse/CubeTestSetup.java | 843 ++- .../FieldsCannotBeQueriedTogetherTest.java | 11 +- .../lens/cube/parse/TestBaseCubeQueries.java| 26 +- .../lens/cube/parse/TestCubeRewriter.java | 279 +++--- .../cube/parse/TestDenormalizationResolver.java | 140 +-- .../lens/cube/parse/TestExpressionContext.java | 20 +- .../lens/cube/parse/TestExpressionResolver.java | 102 +-- .../lens/cube/parse/TestJoinResolver.java | 534 +--- .../lens/cube/parse/TestQueryRewrite.java | 10 + .../lens/cube/parse/TestRewriterPlan.java | 10 +- .../parse/TestTimeRangeWriterWithQuery.java | 53 +- .../src/main/resources/cube-queries.sql | 74 +- lens-examples/src/main/resources/customer.xml | 4 +- .../src/main/resources/dimension-queries.sql| 14 +- lens-examples/src/main/resources/sales-cube.xml | 12 +- .../src/main/resources/sample-cube.xml | 21 +- .../main/resources/sample-db-only-dimension.xml | 24 +- .../src/main/resources/sample-dimension.xml | 24 +- .../apache/lens/server/metastore/JAXBUtils.java | 54 +- .../server/metastore/TestMetastoreService.java | 17 +- .../apache/lens/storage/db/TestDBStorage.java | 11 +- 60 files changed, 3156 insertions(+), 3293 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-api/src/main/resources/cube-0.1.xsd -- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 4092133..d195b08 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -393,25 +393,13 @@ - + -Reference specifiction needs to be specified if the attribute is a reference attribute. It -can either be table reference or a chained col
[1/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
Repository: lens
Updated Branches:
refs/heads/master c73d5844b -> 908530f58
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index e0c0923..b0044da 100644
---
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -309,8 +309,7 @@ public class TestMetastoreService extends LensJerseyTest {
XChainColumn xcc = new XChainColumn();
xcc.setChainName("chain1");
xcc.setRefCol("col2");
-xd3.setRefSpec(cubeObjectFactory.createXDimAttributeRefSpec());
-xd3.getRefSpec().getChainRefColumn().add(xcc);
+xd3.getChainRefColumn().add(xcc);
xd3.setNumDistinctValues(1000L);
// add attribute with complex type
@@ -602,7 +601,7 @@ public class TestMetastoreService extends LensJerseyTest {
boolean chainValidated = false;
for (XDimAttribute attr : actual.getDimAttributes().getDimAttribute()) {
if (attr.getName().equalsIgnoreCase("testdim2col2")) {
-
assertEquals(attr.getRefSpec().getChainRefColumn().get(0).getDestTable(),
"testdim");
+ assertEquals(attr.getChainRefColumn().get(0).getDestTable(),
"testdim");
chainValidated = true;
break;
}
@@ -619,7 +618,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(hcube.getDimAttributeByName("testdim2col2").getDescription(), "ref
chained dimension");
assertEquals(((BaseDimAttribute)
hcube.getDimAttributeByName("dim4")).getType(),
"struct,c:map>>");
- ReferencedDimAtrribute testdim2col2 = (ReferencedDimAtrribute)
hcube.getDimAttributeByName("testdim2col2");
+ ReferencedDimAttribute testdim2col2 = (ReferencedDimAttribute)
hcube.getDimAttributeByName("testdim2col2");
assertEquals(testdim2col2.getType(), "string");
assertEquals(testdim2col2.getChainRefColumns().get(0).getChainName(),
"chain1");
assertEquals(testdim2col2.getChainRefColumns().get(0).getRefColumn(),
"col2");
@@ -1095,8 +1094,7 @@ public class TestMetastoreService extends LensJerseyTest {
XChainColumn xcc = new XChainColumn();
xcc.setChainName("chain1");
xcc.setRefCol("col2");
-hd3.setRefSpec(cubeObjectFactory.createXDimAttributeRefSpec());
-hd3.getRefSpec().getChainRefColumn().add(xcc);
+hd3.getChainRefColumn().add(xcc);
hd3.setNumDistinctValues(1000L);
hierarchy.getDimAttribute().add(hd3);
xd4.setHierarchy(hierarchy);
@@ -1106,8 +1104,7 @@ public class TestMetastoreService extends LensJerseyTest {
xd5.setType("INT");
xd5.setDescription("ref column");
xd5.setDisplayString("Column5");
-xd5.setRefSpec(cubeObjectFactory.createXDimAttributeRefSpec());
-xd5.getRefSpec().getChainRefColumn().add(xcc);
+xd5.getChainRefColumn().add(xcc);
xd5.getValues().add("1");
xd5.getValues().add("2");
xd5.getValues().add("3");
@@ -1225,7 +1222,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(col4h2.getType(), "string");
assertEquals(col4h2.getDescription(), "base column");
assertEquals(col4h2.getDisplayString(), "Column4-h2");
- ReferencedDimAtrribute col4h3 = (ReferencedDimAtrribute)
col4.getHierarchy().get(2);
+ ReferencedDimAttribute col4h3 = (ReferencedDimAttribute)
col4.getHierarchy().get(2);
assertEquals(col4h3.getName(), "col4-h3");
assertEquals(col4h3.getDescription(), "ref column");
assertEquals(col4h3.getDisplayString(), "Column4-h3");
@@ -1234,7 +1231,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(col4h3.getChainRefColumns().get(0).getRefColumn(), "col2");
assertEquals(col4h3.getNumOfDistinctValues().get(), (Long) 1000L);
assertNotNull(dim.getAttributeByName("col5"));
- ReferencedDimAtrribute col5 = (ReferencedDimAtrribute)
dim.getAttributeByName("col5");
+ ReferencedDimAttribute col5 = (ReferencedDimAttribute)
dim.getAttributeByName("col5");
assertEquals(col5.getDescription(), "ref column");
assertEquals(col5.getDisplayString(), "Column5");
assertEquals(col5.getType(), "int");
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
--
diff --git
a/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
b/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
index 92a0027..55c32e8 100644
---
a/lens-storage-db/src/test/java/org/apache/l
[6/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/JoinPath.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/JoinPath.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/JoinPath.java
new file mode 100644
index 000..48f04bb
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/JoinPath.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.join;
+
+import java.util.*;
+
+import org.apache.lens.cube.metadata.AbstractCubeTable;
+
+/**
+ * A list of table relationships that can be combined to get a join clause
+ */
+public class JoinPath {
+ final List edges;
+ // Store the map of a table against all columns of that table which are in
the path
+ private Map> columnsForTable = new
HashMap<>();
+
+ public JoinPath() {
+edges = new ArrayList<>();
+ }
+
+ public JoinPath(JoinPath other) {
+edges = new ArrayList<>(other.edges);
+ }
+
+ public void initColumnsForTable() {
+if (!columnsForTable.isEmpty()) {
+ // already initialized
+ return;
+}
+for (TableRelationship edge : edges) {
+ addColumnsForEdge(edge);
+}
+ }
+
+ public void addEdge(TableRelationship edge) {
+edges.add(edge);
+ }
+
+ public boolean isEmpty() {
+return edges.isEmpty();
+ }
+
+ public List getEdges() {
+return edges;
+ }
+
+ private void addColumnsForEdge(TableRelationship edge) {
+addColumn(edge.getFromTable(), edge.getFromColumn());
+addColumn(edge.getToTable(), edge.getToColumn());
+ }
+
+ private void addColumn(AbstractCubeTable table, String column) {
+if (table == null || column == null) {
+ return;
+}
+List columns = columnsForTable.get(table);
+if (columns == null) {
+ columns = new ArrayList<>();
+ columnsForTable.put(table, columns);
+}
+columns.add(column);
+ }
+
+ public List getColumnsForTable(AbstractCubeTable table) {
+return columnsForTable.get(table);
+ }
+
+ public Set getAllTables() {
+return columnsForTable.keySet();
+ }
+
+ public boolean containsColumnOfTable(String column, AbstractCubeTable table)
{
+for (TableRelationship edge : edges) {
+ if ((table.equals(edge.getFromTable()) &&
column.equals(edge.getFromColumn()))
+|| table.equals(edge.getToTable()) &&
column.equals(edge.getToColumn())) {
+return true;
+ }
+}
+return false;
+ }
+
+ public String toString() {
+return edges.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/TableRelationship.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/TableRelationship.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/TableRelationship.java
new file mode 100644
index 000..dabb9ef
--- /dev/null
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/join/TableRelationship.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.join;
+
+import org.apache.lens.cube.metadata.AbstractCubeTable;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import l
[4/7] lens git commit: LENS-735 : Remove accepting TableReferences for ReferenceDimAttribute
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index ad20ae1..4366938 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -31,7 +31,7 @@ import java.util.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
-import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
+import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol;
import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
@@ -519,14 +519,10 @@ public class CubeTestSetup {
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("d_time",
"timestamp", "d time")));
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("processing_time",
"timestamp", "processing time")));
List locationHierarchy = new
ArrayList();
-locationHierarchy.add(new ReferencedDimAtrribute(new
FieldSchema("zipcode", "int", "zip"), "Zip refer",
- new TableReference("zipdim", "code")));
-locationHierarchy.add(new ReferencedDimAtrribute(new FieldSchema("cityid",
"int", "city"), "City refer",
- new TableReference("citydim", "id")));
-locationHierarchy.add(new ReferencedDimAtrribute(new
FieldSchema("stateid", "int", "state"), "State refer",
- new TableReference("statedim", "id")));
-locationHierarchy.add(new ReferencedDimAtrribute(new
FieldSchema("countryid", "int", "country"), "Country refer",
- new TableReference("countrydim", "id")));
+locationHierarchy.add(new BaseDimAttribute(new FieldSchema("zipcode",
"int", "zip")));
+locationHierarchy.add(new BaseDimAttribute(new FieldSchema("cityid",
"int", "city")));
+locationHierarchy.add(new BaseDimAttribute(new FieldSchema("stateid",
"int", "state")));
+locationHierarchy.add(new BaseDimAttribute(new FieldSchema("countryid",
"int", "country")));
List regions = Arrays.asList("APAC", "EMEA", "USA");
locationHierarchy.add(new BaseDimAttribute(new FieldSchema("regionname",
"string", "region"), "regionname", null,
null, null, null, regions));
@@ -535,66 +531,41 @@ public class CubeTestSetup {
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "string",
"basedim")));
// Added for ambiguity test
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("ambigdim1",
"string", "used in testColumnAmbiguity")));
-cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2",
"int", "ref dim"), "Dim2 refer",
- new TableReference("testdim2", "id")));
-cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("cdim2",
"int", "ref dim"), "Dim2 refer",
- new TableReference("cycledim1", "id"), NOW, null, null));
-cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("urdimid",
"int", "ref dim"), "urdim refer",
- new TableReference("unreachableDim", "id"), null, null, null, false,
10L));
-
+cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2",
"int", "ref dim"), "dim2 refer",
+ "dim2chain", "id", null, null, 0.0));
+cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cdim2", "int",
"ref dim"), "Dim2 refer", NOW, null, null));
+cubeDimensions.add(new BaseDimAttribute(new FieldSchema("urdimid", "int",
"ref dim"), "urdim refer",
+ null, null, 10.0));
+cubeDimensions.add(new ReferencedDimAttribute(new
FieldSchema("unreachableName", "string", ""), "urdim name",
+ "unreachableDim_chain", "name", null, null, 10.0));
// denormalized reference
-cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2big1",
"bigint", "ref dim"), "Dim2 refer",
- new TableReference("testdim2", "bigid1")));
-cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2big2",
"bigint", "ref dim"), "Dim2 refer",
- new TableReference("testdim2", "bigid2")));
-cubeDimensions.add(new ReferencedDimAtrribute(new
FieldSchema("dim2bignew", "bigint", "ref dim"), "Dim2 refer",
- new TableReference("testdim2", "bigidnew"), NOW, null, null));
-cubeDimensions.add(new ReferencedDimAtrribute(new
FieldSchema("test_time_dim_hour_id", "int", "ref dim"),
- "Timedim reference", new TableReference("hourdim", "id"), null, null,
null));
-cubeDimensions.add(new ReferencedDimAtrribute(new
FieldSchema("test_time_dim_day_id", "int", "ref dim"),
- "Timedim reference", new TableReference("daydim", "id"), null, null,
null));
-// not creating test_time_dim_hour_id2
lens git commit: LENS-926 : Fixes Chain ref column from bridge table to be resolved correctly
Repository: lens
Updated Branches:
refs/heads/master 5d2dccb00 -> 91ccec775
LENS-926 : Fixes Chain ref column from bridge table to be resolved correctly
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/91ccec77
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/91ccec77
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/91ccec77
Branch: refs/heads/master
Commit: 91ccec775bfff7732da41b6a6d13e01752750f7f
Parents: 5d2dccb
Author: Amareshwari Sriramadasu
Authored: Thu Jan 21 17:43:26 2016 +0530
Committer: Rajat Khandelwal
Committed: Thu Jan 21 17:43:26 2016 +0530
--
.../cube/parse/DenormalizationResolver.java | 46 +--
.../lens/cube/parse/join/AutoJoinContext.java | 2 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 20 +-
.../lens/cube/parse/TestBridgeTableQueries.java | 293 +++
.../lens/cube/parse/TestCubeRewriter.java | 3 +-
.../lens/cube/parse/TestJoinResolver.java | 225 ++
6 files changed, 354 insertions(+), 235 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/91ccec77/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index c83b9ac..f2dc2e5 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.antlr.runtime.CommonToken;
+import lombok.Getter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
@@ -67,14 +68,13 @@ public class DenormalizationResolver implements
ContextRewriter {
@ToString
public static class PickedReference {
-TableReference reference;
+@Getter
ChainRefCol chainRef;
String srcAlias;
String pickedFor;
-PickedReference(TableReference reference, String srcAlias, String
pickedFor) {
+PickedReference(String srcAlias, String pickedFor) {
this.srcAlias = srcAlias;
- this.reference = reference;
this.pickedFor = pickedFor;
}
@@ -83,38 +83,22 @@ public class DenormalizationResolver implements
ContextRewriter {
this.chainRef = chainRef;
this.pickedFor = pickedFor;
}
-
-String getDestTable() {
- if (chainRef != null) {
-return chainRef.getChainName();
- }
- return reference.getDestTable();
-}
-
-String getRefColumn() {
- if (chainRef != null) {
-return chainRef.getRefColumn();
- }
- return reference.getDestColumn();
-}
}
public static class DenormalizationContext {
// map of column name to all references
-private Map> referencedCols =
- new HashMap>();
+private Map> referencedCols = new
HashMap<>();
// candidate table name to all the references columns it needs
-private Map> tableToRefCols =
- new HashMap>();
+private Map> tableToRefCols = new
HashMap<>();
private CubeQueryContext cubeql;
// set of all picked references once all candidate tables are picked
-private Set pickedRefs = new HashSet();
+private Set pickedRefs = new HashSet<>();
// index on column name for picked references with map from column name to
// pickedrefs
-private Map> pickedReferences = new
HashMap>();
+private Map> pickedReferences = new
HashMap<>();
DenormalizationContext(CubeQueryContext cubeql) {
this.cubeql = cubeql;
@@ -123,7 +107,7 @@ public class DenormalizationResolver implements
ContextRewriter {
void addReferencedCol(String col, ReferencedQueriedColumn refer) {
Set refCols = referencedCols.get(col);
if (refCols == null) {
-refCols = new HashSet();
+refCols = new HashSet<>();
referencedCols.put(col, refCols);
}
refCols.add(refer);
@@ -144,7 +128,7 @@ public class DenormalizationResolver implements
ContextRewriter {
log.info("Adding denormalized column for column:{} for table:{}",
col, table);
Set refCols =
tableToRefCols.get(table.getName());
if (refCols == null) {
- refCols = new HashSet();
+ refCols = new HashSet<>();
tableToRefCols.put(table.getName(), refCols);
}
refCols.add(refer);
@@ -167,7 +151,7 @@ public class DenormalizationResolver implements
ContextRewriter {
private void addPickedReference(String col, PickedReference refer) {
Set refCols = pickedReferences.get(col);
if (refCols == null)
lens git commit: LENS-916: docker image creation is failing.
Repository: lens Updated Branches: refs/heads/master 91ccec775 -> 27a0cad0d LENS-916: docker image creation is failing. Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/27a0cad0 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/27a0cad0 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/27a0cad0 Branch: refs/heads/master Commit: 27a0cad0d1ece88dc7d0937b05848ad6f92208dd Parents: 91ccec7 Author: piyush Authored: Mon Jan 25 11:40:58 2016 +0530 Committer: Rajat Khandelwal Committed: Mon Jan 25 11:40:58 2016 +0530 -- lens-docker/lens-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/27a0cad0/lens-docker/lens-test/Dockerfile -- diff --git a/lens-docker/lens-test/Dockerfile b/lens-docker/lens-test/Dockerfile index 734f964..885674d 100644 --- a/lens-docker/lens-test/Dockerfile +++ b/lens-docker/lens-test/Dockerfile @@ -17,7 +17,7 @@ FROM inmobi/docker-hive -RUN wget http://apache.mirrors.lucidnetworks.net/spark/spark-1.3.0/spark-1.3.0-bin-hadoop2.4.tgz +RUN wget https://archive.apache.org/dist/spark/spark-1.3.0/spark-1.3.0-bin-hadoop2.4.tgz RUN gunzip spark-1.3.0-bin-hadoop2.4.tgz RUN tar -xvf spark-1.3.0-bin-hadoop2.4.tar RUN mv spark-1.3.0-bin-hadoop2.4 /usr/local
lens git commit: LENS-927 : Fix intermittent test failure due to alias look up in JoinTree
Repository: lens
Updated Branches:
refs/heads/master 27a0cad0d -> 7035de9d7
LENS-927 : Fix intermittent test failure due to alias look up in JoinTree
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7035de9d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7035de9d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7035de9d
Branch: refs/heads/master
Commit: 7035de9d73f2df1e72111d43875b0e21d82feaf5
Parents: 27a0cad
Author: Amareshwari Sriramadasu
Authored: Mon Jan 25 14:00:31 2016 +0530
Committer: Rajat Khandelwal
Committed: Mon Jan 25 14:00:31 2016 +0530
--
.../main/java/org/apache/lens/cube/parse/join/JoinClause.java | 7 ++-
.../main/java/org/apache/lens/cube/parse/join/JoinTree.java | 6 ++
.../apache/lens/cube/parse/TestDenormalizationResolver.java | 4 ++--
3 files changed, 6 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7035de9d/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
index acc9d5c..4325252 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
@@ -122,13 +122,10 @@ public class JoinClause implements Comparable
{
// Last element in this list is link from cube to first dimension
for (int i = entry.getValue().size() - 1; i >= 0; i--) {
// Adds a child if needed, or returns a child already existing
corresponding to the given link.
-current = current.addChild(entry.getValue().get(i), cubeql,
aliasUsage);
+current = current.addChild(entry.getValue().get(i), aliasUsage);
}
// This is a destination table. Decide alias separately. e.g. chainname
- // nullcheck is necessary because dimensions can be destinations too. In
that case getAlias() == null
- if (entry.getKey().getAlias() != null) {
-current.setAlias(entry.getKey().getAlias());
- }
+ current.setAlias(entry.getKey().getAlias());
}
if (root.getSubtrees().size() > 0) {
root.setAlias(cubeql.getAliasForTableName(
http://git-wip-us.apache.org/repos/asf/lens/blob/7035de9d/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinTree.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinTree.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinTree.java
index 197847c..bcbfed5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinTree.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinTree.java
@@ -22,7 +22,6 @@ import java.util.*;
import org.apache.lens.cube.metadata.AbstractCubeTable;
import org.apache.lens.cube.metadata.join.TableRelationship;
-import org.apache.lens.cube.parse.CubeQueryContext;
import org.apache.hadoop.hive.ql.parse.JoinType;
@@ -57,8 +56,7 @@ public class JoinTree {
this.depthFromRoot = depthFromRoot;
}
- public JoinTree addChild(TableRelationship tableRelationship,
- CubeQueryContext query, Map
aliasUsage) {
+ public JoinTree addChild(TableRelationship tableRelationship, Map aliasUsage) {
if (getSubtrees().get(tableRelationship) == null) {
JoinTree current = new JoinTree(this, tableRelationship,
this.depthFromRoot + 1);
@@ -68,7 +66,7 @@ public class JoinTree {
// And for destination tables, an alias will be decided from here but
might be
// overridden outside this function.
AbstractCubeTable destTable = tableRelationship.getToTable();
- current.setAlias(query.getAliasForTableName(destTable.getName()));
+ current.setAlias(destTable.getName());
if (aliasUsage.get(current.getAlias()) == null) {
aliasUsage.put(current.getAlias(), 0);
} else {
http://git-wip-us.apache.org/repos/asf/lens/blob/7035de9d/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index d7707a9..51ba636 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -89,8 +89,8 @@ public class TestDenormalizationResolver extends
TestQueryRewrite {
lens git commit: LENS-929: Fact skipping should be logged
Repository: lens
Updated Branches:
refs/heads/master 7035de9d7 -> b1f38d55e
LENS-929: Fact skipping should be logged
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b1f38d55
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b1f38d55
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b1f38d55
Branch: refs/heads/master
Commit: b1f38d55eddf261426aa47bfaa37c54a1eefa3d0
Parents: 7035de9
Author: Rajat Khandelwal
Authored: Mon Jan 25 14:03:55 2016 +0530
Committer: Rajat Khandelwal
Committed: Mon Jan 25 14:03:55 2016 +0530
--
.../apache/lens/cube/parse/CubeQueryContext.java| 1 +
.../server/api/driver/MinQueryCostSelector.java | 16
2 files changed, 13 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b1f38d55/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 3e930de..79dd88c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -465,6 +465,7 @@ public class CubeQueryContext implements
TrackQueriedColumns, QueryAST {
}
public void addFactPruningMsgs(CubeFactTable fact, CandidateTablePruneCause
factPruningMsg) {
+log.info("Pruning fact {} with cause: {}", fact, factPruningMsg);
factPruningMsgs.addPruningMsg(fact, factPruningMsg);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b1f38d55/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
index 8fdde1d..6f17327 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
@@ -18,13 +18,18 @@
*/
package org.apache.lens.server.api.driver;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.conf.Configuration;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class MinQueryCostSelector implements DriverSelector {
/**
@@ -38,12 +43,15 @@ public class MinQueryCostSelector implements DriverSelector
{
public LensDriver select(final AbstractQueryContext ctx, final Configuration
conf) {
final Collection drivers =
ctx.getDriverContext().getDriversWithValidQueryCost();
-
+log.info("Candidate drivers: {}", drivers);
+for (LensDriver driver : drivers) {
+ log.debug("Cost on driver {}: {}", driver,
ctx.getDriverQueryCost(driver));
+}
return Collections.min(drivers, new Comparator() {
@Override
public int compare(LensDriver d1, LensDriver d2) {
-final QueryCost c1 = ctx.getDriverContext().getDriverQueryCost(d1);
-final QueryCost c2 = ctx.getDriverContext().getDriverQueryCost(d2);
+final QueryCost c1 = ctx.getDriverQueryCost(d1);
+final QueryCost c2 = ctx.getDriverQueryCost(d2);
return c1.compareTo(c2);
}
});
lens git commit: LENS-813: For multifact queries, having clauses are getting added to both sub queries.
Repository: lens
Updated Branches:
refs/heads/master 2539f338a -> 2f0e5fdbf
LENS-813: For multifact queries, having clauses are getting added to both sub
queries.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2f0e5fdb
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2f0e5fdb
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2f0e5fdb
Branch: refs/heads/master
Commit: 2f0e5fdbf079c301bcd41724e61b90b9af4fad33
Parents: 2539f33
Author: Rajat Khandelwal
Authored: Mon Feb 1 21:23:01 2016 +0530
Committer: Rajat Khandelwal
Committed: Mon Feb 1 21:23:01 2016 +0530
--
.../lens/cube/error/LensCubeErrorCode.java | 1 -
.../apache/lens/cube/parse/AliasDecider.java| 26 ++
.../apache/lens/cube/parse/CandidateFact.java | 72 --
.../lens/cube/parse/CubeQueryContext.java | 36 ++-
.../lens/cube/parse/DefaultAliasDecider.java| 31 +++
.../lens/cube/parse/ExpressionResolver.java | 4 +-
.../org/apache/lens/cube/parse/HQLParser.java | 41 +++
.../lens/cube/parse/MultiFactHQLContext.java| 85 ++-
.../parse/SingleFactMultiStorageHQLContext.java | 50 +---
.../lens/cube/parse/TestBaseCubeQueries.java| 249 +--
10 files changed, 491 insertions(+), 104 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2f0e5fdb/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
index 61d08b2..e49b58b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
@@ -57,7 +57,6 @@ public enum LensCubeErrorCode {
STORAGE_UNION_DISABLED(3031, 1500),
COULD_NOT_PARSE_EXPRESSION(3032, 1500),
QUERIED_TABLE_NOT_FOUND(3033, 0),
-
// Error codes greater than 3100 are errors while doing a metastore
operation.
ERROR_IN_ENTITY_DEFINITION(3101, 100),
TIMELINE_ABSENT(3102, 100),
http://git-wip-us.apache.org/repos/asf/lens/blob/2f0e5fdb/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasDecider.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasDecider.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasDecider.java
new file mode 100644
index 000..e9ddb9f
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasDecider.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+
+
+public interface AliasDecider {
+ String decideAlias(ASTNode node);
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/2f0e5fdb/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index c305244..4faebe1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -29,6 +29,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -54,19 +55,26 @@ public class CandidateFact implements CandidateTable,
QueryAST {
private final Set partsQueried = Sets.newHashSet();
private CubeInterface baseTable;
- @Getter @Setter
+ @Getter
+ @Setter
private ASTN
lens git commit: LENS-950: Bridge table join should ignore join type config
Repository: lens
Updated Branches:
refs/heads/master 0a60ef331 -> 8a90d8b93
LENS-950: Bridge table join should ignore join type config
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8a90d8b9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8a90d8b9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8a90d8b9
Branch: refs/heads/master
Commit: 8a90d8b9383994e61015f1eae332e8191e0a19d6
Parents: 0a60ef3
Author: Amareshwari Sriramadasu
Authored: Mon Feb 15 18:51:11 2016 +0530
Committer: Rajat Khandelwal
Committed: Mon Feb 15 18:51:11 2016 +0530
--
.../lens/cube/parse/join/AutoJoinContext.java | 2 +-
.../lens/cube/parse/TestBridgeTableQueries.java | 31
2 files changed, 32 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/8a90d8b9/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
index f70cbe4..e14a898 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
@@ -286,7 +286,7 @@ public class AutoJoinContext {
} else {
// if bridge clauses are already inited, this is a next table
getting joined with bridge table
// we will append a simple join clause
- bridgeFromClause.append(joinTypeStr).append(" join ");
+ bridgeFromClause.append(" join ");
bridgeFromClause.append(dimsToQuery.get(rel.getToTable()).getStorageString(toAlias));
bridgeFromClause.append(" on ").append(fromAlias).append(".")
.append(rel.getFromColumn()).append(" = ").append(toAlias)
http://git-wip-us.apache.org/repos/asf/lens/blob/8a90d8b9/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
index e66c39b..e8ad8ff 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
@@ -240,6 +240,37 @@ public class TestBridgeTableQueries extends
TestQueryRewrite {
}
@Test
+ public void testBridgeTablesWithMultipleChainsWithJoinType() throws
Exception {
+Configuration conf = new Configuration(hConf);
+conf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "LEFTOUTER");
+String query = "select usersports.name, xusersports.name,
yusersports.name, sum(msr2) from basecube where "
+ + TWO_DAYS_RANGE;
+String hqlQuery = rewrite(query, conf);
+String expected = getExpectedQuery("basecube", "select usersports.name,
xusersports.name, yusersports.name,"
+ + " sum(basecube.msr2) FROM ",
+ " left outer join " + getDbName() + "c1_usertable userdim_1 on
basecube.userid = userdim_1.id "
+ + " left outer join (select user_interests_1.user_id as user_id,
collect_set(usersports.name) as name from "
+ + getDbName() + "c1_user_interests_tbl user_interests_1 join " +
getDbName() + "c1_sports_tbl usersports on "
+ + "user_interests_1.sport_id = usersports.id group by
user_interests_1.user_id) "
+ + "usersports on userdim_1.id = usersports.user_id"
+ + " left outer join " + getDbName() + "c1_usertable userdim_0 on
basecube.yuserid = userdim_0.id "
+ + " left outer join (select user_interests_0.user_id as
user_id,collect_set(yusersports.name) as name from "
+ + getDbName() + "c1_user_interests_tbl user_interests_0 join " +
getDbName() + "c1_sports_tbl yusersports on "
+ + " user_interests_0.sport_id = yusersports.id group by
user_interests_0.user_id) yusersports on userdim_0.id ="
+ + " yusersports.user_id left outer join " + getDbName()
+ + "c1_usertable userdim on basecube.xuserid = userdim.id"
+ + " left outer join (select user_interests.user_id as
user_id,collect_set(xusersports.name) as name from "
+ + getDbName() + "c1_user_interests_tbl user_interests join " +
getDbName() + "c1_sports_tbl xusersports"
+ + " on user_interests.sport_id = xusersports.id group by
user_interests.user_id) xusersports on userdim.id = "
+ + " xusersports.user_id", null, "group by usersports.name,
xusersports.name, yusersports.name", null,
+ getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
+TestCubeRewriter.compareQueries(hqlQuery, e
lens git commit: LENS-903: No candidate dim available exception should contain only brief error
Repository: lens
Updated Branches:
refs/heads/master 8a90d8b93 -> 223da282d
LENS-903: No candidate dim available exception should contain only brief error
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/223da282
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/223da282
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/223da282
Branch: refs/heads/master
Commit: 223da282dbce288c1e825c2f07704d44ea7a8ec3
Parents: 8a90d8b
Author: Sushil Mohanty
Authored: Tue Feb 16 13:25:36 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue Feb 16 13:25:36 2016 +0530
--
.../java/org/apache/lens/cube/parse/CandidateTableResolver.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/223da282/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 00ccf36..31d2d2c 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -122,8 +122,8 @@ class CandidateTableResolver implements ContextRewriter {
List dimtables =
cubeql.getMetastoreClient().getAllDimensionTables(dim);
if (dimtables.isEmpty()) {
if (!optional) {
- throw new
LensException(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(),
dim.getName(),
-"Dimension tables do not exist");
+ throw new
LensException(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(),
+ dim.getName().concat(" has no dimension tables"));
} else {
log.info("Not considering optional dimension {} as, No dimension
tables exist", dim);
removeOptionalDimWithoutAlias(cubeql, dim);
lens git commit: LENS-960: getting string from AST should give flat string wherever possible
Repository: lens
Updated Branches:
refs/heads/master c0199ead2 -> 6be4d48fd
LENS-960: getting string from AST should give flat string wherever possible
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6be4d48f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6be4d48f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6be4d48f
Branch: refs/heads/master
Commit: 6be4d48fd4620e13435a266380f84119f87a9233
Parents: c0199ea
Author: Rajat Khandelwal
Authored: Fri Feb 19 13:36:40 2016 +0530
Committer: Rajat Khandelwal
Committed: Fri Feb 19 13:36:40 2016 +0530
--
.../org/apache/lens/cube/parse/HQLParser.java | 119 +--
.../lens/cube/parse/TestBaseCubeQueries.java| 12 +-
.../lens/cube/parse/TestCubeRewriter.java | 8 +-
.../lens/cube/parse/TestExpressionContext.java | 87 +++---
.../apache/lens/cube/parse/TestHQLParser.java | 92 --
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 21 ++--
.../driver/jdbc/TestColumnarSQLRewriter.java| 10 +-
7 files changed, 179 insertions(+), 170 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/6be4d48f/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index fdef3f1..8d6105f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -83,13 +83,14 @@ public final class HQLParser {
}
public static final Set BINARY_OPERATORS;
+ public static final Set N_ARY_OPERATORS;
public static final Set FILTER_OPERATORS;
public static final Set ARITHMETIC_OPERATORS;
public static final Set UNARY_OPERATORS;
public static final Set PRIMITIVE_TYPES;
static {
-HashSet ops = new HashSet();
+HashSet ops = new HashSet<>();
ops.add(DOT);
ops.add(KW_AND);
ops.add(KW_OR);
@@ -113,20 +114,22 @@ public final class HQLParser {
ops.add(BITWISEXOR);
BINARY_OPERATORS = Collections.unmodifiableSet(ops);
+N_ARY_OPERATORS = Collections.unmodifiableSet(Sets.newHashSet(KW_AND,
KW_OR, PLUS, STAR,
+ AMPERSAND, BITWISEOR, BITWISEXOR));
-ARITHMETIC_OPERATORS = new HashSet();
+ARITHMETIC_OPERATORS = new HashSet<>();
ARITHMETIC_OPERATORS.add(PLUS);
ARITHMETIC_OPERATORS.add(MINUS);
ARITHMETIC_OPERATORS.add(STAR);
ARITHMETIC_OPERATORS.add(DIVIDE);
ARITHMETIC_OPERATORS.add(MOD);
-HashSet unaryOps = new HashSet();
+HashSet unaryOps = new HashSet<>();
unaryOps.add(KW_NOT);
unaryOps.add(TILDE);
UNARY_OPERATORS = Collections.unmodifiableSet(unaryOps);
-HashSet primitiveTypes = new HashSet();
+HashSet primitiveTypes = new HashSet<>();
primitiveTypes.add(TOK_TINYINT);
primitiveTypes.add(TOK_SMALLINT);
primitiveTypes.add(TOK_INT);
@@ -148,10 +151,6 @@ public final class HQLParser {
EQUAL_NS);
}
- public static boolean isArithmeticOp(int tokenType) {
-return ARITHMETIC_OPERATORS.contains(tokenType);
- }
-
public static ASTNode parseHQL(String query, HiveConf conf) throws
LensException {
ParseDriver driver = new ParseDriver();
ASTNode tree = null;
@@ -199,8 +198,10 @@ public final class HQLParser {
/**
* Debug function for printing query AST to stdout
*
- * @param node
- * @param level
+ * @param tokenMapping token mapping
+ * @param node node
+ * @param levellevel
+ * @param childchild
*/
public static void printAST(Map tokenMapping, ASTNode node,
int level, int child) {
if (node == null || node.isNil()) {
@@ -211,7 +212,7 @@ public final class HQLParser {
System.out.print(" ");
}
-System.out.print(node.getText() + " [" +
tokenMapping.get(node.getToken().getType()) + "]");
+System.out.print(node.getText() + " [" + tokenMapping.get(node.getType())
+ "]");
System.out.print(" (l" + level + "c" + child + "p" +
node.getCharPositionInLine() + ")");
if (node.getChildCount() > 0) {
@@ -239,7 +240,7 @@ public final class HQLParser {
}
public static Map getHiveTokenMapping() throws Exception {
-Map mapping = new HashMap();
+Map mapping = new HashMap<>();
for (Field f : HiveParser.class.getFields()) {
if (f.getType() == int.class) {
@@ -256,9 +257,9 @@ public final class HQLParser {
* Find a node in the tree rooted at root, given the path of type of tokens
from the root's children to the desired
* node
*
- * @param root
+ * @param root node from which searching is to be started
* @param path
lens git commit: LENS-977: 'AND' in having clause is not resolved properly with multi fact query
Repository: lens
Updated Branches:
refs/heads/master 2da114407 -> ee2bda93f
LENS-977: 'AND' in having clause is not resolved properly with multi fact query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ee2bda93
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ee2bda93
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ee2bda93
Branch: refs/heads/master
Commit: ee2bda93f1587148a8a5e86ba88d04d1f3549d4f
Parents: 2da1144
Author: Amareshwari Sriramadasu
Authored: Sun Feb 28 20:01:31 2016 +0530
Committer: Rajat Khandelwal
Committed: Sun Feb 28 20:01:31 2016 +0530
--
.../java/org/apache/lens/cube/parse/CandidateFact.java | 4 ++--
.../org/apache/lens/cube/parse/TestBaseCubeQueries.java | 11 +++
2 files changed, 13 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/ee2bda93/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 1b0de34..82ca4f4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -117,9 +117,9 @@ public class CandidateFact implements CandidateTable,
QueryAST {
return;
}
ASTNode existingHavingAST = (ASTNode) getHavingAST().getChild(0);
-ASTNode newHavingAST = new ASTNode(new CommonToken(KW_AND, "KW_AND"));
-newHavingAST.addChild(ast);
+ASTNode newHavingAST = new ASTNode(new CommonToken(KW_AND, "AND"));
newHavingAST.addChild(existingHavingAST);
+newHavingAST.addChild(ast);
getHavingAST().setChild(0, newHavingAST);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/ee2bda93/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 0415434..fec5d51 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -763,6 +763,17 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.contains(joinSubString) &&
hqlQuery.endsWith(endSubString), hqlQuery);
hqlQuery = rewrite("select dim1, dim11 from basecube where " +
TWO_DAYS_RANGE
+ + "having msr12 > 2 and roundedmsr2 > 0 and msr2 > 100", conf);
+expected2 = getExpectedQuery(cubeName,
+ "select basecube.dim1 as dim1, basecube.dim11 as dim11 FROM ", null,
+ " group by basecube.dim1, basecube.dim11 HAVING
round(sum(basecube.msr2)/1000) > 0 and sum(basecube.msr2) > 100",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
+compareContains(expected1, hqlQuery);
+compareContains(expected2, hqlQuery);
+assertTrue(hqlQuery.toLowerCase().startsWith(begin), hqlQuery);
+assertTrue(hqlQuery.contains(joinSubString) &&
hqlQuery.endsWith(endSubString), hqlQuery);
+
+hqlQuery = rewrite("select dim1, dim11 from basecube where " +
TWO_DAYS_RANGE
+ "having flooredmsr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"select basecube.dim1 as dim1, basecube.dim11 as dim11, "
lens git commit: LENS-980: Creating examples is failing on master
Repository: lens
Updated Branches:
refs/heads/master a57426e4b -> 4f00c11b4
LENS-980: Creating examples is failing on master
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4f00c11b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4f00c11b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4f00c11b
Branch: refs/heads/master
Commit: 4f00c11b49a46a97420a4d1617653c14e0002535
Parents: a57426e
Author: Amareshwari Sriramadasu
Authored: Wed Mar 2 15:13:04 2016 +0530
Committer: Rajat Khandelwal
Committed: Wed Mar 2 15:13:04 2016 +0530
--
.../src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/4f00c11b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
index e517c92..14fc4aa 100644
--- a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
+++ b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
@@ -120,7 +120,7 @@ public class LensJAXBContext extends JAXBContext {
return ((JAXBElement) UNMARSHALLER.unmarshal(file)).getValue();
} else {
// load from classpath
- InputStream stream = LensJAXBContext.class.getResourceAsStream(filename);
+ InputStream stream =
Thread.currentThread().getContextClassLoader().getResourceAsStream(filename);
if (stream == null) {
throw new IOException("File not found:" + filename);
}
lens git commit: LENS-953: Unclosed InputStream in DatabaseUtil#initializeDatabaseStorage()
Repository: lens
Updated Branches:
refs/heads/master 8854cf512 -> e0c495e28
LENS-953: Unclosed InputStream in DatabaseUtil#initializeDatabaseStorage()
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e0c495e2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e0c495e2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e0c495e2
Branch: refs/heads/master
Commit: e0c495e289a823517ad307ba8504766b7f709a48
Parents: 8854cf5
Author: Ajay Yadava
Authored: Thu Mar 17 12:57:27 2016 +0530
Committer: Rajat Khandelwal
Committed: Thu Mar 17 12:57:27 2016 +0530
--
.../org/apache/lens/examples/DatabaseUtil.java | 29
1 file changed, 17 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/e0c495e2/lens-examples/src/main/java/org/apache/lens/examples/DatabaseUtil.java
--
diff --git
a/lens-examples/src/main/java/org/apache/lens/examples/DatabaseUtil.java
b/lens-examples/src/main/java/org/apache/lens/examples/DatabaseUtil.java
index f81504e..40632f8 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/DatabaseUtil.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/DatabaseUtil.java
@@ -40,7 +40,6 @@ public final class DatabaseUtil {
* @throws Exception the exception
*/
public static void initializeDatabaseStorage() throws Exception {
-
try {
Class.forName("org.hsqldb.jdbcDriver");
} catch (ClassNotFoundException e) {
@@ -50,19 +49,25 @@ public final class DatabaseUtil {
con.setAutoCommit(true);
Statement statement = con.createStatement();
-
-InputStream file =
DatabaseUtil.class.getClassLoader().getResourceAsStream("db-storage-schema.sql");
-BufferedReader reader = new BufferedReader(new InputStreamReader(file,
"UTF-8"));
-String line;
-while ((line = reader.readLine()) != null) {
- if (line.trim().equals("") || line.startsWith("--")) {
-continue;
+BufferedReader reader = null;
+try {
+ InputStream file =
DatabaseUtil.class.getClassLoader().getResourceAsStream("db-storage-schema.sql");
+ reader = new BufferedReader(new InputStreamReader(file, "UTF-8"));
+ String line;
+ while ((line = reader.readLine()) != null) {
+if (line.trim().equals("") || line.startsWith("--")) {
+ continue;
+}
+statement.executeUpdate(line);
+ }
+} finally {
+ if (reader != null) {
+reader.close();
}
- statement.executeUpdate(line);
+ statement.execute("SHUTDOWN");
+ statement.close();
+ con.close();
}
-statement.execute("SHUTDOWN");
-statement.close();
-con.close();
}
/**
lens git commit: LENS-893 : Add exponential backoff for next update of status update, incase of failures from driver
Repository: lens
Updated Branches:
refs/heads/master dba885ca9 -> 5faaf11b4
LENS-893 : Add exponential backoff for next update of status update, incase of
failures from driver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5faaf11b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5faaf11b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5faaf11b
Branch: refs/heads/master
Commit: 5faaf11b4d4a75972aa113101c6f2b63ac32763d
Parents: dba885c
Author: Amareshwari Sriramadasu
Authored: Wed Mar 30 17:18:13 2016 +0530
Committer: Rajat Khandelwal
Committed: Wed Mar 30 17:18:13 2016 +0530
--
.../lens/server/api/LensConfConstants.java | 31 +++
.../server/api/common/BackOffRetryHandler.java | 71 ++
.../lens/server/api/common/FailureContext.java | 43 +
...FibonacciExponentialBackOffRetryHandler.java | 77 +++
.../common/OperationRetryHandlerFactory.java| 41
.../lens/server/api/query/QueryContext.java | 39
.../apache/lens/server/api/util/LensUtil.java | 20 +++-
.../TestExponentialBackOffRetryHandler.java | 52 +++
.../lens/server/api/driver/MockDriver.java | 24 +
.../api/query/TestAbstractQueryContext.java | 1 +
.../lens/server/api/query/TestQueryContext.java | 98
.../server/query/QueryExecutionServiceImpl.java | 20 +++-
.../src/main/resources/lensserver-default.xml | 20
lens-server/src/test/resources/lens-site.xml| 6 ++
src/site/apt/admin/config.apt | 56 ++-
15 files changed, 570 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/5faaf11b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index f8a936a..724c868 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -1046,4 +1046,35 @@ public final class LensConfConstants {
* Default value of INMEMORY_RESULT_SET_TTL_SECS is 300 secs (5 minutes)
*/
public static final int DEFAULT_INMEMORY_RESULT_SET_TTL_SECS = 300;
+
+ /**
+ * Number of retries status update will be retried, in case of transient
failures
+ */
+ public static final String STATUS_UPDATE_EXPONENTIAL_RETRIES = SERVER_PFX +
"status.update.num.retries";
+
+ /**
+ * Default value of STATUS_UPDATE_EXPONENTIAL_RETRIES is 10
+ */
+ public static final int DEFAULT_STATUS_UPDATE_EXPONENTIAL_RETRIES = 10;
+
+ /**
+ * Maximum delay a status update can wait for next update, in case of
transient failures
+ */
+ public static final String MAXIMUM_STATUS_UPDATE_DELAY = SERVER_PFX +
"status.update.maximum.delay.secs";
+
+ /**
+ * Default value of MAXIMUM_STATUS_UPDATE_DELAY is 1800 secs (30 minutes)
+ */
+ public static final long DEFAULT_MAXIMUM_STATUS_UPDATE_DELAY = 1800;
+
+ /**
+ * Number of seconds that would grow exponentially for next update, incase
of transient failures.
+ */
+ public static final String STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR = SERVER_PFX
++ "status.update.exponential.wait.millis";
+
+ /**
+ * Default value of DEFAULT_STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR is 3
millis (30 seconds)
+ */
+ public static final long DEFAULT_STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR =
3;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5faaf11b/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
new file mode 100644
index 000..17bfba0
--- /dev/null
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the
[1/2] lens git commit: LENS-1003: Add list query tests to regression
Repository: lens
Updated Branches:
refs/heads/master 33dfee1d6 -> 44bb7edb6
http://git-wip-us.apache.org/repos/asf/lens/blob/44bb7edb/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java
--
diff --git
a/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java
b/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java
new file mode 100644
index 000..821c388
--- /dev/null
+++
b/lens-regression/src/test/java/org/apache/lens/regression/client/ITListQueryTest.java
@@ -0,0 +1,515 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.client;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+import javax.ws.rs.client.WebTarget;
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.query.*;
+import org.apache.lens.regression.core.constants.QueryInventory;
+import org.apache.lens.regression.core.helpers.*;
+import org.apache.lens.regression.core.testHelper.BaseTestClass;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+import org.testng.Assert;
+import org.testng.annotations.*;
+
+
+public class ITListQueryTest extends BaseTestClass {
+
+ WebTarget servLens;
+ private String sessionHandleString;
+
+ LensServerHelper lens = getLensServerHelper();
+ MetastoreHelper mHelper = getMetastoreHelper();
+ SessionHelper sHelper = getSessionHelper();
+ QueryHelper qHelper = getQueryHelper();
+
+ private final String hdfsJarPath = lens.getServerHdfsUrl() + "/tmp";
+ private final String localJarPath = new File("").getAbsolutePath() +
"/lens-regression/target/testjars/";
+ private final String hiveUdfJar = "hiveudftest.jar";
+ private final String serverResourcePath = "/tmp/regression/resources";
+ String jdbcDriver = "jdbc/jdbc1", hiveDriver = "hive/hive1";
+
+
+ private static Logger logger = Logger.getLogger(ITListQueryTest.class);
+
+ @BeforeClass(alwaysRun = true)
+ public void initialize() throws IOException, JAXBException, LensException {
+servLens = ServiceManagerHelper.init();
+//HadoopUtil.uploadJars(localJarPath + "/" + hiveUdfJar, hdfsJarPath);
+logger.info("Creating a new Session");
+sessionHandleString = lens.openSession(lens.getCurrentDB());
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setUp(Method method) throws Exception {
+logger.info("Test Name: " + method.getName());
+ }
+
+ @AfterClass(alwaysRun = true)
+ public void closeSession() throws Exception {
+logger.info("Closing Session");
+lens.closeSession();
+ }
+
+ @DataProvider(name = "DP")
+ public Object[][] getData() throws Exception {
+
+String[] queries = new String[]{QueryInventory.HIVE_CUBE_QUERY,
QueryInventory.HIVE_DIM_QUERY,
+ QueryInventory.JDBC_DIM_QUERY, };
+Object[][] testData = new Object[3][1];
+
+int i = 0;
+for (String query : queries) {
+ testData[i][0] = query;
+ i++;
+}
+return testData;
+ }
+
+
+ @Test(enabled = true, dataProvider = "DP")
+ public void getQueryByQueryNameAllQuery(String queryString) throws Exception
{
+
+String queryName = "queryNameFirst";
+String queryName1 = "queryNameSecond";
+String queryName2 = "Name";
+
+//Running Query with queryName
+QueryHandle queryHandle = (QueryHandle) qHelper.executeQuery(queryString,
queryName).getData();
+LensQuery lensQuery = qHelper.waitForCompletion(queryHandle);
+Assert.assertEquals(lensQuery.getStatus().getStatus(),
QueryStatus.Status.SUCCESSFUL, "Query did not succeed");
+
+//Running Query with queryName1
+QueryHandle queryHandle1 = (QueryHandle) qHelper.executeQuery(queryString,
queryName1).getData();
+lensQuery = qHelper.waitForCompletion(queryHandle1);
+Assert.assertEquals(lensQuery.getStatus().getStatus(),
QueryStatus.Status.SUCCESSFUL, "Query did not succeed");
+
+//Running Query with queryName
+QueryHandle queryHandle2 = (QueryHandle) qHelper.executeQuery(queryString,
queryName).getData();
+lensQuery = qHelper.waitForCompletion
[2/2] lens git commit: LENS-1003: Add list query tests to regression
LENS-1003: Add list query tests to regression
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/44bb7edb
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/44bb7edb
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/44bb7edb
Branch: refs/heads/master
Commit: 44bb7edb63e24c972621e7d700f82a25a55ad4f0
Parents: 33dfee1
Author: Archana H
Authored: Wed Apr 13 19:09:32 2016 +0530
Committer: Rajat Khandelwal
Committed: Wed Apr 13 19:09:32 2016 +0530
--
.../src/additional/java/SampleUdf.java | 22 +-
.../core/constants/QueryInventory.java | 30 +-
.../core/helpers/LensServerHelper.java | 8 +-
.../regression/core/helpers/QueryHelper.java| 254 +
.../core/helpers/ServiceManagerHelper.java | 48 +-
.../regression/core/helpers/SessionHelper.java | 108 ++--
.../org/apache/lens/regression/util/Util.java | 34 +-
.../lens/regression/client/ITListQueryTest.java | 515 +++
.../regression/client/SessionResourceTests.java | 33 +-
9 files changed, 836 insertions(+), 216 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/44bb7edb/lens-regression/src/additional/java/SampleUdf.java
--
diff --git a/lens-regression/src/additional/java/SampleUdf.java
b/lens-regression/src/additional/java/SampleUdf.java
index e340ffd..0c4e089 100644
--- a/lens-regression/src/additional/java/SampleUdf.java
+++ b/lens-regression/src/additional/java/SampleUdf.java
@@ -21,16 +21,20 @@ import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.Text;
public final class SampleUdf extends UDF {
-public Text evaluate(final Text s, Text sleepTime) throws
InterruptedException {
+ public Text evaluate(final Text s, Text sleepTime) throws
InterruptedException {
-if(sleepTime!=null){
-Thread.sleep(Long.parseLong(sleepTime.toString()));
-}else{
-Thread.sleep(18);
-}
+Long time = 180*1000L;
-if (s == null) { return null; }
-
-return new Text(s.toString().toLowerCase());
+if(sleepTime != null){
+ time = Long.parseLong(sleepTime.toString()) * 1000L;
}
+
+System.out.println("Sleep Time : " + time);
+
+Thread.sleep(time);
+
+if (s == null) { return null; }
+
+return new Text(s.toString().toLowerCase());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/44bb7edb/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
--
diff --git
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
index e41ca86..eb889c4 100644
---
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
+++
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
@@ -19,8 +19,25 @@
package org.apache.lens.regression.core.constants;
+import java.util.Properties;
+
+import org.apache.lens.regression.util.Util;
+
+
public class QueryInventory {
+ static final String QUERIES_PROPERTIES = "queries.properties";
+ static Properties queryMap = null;
+
+ static {
+queryMap = Util.getPropertiesObj(QUERIES_PROPERTIES);
+ }
+
+ public static String getQueryFromInventory(String queryName){
+return queryMap.get(queryName).toString();
+ }
+
+
private QueryInventory() {
}
@@ -29,12 +46,12 @@ public class QueryInventory {
public static final String WRONG_QUERY = "cube select NO_ID from sample_dim
where name != 'first'";
public static final String DIM_QUERY = "cube select id,name from sample_dim
where name != 'first'";
- public static final String CUBE_QUERY = "cube select sample_dim.name,
measure4 from sample_cube where "
+ public static final String CUBE_QUERY = "cube select sample_dim_chain.name,
measure4 from sample_cube where "
+ "time_range_in(dt, '2014-06-24-23', '2014-06-25-00')";
public static final String WRONG_DIM_QUERY = "cube select NO_ID from
sample_dim where name != 'first'";
public static final String HIVE_DIM_QUERY = "cube select id,name from
sample_dim2 where name != 'first'";
- public static final String HIVE_CUBE_QUERY = "cube select sample_dim.name,
measure4 from sample_cube where "
+ public static final String HIVE_CUBE_QUERY = "cube select
sample_dim_chain.name, measure4 from sample_cube where "
+ "time_range_in(dt, '2014-06-24-23', '2014-06-25-00')";
public static final String JDBC_CUBE_QUERY = "cube select product_id from
sales where "
@@ -43,7 +60,7 @@ public class QueryInventory
lens git commit: LENS-997: Add kill query tests to regression
Repository: lens
Updated Branches:
refs/heads/master 44bb7edb6 -> e269651cf
LENS-997: Add kill query tests to regression
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e269651c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e269651c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e269651c
Branch: refs/heads/master
Commit: e269651cfb0e298d03d5830a922e1221867b0297
Parents: 44bb7ed
Author: Archana H
Authored: Wed Apr 13 19:11:20 2016 +0530
Committer: Rajat Khandelwal
Committed: Wed Apr 13 19:11:20 2016 +0530
--
.../lens/regression/client/KillQueryTests.java | 373 +++
1 file changed, 373 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/e269651c/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java
--
diff --git
a/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java
b/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java
new file mode 100644
index 000..e2148e3
--- /dev/null
+++
b/lens-regression/src/test/java/org/apache/lens/regression/client/KillQueryTests.java
@@ -0,0 +1,373 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.client;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+
+import javax.ws.rs.client.WebTarget;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.query.*;
+import org.apache.lens.regression.core.constants.QueryInventory;
+import org.apache.lens.regression.core.helpers.*;
+import org.apache.lens.regression.core.testHelper.BaseTestClass;
+import org.apache.lens.regression.util.HadoopUtil;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+import org.testng.Assert;
+import org.testng.annotations.*;
+
+
+public class KillQueryTests extends BaseTestClass {
+
+ WebTarget servLens;
+ private String sessionHandleString;
+
+ LensServerHelper lens = getLensServerHelper();
+ MetastoreHelper mHelper = getMetastoreHelper();
+ SessionHelper sHelper = getSessionHelper();
+ QueryHelper qHelper = getQueryHelper();
+
+ private final String hdfsJarPath = lens.getServerHdfsUrl() + "/tmp";
+ private final String localJarPath = new File("").getAbsolutePath() +
"/lens-regression/target/testjars/";
+ private final String hiveUdfJar = "hiveudftest.jar";
+ private final String serverResourcePath = "/tmp/regression/resources";
+
+ private static Logger logger = Logger.getLogger(KillQueryTests.class);
+
+ @BeforeClass(alwaysRun = true)
+ public void initialize() throws IOException, JAXBException, LensException,
IllegalAccessException,
+ InstantiationException {
+servLens = ServiceManagerHelper.init();
+logger.info("Creating a new Session");
+sessionHandleString = lens.openSession(lens.getCurrentDB());
+
+HadoopUtil.uploadJars(localJarPath + "/" + hiveUdfJar, hdfsJarPath);
+logger.info("Adding jar for making query to run for longer period of
time");
+sHelper.addResourcesJar(hdfsJarPath + "/" + hiveUdfJar);
+QueryHandle queryHandle = (QueryHandle)
qHelper.executeQuery(QueryInventory.SLEEP_FUNCTION).getData();
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setUp(Method method) throws Exception {
+logger.info("Test Name: " + method.getName());
+ }
+
+
+ @AfterClass(alwaysRun = true)
+ public void closeSession() throws Exception {
+logger.info("Closing Session");
+lens.closeSession();
+ }
+
+
+ @Test(enabled = true)
+ public void killQueryByHandle() throws Exception {
+
+QueryHandle qH = (QueryHandle)
qHelper.executeQuery(QueryInventory.SLEEP_QUERY).getData();
+logger.info("QUERY HANDLE : " + qH);
+
+QueryStatus queryStatus = qHelper.waitForQueryToRun(qH);
+Assert.assertEquals(queryStatus.getStatus(), QueryStatus.Status.RUNNING,
"Query is Not Running");
+
+qHelper.killQueryByQueryHandle(qH);
+
lens git commit: LENS-1004: Add prepared and query API tests
Repository: lens
Updated Branches:
refs/heads/master 90cbd94f4 -> 393153fab
LENS-1004: Add prepared and query API tests
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/393153fa
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/393153fa
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/393153fa
Branch: refs/heads/master
Commit: 393153fab1dbd1f186225330b281685291fa9c42
Parents: 90cbd94
Author: Archana H
Authored: Mon Apr 25 12:33:17 2016 +0530
Committer: Rajat Khandelwal
Committed: Mon Apr 25 12:33:17 2016 +0530
--
lens-regression/pom.xml | 5 +
.../regression/core/constants/DriverConfig.java | 38 ++
.../core/constants/QueryInventory.java | 2 +-
.../regression/core/helpers/QueryHelper.java| 6 +-
.../regression/core/helpers/SessionHelper.java | 16 +-
.../lens/regression/core/type/MapBuilder.java | 7 +
.../src/main/resources/lens.properties | 9 -
.../src/main/resources/template.lens.properties | 9 +
.../lens/regression/client/ITListQueryTest.java | 19 +-
.../regression/client/ITPreparedQueryTests.java | 522 +++
.../lens/regression/client/ITQueryApiTests.java | 335
.../regression/client/SessionResourceTests.java | 61 ++-
pom.xml | 7 +
13 files changed, 976 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/393153fa/lens-regression/pom.xml
--
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 1a656e3..08a7ece 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -78,6 +78,11 @@
hadoop-client
+
+ com.google.protobuf
+ protobuf-java
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/393153fa/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
--
diff --git
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
new file mode 100644
index 000..5a2df11
--- /dev/null
+++
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.constants;
+
+import
org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
+
+public class DriverConfig {
+
+ private DriverConfig() {
+
+ }
+
+ public static final String MAX_CONCURRENT_QUERIES =
MaxConcurrentDriverQueriesConstraintFactory.
+ MAX_CONCURRENT_QUERIES_KEY;
+ public static final String PRIORITY_MAX_CONCURRENT =
MaxConcurrentDriverQueriesConstraintFactory.
+ MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY;
+ public static final String QUEUE_MAX_CONCURRENT =
MaxConcurrentDriverQueriesConstraintFactory.
+ MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY;
+
+}
+
http://git-wip-us.apache.org/repos/asf/lens/blob/393153fa/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
--
diff --git
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
index eb889c4..5f1f607 100644
---
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
+++
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
@@ -76,6 +76,6 @@ public class QueryInventory {
public static final String NO_CUBE_KEYWORD_QUERY = "select
sample_dim_chain.name, measure4 from sample_cube where "
+ "time_range_in(dt, '2014-06-24-23', '2014-06-25-00')";
- public static final String
lens git commit: LENS-1027: Extend MaxConcurrentDriverQueriesConstraint to support limit on ANY queue
Repository: lens
Updated Branches:
refs/heads/master dce5a812d -> 34223e880
LENS-1027: Extend MaxConcurrentDriverQueriesConstraint to support limit on ANY
queue
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/34223e88
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/34223e88
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/34223e88
Branch: refs/heads/master
Commit: 34223e880df7f08912a519349ea6053932f014ea
Parents: dce5a81
Author: Puneet Gupta
Authored: Tue May 3 12:15:41 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue May 3 12:15:41 2016 +0530
--
.../MaxConcurrentDriverQueriesConstraint.java | 7 ++-
.../MaxConcurrentDriverQueriesConstraintFactory.java | 4 +++-
.../MaxConcurrentDriverQueriesConstraintTest.java | 13 -
.../ThreadSafeEstimatedQueryCollectionTest.java| 3 ++-
4 files changed, 19 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
index 58ebd9a..b2319a9 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
@@ -36,6 +36,7 @@ public class MaxConcurrentDriverQueriesConstraint implements
QueryLaunchingConst
private final int maxConcurrentQueries;
private final Map maxConcurrentQueriesPerQueue;
private final Map maxConcurrentQueriesPerPriority;
+ private final Integer defaultMaxConcurrentQueriesPerQueueLimit;
@Override
public boolean allowsLaunchOf(
@@ -57,7 +58,11 @@ public class MaxConcurrentDriverQueriesConstraint implements
QueryLaunchingConst
String queue = candidateQuery.getQueue();
Integer limit = maxConcurrentQueriesPerQueue.get(queue);
if (limit == null) {
- return true;
+ if (defaultMaxConcurrentQueriesPerQueueLimit != null) { //Check if any
default limit is enabled for all queues
+limit = defaultMaxConcurrentQueriesPerQueueLimit;
+ } else {
+return true;
+ }
}
int launchedOnQueue = 0;
for (QueryContext context :
launchedQueries.getQueries(candidateQuery.getSelectedDriver())) {
http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
index 6db7da7..442cd99 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
@@ -36,6 +36,7 @@ public class MaxConcurrentDriverQueriesConstraintFactory
public static final String MAX_CONCURRENT_QUERIES_KEY =
"driver.max.concurrent.launched.queries";
private static final String PREFIX = MAX_CONCURRENT_QUERIES_KEY + ".per.";
public static final String MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY = PREFIX +
"queue";
+ public static final String
DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY = "*";
public static final String MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY = PREFIX
+ "priority";
private static final EntryParser STRING_INT_PARSER = new
EntryParser() {
@Override
@@ -72,7 +73,8 @@ public class MaxConcurrentDriverQueriesConstraintFactory
maxConcurrentQueries = Integer.parseInt(maxConcurrentQueriesValue);
}
return new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries,
maxConcurrentQueriesPerQueue,
- maxConcurrentQueriesPerPriority);
+ maxConcurrentQueriesPerPriority,
+
maxConcurrentQueriesPerQueue.get(DEFAULT_MAX_CONCURRENT_QUERIES_PER_QUEUE_LIMIT_KEY));
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/34223e88/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
--
diff
lens git commit: LENS-1020: Add server config tests to regression
Repository: lens
Updated Branches:
refs/heads/master d6b121692 -> 59c2f8b11
LENS-1020: Add server config tests to regression
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/59c2f8b1
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/59c2f8b1
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/59c2f8b1
Branch: refs/heads/master
Commit: 59c2f8b11d1bfef58865e306b9b589c0ee6e0175
Parents: d6b1216
Author: Archana H
Authored: Tue May 10 14:59:03 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue May 10 14:59:03 2016 +0530
--
lens-regression/pom.xml | 5 +
.../regression/core/constants/DriverConfig.java | 3 +
.../core/constants/QueryInventory.java | 9 +-
.../core/helpers/LensServerHelper.java | 5 +-
.../regression/core/helpers/QueryHelper.java| 9 +-
.../regression/core/helpers/SessionHelper.java | 2 +-
.../apache/lens/regression/util/AssertUtil.java | 1 +
.../org/apache/lens/regression/util/Util.java | 41 +-
.../lens/regression/client/ITListQueryTest.java | 4 +-
.../regression/client/ITPreparedQueryTests.java | 22 +-
.../lens/regression/client/ITQueryApiTests.java | 10 +-
.../lens/regression/client/KillQueryTests.java | 4 +-
.../regression/client/SessionResourceTests.java | 6 +-
.../regression/config/ITServerConfigTests.java | 601 +++
14 files changed, 661 insertions(+), 61 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/pom.xml
--
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 08a7ece..b9841e8 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -49,6 +49,11 @@
${project.version}
+ org.apache.lens
+ lens-driver-jdbc
+ ${project.version}
+
+
com.jcraft
jsch
http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
--
diff --git
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
index 5a2df11..da13715 100644
---
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
+++
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
@@ -19,6 +19,7 @@
package org.apache.lens.regression.core.constants;
+import org.apache.lens.driver.jdbc.JDBCDriverConfConstants;
import
org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
public class DriverConfig {
@@ -33,6 +34,8 @@ public class DriverConfig {
MAX_CONCURRENT_QUERIES_PER_PRIORITY_KEY;
public static final String QUEUE_MAX_CONCURRENT =
MaxConcurrentDriverQueriesConstraintFactory.
MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY;
+ public static final String JDBC_POOL_SIZE =
JDBCDriverConfConstants.ConnectionPoolProperties.
+ JDBC_POOL_MAX_SIZE.getConfigKey();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/59c2f8b1/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
--
diff --git
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
index 869a96b..8de3994 100644
---
a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
+++
b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryInventory.java
@@ -39,7 +39,11 @@ public class QueryInventory {
private QueryInventory() {
+ }
+ public static String getSleepQuery(String time){
+String query = String.format(SLEEP_QUERY_TIME, time);
+return query;
}
public static final String QUERY = "cube select id,name from sample_dim
where name != 'first'";
@@ -70,8 +74,7 @@ public class QueryInventory {
public static final String SLEEP_FUNCTION = "CREATE TEMPORARY FUNCTION sleep
AS 'hive.udf.SampleUdf'";
public static final String SLEEP_QUERY = "cube select sleep(name) from
sample_dim where name != 'first'";
-
- // public static final String SLEEP_QUERY = "cube select sleep(name,2)
from sample_dim where name != 'first'";
+ public static final String SLEEP_QUERY_TIME = "cube select
sleepTime(name,%s) from sample_dim where name != 'first'";
public static final String NO_CUBE_KEYWORD_QUERY = "select
sample_dim_chain.name, measure4 fro
lens git commit: LENS-1032: Add option to kill the query on timeout api on timeout
Repository: lens Updated Branches: refs/heads/master 389daac9a -> 3a265f5e6 LENS-1032: Add option to kill the query on timeout api on timeout Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3a265f5e Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3a265f5e Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3a265f5e Branch: refs/heads/master Commit: 3a265f5e68d385f75a4e92072099df8b8519bdb0 Parents: 389daac Author: Puneet Gupta Authored: Thu May 12 13:27:02 2016 +0530 Committer: Rajat Khandelwal Committed: Thu May 12 13:27:03 2016 +0530 -- src/site/apt/user/client-config.apt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/3a265f5e/src/site/apt/user/client-config.apt -- diff --git a/src/site/apt/user/client-config.apt b/src/site/apt/user/client-config.apt index c91c944..4f0b2b4 100644 --- a/src/site/apt/user/client-config.apt +++ b/src/site/apt/user/client-config.apt @@ -42,6 +42,8 @@ Lens client configuration *--+--+---+--+ |9|lens.client.ws.request.filternames|requestfilter|These JAX-RS filters would be started in the specified order when lens-client starts| *--+--+---+--+ -|10|lens.server.base.url|http://0.0.0.0:/lensapi|The base url for the lens server| +|10|lens.query.cancel.on.timeout|false|Specifies whether to attempt cancellation of a query whose execution takes longer than the timeout value specified while submitting the query for execution.| +*--+--+---+--+ +|11|lens.server.base.url|http://0.0.0.0:/lensapi|The base url for the lens server| *--+--+---+--+ The configuration parameters and their default values
lens git commit: LENS-771 : Add test for multiple chain ref columns having subset chains
Repository: lens
Updated Branches:
refs/heads/master 3a265f5e6 -> dfd332902
LENS-771 : Add test for multiple chain ref columns having subset chains
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dfd33290
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dfd33290
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dfd33290
Branch: refs/heads/master
Commit: dfd332902c24a897b49362ec8f217de664ab3007
Parents: 3a265f5
Author: Amareshwari Sriramadasu
Authored: Thu May 12 13:30:02 2016 +0530
Committer: Rajat Khandelwal
Committed: Thu May 12 13:30:02 2016 +0530
--
.../cube/metadata/ReferencedDimAttribute.java | 7 ++-
.../org/apache/lens/cube/parse/CubeTestSetup.java | 9 +++--
.../cube/parse/TestDenormalizationResolver.java | 18 ++
lens-examples/src/main/resources/sales-cube.xml | 1 +
.../src/test/resources/yaml/sales-cube.yaml | 3 +++
5 files changed, 35 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/dfd33290/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java
index 4c98d86..4a9dd54 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAttribute.java
@@ -42,6 +42,11 @@ public class ReferencedDimAttribute extends BaseDimAttribute
{
public static class ChainRefCol {
private final String chainName;
private final String refColumn;
+
+public ChainRefCol(String chainName, String refColumn) {
+ this.chainName = chainName.toLowerCase();
+ this.refColumn = refColumn.toLowerCase();
+}
}
public ReferencedDimAttribute(FieldSchema column, String displayString,
String chainName, String refColumn,
@@ -52,7 +57,7 @@ public class ReferencedDimAttribute extends BaseDimAttribute {
public ReferencedDimAttribute(FieldSchema column, String displayString,
String chainName, String refColumn,
Date startTime, Date endTime, Double cost, Long numOfDistinctValues)
throws LensException {
this(column, displayString,
- Collections.singletonList(new ChainRefCol(chainName.toLowerCase(),
refColumn.toLowerCase())), startTime, endTime,
+ Collections.singletonList(new ChainRefCol(chainName, refColumn)),
startTime, endTime,
cost, numOfDistinctValues);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/dfd33290/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index c0035a7..b9694d7 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -983,6 +983,11 @@ public class CubeTestSetup {
"state country", "cubestatecountry", "name", null, null, null));
cubeDimensions2.add(new ReferencedDimAttribute(new
FieldSchema("citycountry", "string", ""),
"city country", "cubecitystatecountry", "name", null, null, null));
+List refCols = new ArrayList<>();
+refCols.add(new ChainRefCol("cubeState", "countrycapital"));
+refCols.add(new ChainRefCol("cubeCityStateCountry", "capital"));
+cubeDimensions2.add(new ReferencedDimAttribute(new
FieldSchema("cubeCountryCapital", "String", "ref dim"),
+ "Country capital", refCols, null, null, null, null));
Map cubeProperties = new HashMap<>();
cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(BASE_CUBE_NAME),
@@ -2582,8 +2587,8 @@ public class CubeTestSetup {
});
}
};
-Dimension countryDim = new Dimension(dimName, dimAttrs, null, joinchains,
dimProps, 0L);
-client.createDimension(countryDim);
+Dimension stateDim = new Dimension(dimName, dimAttrs, null, joinchains,
dimProps, 0L);
+client.createDimension(stateDim);
String dimTblName = "statetable";
List dimColumns = new ArrayList();
http://git-wip-us.apache.org/repos/asf/lens/blob/dfd33290/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index 51ba636..5505ed4 10
lens git commit: LENS-1041: Test case org.apache.lens.client.TestLensClient.testTimeout Failing intermittently
Repository: lens
Updated Branches:
refs/heads/master 3d1b48c44 -> efa247f07
LENS-1041: Test case org.apache.lens.client.TestLensClient.testTimeout Failing
intermittently
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/efa247f0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/efa247f0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/efa247f0
Branch: refs/heads/master
Commit: efa247f07c14760c1eb83226ba15eef8eabbc01b
Parents: 3d1b48c
Author: Puneet Gupta
Authored: Thu May 12 16:00:55 2016 +0530
Committer: Rajat Khandelwal
Committed: Thu May 12 16:00:55 2016 +0530
--
.../test/java/org/apache/lens/client/TestLensClient.java | 10 --
1 file changed, 4 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/efa247f0/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
--
diff --git
a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
index ff3b888..5cf7417 100644
--- a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
+++ b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
@@ -249,11 +249,12 @@ public class TestLensClient extends
LensAllApplicationJerseyTest {
public void testTimeout() throws LensAPIException {
LensClientConfig config = createLensClientConfigWithServerUrl();
-//Timeout Expected
-config.setInt(LensClientConfig.READ_TIMEOUT_MILLIS, 200);
LensClient lensClient = new LensClient(config);
assertTrue(lensClient.setDatabase(TEST_DB));
try {
+ // Setting very small timeout. Expecting timeouts after this
+ // Note: Timeout values can be changed even after LensClient has been
created.
+ config.setInt(LensClientConfig.READ_TIMEOUT_MILLIS, 200);
lensClient.executeQueryWithTimeout("cube select id,name from test_dim",
"test1", 10);
fail("Read Timeout was expected");
} catch (Exception e) {
@@ -264,11 +265,8 @@ public class TestLensClient extends
LensAllApplicationJerseyTest {
log.debug("Expected Exception", e);
}
}
-lensClient.closeConnection();
-//No Timeout Expected
-lensClient = new LensClient(config);
-assertTrue(lensClient.setDatabase(TEST_DB));
+//Setting back default timeout. Not expecting timeouts after this
config.setInt(LensClientConfig.READ_TIMEOUT_MILLIS,
LensClientConfig.DEFAULT_READ_TIMEOUT_MILLIS);
QueryHandleWithResultSet result = lensClient.executeQueryWithTimeout("cube
select id,name from test_dim", "test2",
10);
[1/2] lens git commit: LENS-1040 : cubesegmentation commands are conflicting with cube commands
Repository: lens
Updated Branches:
refs/heads/master f3e6bc542 -> 20c1a53f4
http://git-wip-us.apache.org/repos/asf/lens/blob/20c1a53f/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 1e11325..1b6b819 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -502,20 +502,20 @@ public final class JAXBUtils {
return xpList;
}
- public static Set
xCubeSegmentsFromCubeSegments(Set segs) {
-Set xsegs = new HashSet();
+ public static Set xSegmentsFromSegments(Set segs) {
+Set xsegs = new HashSet();
if (segs != null && !segs.isEmpty()) {
- for (CubeSegment seg : segs) {
-XCubeSegment xcubeSeg = XCF.createXCubeSegment();
+ for (Segment seg : segs) {
+XSegment xcubeSeg = XCF.createXSegment();
xcubeSeg.setCubeName(seg.getName());
-xcubeSeg.setSegmentParameters(getXpropertiesFromCubeSegment(seg));
+xcubeSeg.setSegmentParameters(getXpropertiesFromSegment(seg));
xsegs.add(xcubeSeg);
}
}
return xsegs;
}
- public static XProperties getXpropertiesFromCubeSegment(CubeSegment cseg) {
+ public static XProperties getXpropertiesFromSegment(Segment cseg) {
XProperties xproperties = XCF.createXProperties();
for (String prop : cseg.getProperties().keySet()) {
String segPrefix = MetastoreUtil.getSegmentPropertyKey(cseg.getName());
@@ -699,7 +699,7 @@ public final class JAXBUtils {
mapFromXProperties(fact.getProperties()));
}
- public static CubeSegmentation
cubeSegmentationFromXCubeSegmentation(XCubeSegmentation seg) throws
LensException {
+ public static Segmentation segmentationFromXSegmentation(XSegmentation seg)
throws LensException {
Map props = new HashMap<>();
// Skip properties with keyword internal. These properties are internal to
lens
@@ -709,9 +709,9 @@ public final class JAXBUtils {
props.put(prop, mapFromXProperties(seg.getProperties()).get(prop));
}
}
-return new CubeSegmentation(seg.getCubeName(),
+return new Segmentation(seg.getCubeName(),
seg.getName(),
-cubeSegmentsFromXCubeSegments(seg.getCubeSegements()),
+segmentsFromXSegments(seg.getSegements()),
seg.getWeight(),
props);
}
@@ -731,18 +731,18 @@ public final class JAXBUtils {
return fact;
}
- public static XCubeSegmentation
xsegmentationFromCubeSegmentation(CubeSegmentation cSeg) {
-XCubeSegmentation seg = XCF.createXCubeSegmentation();
+ public static XSegmentation xsegmentationFromSegmentation(Segmentation cSeg)
{
+XSegmentation seg = XCF.createXSegmentation();
seg.setName(cSeg.getName());
seg.setProperties(new XProperties());
-seg.setCubeSegements(new XCubeSegments());
+seg.setSegements(new XSegments());
seg.setWeight(cSeg.weight());
seg.setCubeName(cSeg.getBaseCube());
if (xPropertiesFromMap(cSeg.getProperties()) != null) {
seg.getProperties().getProperty().addAll(xPropertiesFromMap(cSeg.getProperties()));
}
-seg.getCubeSegements().getCubeSegment().
-addAll(xCubeSegmentsFromCubeSegments(cSeg.getCubeSegments()));
+seg.getSegements().getSegment().
+addAll(xSegmentsFromSegments(cSeg.getSegments()));
return seg;
}
@@ -843,16 +843,16 @@ public final class JAXBUtils {
return storageTableMap;
}
- public static Set cubeSegmentsFromXCubeSegments(XCubeSegments
segs) {
-Set cubeSegs = new HashSet<>();
-for (XCubeSegment xcube : segs.getCubeSegment()){
+ public static Set segmentsFromXSegments(XSegments segs) {
+Set cubeSegs = new HashSet<>();
+for (XSegment xcube : segs.getSegment()){
Map segProp = new HashMap<>();
if (xcube.getSegmentParameters() != null) {
for (XProperty prop : xcube.getSegmentParameters().getProperty()) {
segProp.put(prop.getName(), prop.getValue());
}
}
- cubeSegs.add(new CubeSegment(xcube.getCubeName(), segProp));
+ cubeSegs.add(new Segment(xcube.getCubeName(), segProp));
}
return cubeSegs;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/20c1a53f/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 7395c83..6f10d58 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++
b/lens-server
[2/2] lens git commit: LENS-1040 : cubesegmentation commands are conflicting with cube commands
LENS-1040 : cubesegmentation commands are conflicting with cube commands Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/20c1a53f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/20c1a53f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/20c1a53f Branch: refs/heads/master Commit: 20c1a53f4c186c88eab94ce0932417bf17418c75 Parents: f3e6bc5 Author: Sushil Mohanty Authored: Wed May 18 14:55:13 2016 +0530 Committer: Rajat Khandelwal Committed: Wed May 18 14:55:13 2016 +0530 -- lens-api/src/main/resources/cube-0.1.xsd| 48 ++--- .../LensCubeSegmentationCommands.java | 137 - .../annotations/LensSegmentationCommands.java | 137 + .../cli/TestLensCubeSegmentationCommands.java | 156 --- .../lens/cli/TestLensSegmentationCommands.java | 156 +++ lens-cli/src/test/resources/seg1.xml| 20 +- .../java/org/apache/lens/client/LensClient.java | 24 +-- .../apache/lens/client/LensMetadataClient.java | 56 +++--- .../lens/cube/metadata/CubeMetastoreClient.java | 88 - .../apache/lens/cube/metadata/CubeSegment.java | 35 .../lens/cube/metadata/CubeSegmentation.java| 190 --- .../lens/cube/metadata/MetastoreConstants.java | 4 +- .../org/apache/lens/cube/metadata/Segment.java | 35 .../apache/lens/cube/metadata/Segmentation.java | 190 +++ .../cube/metadata/TestCubeMetastoreClient.java | 46 ++--- .../apache/lens/examples/SampleMetastore.java | 10 +- lens-examples/src/main/resources/seg1.xml | 20 +- lens-examples/src/test/resources/yaml/seg1.yaml | 2 +- .../api/metastore/CubeMetastoreService.java | 28 +-- .../metastore/CubeMetastoreServiceImpl.java | 32 ++-- .../apache/lens/server/metastore/JAXBUtils.java | 36 ++-- .../server/metastore/MetastoreResource.java | 90 - .../server/metastore/TestMetastoreService.java | 94 - 23 files changed, 817 insertions(+), 817 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/20c1a53f/lens-api/src/main/resources/cube-0.1.xsd -- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 4a6a5d7..d6c6bd2 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -1020,10 +1020,10 @@ - + -cube segment belong to the segmentation +Segment belong to the segmentation @@ -1031,8 +1031,8 @@ Segment properties. -The following properties can be specified at a cube segment level : -1. lens.metastore.cube.column.mapping : The column mapping for columns of cube segment if they are +The following properties can be specified at a segment level : +1. lens.metastore.cube.column.mapping : The column mapping for columns of segment if they are different in underlying cube. The value is speciified with comma separated map entries specified with key-values separated by equalto. Example value: id=id1,name=name1 @@ -1042,14 +1042,14 @@ - + -Segments in segmentation. There can be two or more such cube segments. +Segments in segmentation. There can be two or more such segments. - + @@ -1271,56 +1271,56 @@ - + - + -Cube segmentation is associated to a base cube and consists of two or more cube segments. All -cube segments together make cube segmentation complete. +Segmentation is associated to a base cube and consists of two or more segments. All +segments together make segmentation complete. -Properties that can be set for cubesegmentation are -1. cube.segmentation.absolute.start.time: start time of the cube segmentation. -For queries that ask for time before this, this cubesegmentation is not a candidate. Time format +Properties that can be set for segmentation are +1. cube.segmentation.absolute.start.time: start time of the segmentation. +For queries that ask for time before this, this segmentation is not a candidate. Time format can be specified in [-mm[-dd[-hh[:MM[:ss[,SSS]] format. -2. cube.segmentation.relative.start.time: Here you can specify cubesegmentations's relative validity -relative to current time. Useful if you want to specify e.g. this cubesegmentation is valid +2. cube.segmentation.relat
lens git commit: LENS-1143: Fix test cases to incorporate LENS-1019(Same session handle on duplicate query) changes
Repository: lens
Updated Branches:
refs/heads/master 20c1a53f4 -> 18999b974
LENS-1143: Fix test cases to incorporate LENS-1019(Same session handle on
duplicate query) changes
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/18999b97
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/18999b97
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/18999b97
Branch: refs/heads/master
Commit: 18999b9742920774fbb297d3600e9c8ba85a9e06
Parents: 20c1a53
Author: Lavkesh Lahngir
Authored: Wed May 18 15:59:17 2016 +0530
Committer: Rajat Khandelwal
Committed: Wed May 18 15:59:17 2016 +0530
--
lens-ml-lib/src/test/resources/lens-site.xml| 5 +-
.../lens/server/query/TestQueryService.java | 122
.../server/query/TestQueryServiceDuplicate.java | 293 +++
lens-server/src/test/resources/lens-site.xml| 5 +
4 files changed, 302 insertions(+), 123 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/18999b97/lens-ml-lib/src/test/resources/lens-site.xml
--
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml
b/lens-ml-lib/src/test/resources/lens-site.xml
index 9be7850..3398b8d 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -152,11 +152,14 @@
local
-
lens.server.mocknonlens.service.impl
org.apache.lens.server.MockNonLensService
Implementation class for session service
+
+lens.server.duplicate.query.allowed
+true
+
http://git-wip-us.apache.org/repos/asf/lens/blob/18999b97/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 692a0a0..797886d 100644
---
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -774,128 +774,6 @@ public class TestQueryService extends LensJerseyTest {
validateHttpEndPoint(target(), null, handle3, null);
}
- @Test(dataProvider = "mediaTypeData")
- public void testExecuteAsyncDuplicate(MediaType mt) throws
InterruptedException, IOException, LensException {
-// test post execute op
-final WebTarget target = target().path("queryapi/queries");
-
-queryService.pauseQuerySubmitter(true);
-final FormDataMultiPart mp = new FormDataMultiPart();
-mp.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(),
lensSessionId, mt));
-mp.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID,
IDSTR from "
-+ TEST_TABLE));
-mp.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("operation").build(),
"execute"));
-mp.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(),
-mt));
-// Dummy query
-final QueryHandle handle = target.request(mt)
-.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new
GenericType>() {
-}).getData();
-
-target.path(handle.toString()).queryParam("sessionid",
lensSessionId).request(mt).delete(APIResult.class);
-
-final QueryHandle handle1 = target.request(mt)
-.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new
GenericType>() {
-}).getData();
-
-final QueryHandle handle2 = target.request(mt)
-.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new
GenericType>() {
-}).getData();
-
-assertNotNull(handle1);
-assertNotNull(handle2);
-assertEquals(handle1, handle2);
-// Cancel the query
-target.path(handle1.toString()).queryParam("sessionid",
lensSessionId).request(mt).delete(APIResult.class);
-
-// Create a different query
-final FormDataMultiPart mp1 = new FormDataMultiPart();
-mp1.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(),
lensSessionId, mt));
-mp1.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID,
IDSTR from "
-+ TEST_TABLE));
-mp1.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("operation").build(),
"execute"));
-mp1.bodyPart(new
FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(),
-mt));
-final QueryHandle handle3 = target.request(mt)
-.post(Entity.entity(mp1, MediaType.MULTIPART_FORM_DATA_TYPE), new
GenericType>() {
-}).getData();
-ass
lens git commit: LENS-1150: HiveDriver should create sessions independent of the transport. Also, lens doesn't need operation logs.
Repository: lens
Updated Branches:
refs/heads/master bf4d540b4 -> 2feb8e201
LENS-1150: HiveDriver should create sessions independent of the transport.
Also, lens doesn't need operation logs.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2feb8e20
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2feb8e20
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2feb8e20
Branch: refs/heads/master
Commit: 2feb8e201b5131947ae5f737294d0e1dc6641eaf
Parents: bf4d540
Author: Rajat Khandelwal
Authored: Tue May 24 14:52:41 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue May 24 14:52:41 2016 +0530
--
.../org/apache/lens/driver/hive/HiveDriver.java | 7 +-
.../src/main/resources/lensserver-default.xml | 9 +
src/site/apt/admin/config.apt | 220 ++-
3 files changed, 126 insertions(+), 110 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2feb8e20/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index b85a66f..04d059d 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -154,6 +154,11 @@ public class HiveDriver extends AbstractLensDriver {
// package-local. Test case can change.
boolean whetherCalculatePriority;
private DriverQueryHook queryHook;
+ private static final Map SESSION_CONF = new HashMap() {
+{
+ put(HiveConf.ConfVars.HIVE_SERVER2_CLOSE_SESSION_ON_DISCONNECT.varname,
"false");
+}
+ };
@Getter
protected ImmutableSet queryConstraints;
@@ -974,7 +979,7 @@ public class HiveDriver extends AbstractLensDriver {
SessionHandle hiveSession;
if (!lensToHiveSession.containsKey(sessionDbKey)) {
try {
- hiveSession = getClient().openSession(ctx.getClusterUser(), "");
+ hiveSession = getClient().openSession(ctx.getClusterUser(), "",
SESSION_CONF);
lensToHiveSession.put(sessionDbKey, hiveSession);
log.info("New hive session for user: {} , lens session: {} , hive
session handle: {} , driver : {}",
ctx.getClusterUser(), sessionDbKey,
hiveSession.getHandleIdentifier(), getFullyQualifiedName());
http://git-wip-us.apache.org/repos/asf/lens/blob/2feb8e20/lens-server/src/main/resources/lensserver-default.xml
--
diff --git a/lens-server/src/main/resources/lensserver-default.xml
b/lens-server/src/main/resources/lensserver-default.xml
index 25e0518..5492132 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -526,6 +526,15 @@
+hive.server2.logging.operation.enabled
+false
+Disable the operation logging, since there are no execution
operation happening in lens session.
+ Keeping it true might conflict with Hive Server if both are running on
the same machine causing disappearance
+ of the operation log directory.
+
+
+
+
hive.server2.authentication
NONE
http://git-wip-us.apache.org/repos/asf/lens/blob/2feb8e20/src/site/apt/admin/config.apt
--
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 40da099..7ad1843 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -38,227 +38,229 @@ Lens server configuration
*--+--+---+--+
|7|hive.server2.log.redirection.enabled|false|Disable the log direction on the
lens server sessions. There are no logs required for each session in lens
sessions. This will decrease number of file handles associated to lens server.|
*--+--+---+--+
-|8|lens.server.appevent.ws.listener.impl|org.apache.lens.server.LensApplicationListener|Implementation
class for Lens Application Listener|
+|8|hive.server2.logging.operation.enabled|false|Disable the operation logging,
since there are no execution operation happening in lens session. Keeping it
true might conflict with Hive Server if both are running on the same machine
causing disappearance of the operation log directory.|
*--+--+---+--+
-|9|lens.server.authentication.ws.filter.impl|org.apache.lens.server.AuthenticationFilter|Implementation
class for Authentication Filter|
+|9|lens.server.appevent.ws.listener.impl|org.apache.lens.server.LensApplicationListener|Implementation
class for Lens Application Listener|
*--+--+---+--+
-|10|lens.server.base.url|http://0.0.0.0:999
lens git commit: LENS-1010: Make LensClient implement java.lang.AutoClosable
Repository: lens
Updated Branches:
refs/heads/master 84678c5e3 -> 62cafa448
LENS-1010: Make LensClient implement java.lang.AutoClosable
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/62cafa44
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/62cafa44
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/62cafa44
Branch: refs/heads/master
Commit: 62cafa4484ffaebc4f6376f50ca2900936c1a0fc
Parents: 84678c5
Author: Rajat Khandelwal
Authored: Tue May 24 18:41:20 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue May 24 18:41:20 2016 +0530
--
.../java/org/apache/lens/client/LensClient.java | 10 ++-
.../org/apache/lens/client/LensConnection.java | 68 +++-
.../org/apache/lens/client/TestLensClient.java | 5 +-
.../src/main/resources/olap-query-conf.xml | 9 +++
.../org/apache/lens/server/BaseLensService.java | 2 +-
.../org/apache/lens/server/LensServices.java| 8 +++
.../server/metastore/MetastoreResource.java | 5 +-
.../org/apache/lens/server/TestServerMode.java | 3 -
src/site/apt/user/olap-query-conf.apt | 12 ++--
9 files changed, 73 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/62cafa44/lens-client/src/main/java/org/apache/lens/client/LensClient.java
--
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 480b6f8..439ca90 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -53,7 +53,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class LensClient {
+public class LensClient implements AutoCloseable {
public static final String CLILOGGER = "cliLogger";
private static final String DEFAULT_PASSWORD = "";
@Getter
@@ -403,9 +403,9 @@ public class LensClient {
return this.connection.getConnectionParams(key);
}
- public APIResult closeConnection() {
+ public void closeConnection() {
log.debug("Closing lens connection: {}", new LensConnectionParams(conf));
-return this.connection.close();
+this.connection.close();
}
public APIResult addJarResource(String path) {
@@ -719,4 +719,8 @@ public class LensClient {
return mc.dropSegmentation(segName);
}
+ @Override
+ public void close() {
+closeConnection();
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/62cafa44/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
index 4951866..bb15b23 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
@@ -22,7 +22,6 @@ import static org.apache.lens.client.LensClientConfig.*;
import java.net.ConnectException;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -39,6 +38,7 @@ import org.apache.lens.api.APIResult;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver;
+import org.apache.lens.client.exceptions.LensClientException;
import org.apache.lens.client.exceptions.LensClientServerConnectionException;
import org.glassfish.jersey.client.ClientProperties;
@@ -55,7 +55,7 @@ import lombok.extern.slf4j.Slf4j;
* Top level client connection class which is used to connect to a lens server.
*/
@Slf4j
-public class LensConnection {
+public class LensConnection implements AutoCloseable {
/** The params. */
private final LensConnectionParams params;
@@ -66,6 +66,7 @@ public class LensConnection {
/** The session handle. */
@Getter
private LensSessionHandle sessionHandle;
+ private boolean closed = false;
/**
* Construct a connection to lens server specified by connection parameters.
@@ -119,9 +120,8 @@ public class LensConnection {
public Client buildClient() {
ClientBuilder cb =
ClientBuilder.newBuilder().register(MultiPartFeature.class).register(MoxyJsonFeature.class)
.register(MoxyJsonConfigurationContextResolver.class);
-Iterator> itr = params.getRequestFilters().iterator();
-while (itr.hasNext()) {
- cb.register(itr.next());
+for (Class aClass : params.getRequestFilters()) {
+ cb.register(aClass);
}
Client client = cb.build();
@@ -200,28 +200,43 @@ public class
lens git commit: LENS-1159: Lens startup should not be adding HIVE_HOME/lib/* to app class path
Repository: lens
Updated Branches:
refs/heads/master 5c25e68fe -> 478d6d8a1
LENS-1159: Lens startup should not be adding HIVE_HOME/lib/* to app class path
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/478d6d8a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/478d6d8a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/478d6d8a
Branch: refs/heads/master
Commit: 478d6d8a12e6d6f876af563ccfce369e9b1aa705
Parents: 5c25e68
Author: Rajat Khandelwal
Authored: Tue May 31 20:06:35 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue May 31 20:06:35 2016 +0530
--
tools/scripts/lens-ctl | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/478d6d8a/tools/scripts/lens-ctl
--
diff --git a/tools/scripts/lens-ctl b/tools/scripts/lens-ctl
index 8523360..b1a5233 100755
--- a/tools/scripts/lens-ctl
+++ b/tools/scripts/lens-ctl
@@ -123,15 +123,15 @@ start() {
fi
if [ "$HIVE_HOME" != "" ]; then
-echo "HIVE_HOME is set, adding ${HIVE_HOME}/lib/* into lens classpath"
-LENSCPPATH=${LENSCPPATH}:`ls ${HIVE_HOME}/lib/* 2>/dev/null | tr "\n" ':'
2>/dev/null`
+echo "HIVE_HOME is set, adding ${HIVE_HOME}/lib/*jar into lens classpath"
+LENSCPPATH=${LENSCPPATH}:`ls ${HIVE_HOME}/lib/*jar 2>/dev/null | tr "\n"
':' 2>/dev/null`
else
echo "HIVE_HOME is not set. Set HIVE_HOME and try again"
exit 1
fi
# Add HIVE_HOME to HADOOP_CLASS_PATH
- HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HIVE_HOME}/lib/*"
+ HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HIVE_HOME}/lib/*jar"
export HADOOP_CLASSPATH
mkdir -p $LENS_LOG_DIR
lens git commit: LENS-661: Added Weighted driver selector
Repository: lens
Updated Branches:
refs/heads/master 478d6d8a1 -> 1774ced2e
LENS-661: Added Weighted driver selector
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1774ced2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1774ced2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1774ced2
Branch: refs/heads/master
Commit: 1774ced2e49d4ffa8f68f1493c1e9a26fc2e119c
Parents: 478d6d8
Author: Anshul Gupta
Authored: Tue May 31 20:13:37 2016 +0530
Committer: Rajat Khandelwal
Committed: Tue May 31 20:13:37 2016 +0530
--
.../lens/driver/cube/TestMinCostSelector.java | 116
.../lens/server/api/LensConfConstants.java | 15 +
.../driver/WeightedQueryCostDriverSelector.java | 101 +++
.../server/api/driver/TestMinCostSelector.java | 104 +++
.../api/driver/TestWeightedCostSelector.java| 288 +++
src/site/apt/admin/jdbcdriver-config.apt| 3 +-
6 files changed, 510 insertions(+), 117 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/1774ced2/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
deleted file mode 100644
index 8267229..000
---
a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.cube;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.driver.MinQueryCostSelector;
-import org.apache.lens.server.api.driver.MockDriver;
-import org.apache.lens.server.api.driver.MockFailDriver;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.MockQueryContext;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- * The Class TestMinCostSelector.
- */
-public class TestMinCostSelector {
-
- private MockQueryContext createMockContext(String query, Configuration conf,
LensConf lensConf,
-Map driverQueries) throws LensException {
-MockQueryContext ctx = new MockQueryContext(query, lensConf, conf,
driverQueries.keySet());
-ctx.setDriverQueries(driverQueries);
-ctx.estimateCostForDrivers();
-return ctx;
- }
-
- private MockQueryContext createMockContext(String query, Configuration conf,
LensConf lensConf,
-List drivers, Map driverQueries) throws
LensException {
-MockQueryContext ctx = new MockQueryContext(query, lensConf, conf,
driverQueries.keySet());
-ctx.setDriverQueries(driverQueries);
-ctx.estimateCostForDrivers();
-return ctx;
- }
-
- @Test
- public void testMinCostSelector() throws LensException {
-MinQueryCostSelector selector = new MinQueryCostSelector();
-List drivers = new ArrayList();
-Map driverQueries = new HashMap();
-Configuration conf = new Configuration();
-LensConf qconf = new LensConf();
-
-MockDriver d1 = new MockDriver();
-d1.configure(conf, null, null);
-MockDriver d2 = new MockDriver();
-d2.configure(conf, null, null);
-MockFailDriver fd1 = new MockFailDriver();
-fd1.configure(conf, null, null);
-MockFailDriver fd2 = new MockFailDriver();
-fd2.configure(conf, null, null);
-
-drivers.add(d1);
-drivers.add(d2);
-String query = "test query";
-driverQueries.put(d1, query);
-
-MockQueryContext ctx = createMockContext(query, conf, qconf,
driverQueries);
-LensDriver selected = selector.select(ctx, conf);
-
-Assert.assertEquals(d1, selected);
-driverQueries.put(d2, query);
-driverQueries.remove(d1);
-ctx = createMockContext(
lens git commit: Correcting description of lens.driver.jdbc.get.connection.timeout
Repository: lens Updated Branches: refs/heads/master fcc0cafd6 -> c7a39bc93 Correcting description of lens.driver.jdbc.get.connection.timeout Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c7a39bc9 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c7a39bc9 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c7a39bc9 Branch: refs/heads/master Commit: c7a39bc93d978f0273dc99e39bca89a1cd776845 Parents: fcc0caf Author: Puneet Gupta Authored: Fri Jun 3 17:59:00 2016 +0530 Committer: Rajat Khandelwal Committed: Fri Jun 3 17:59:00 2016 +0530 -- lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml | 6 -- src/site/apt/admin/jdbcdriver-config.apt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/c7a39bc9/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml -- diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml index b446f7a..66645b9 100644 --- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml +++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml @@ -91,8 +91,10 @@ lens.driver.jdbc.get.connection.timeout 1 -Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket - +The number of milliseconds a client calling getConnection() will wait for a Connection to be + checked-in or acquired when the pool is exhausted. Zero means wait indefinitely. Setting any positive value + will cause the getConnection () call to time-out and break with an SQLException after the specified number + of milliseconds. The default value of this property is 10 secs. http://git-wip-us.apache.org/repos/asf/lens/blob/c7a39bc9/src/site/apt/admin/jdbcdriver-config.apt -- diff --git a/src/site/apt/admin/jdbcdriver-config.apt b/src/site/apt/admin/jdbcdriver-config.apt index 8fd5295..96c98ad 100644 --- a/src/site/apt/admin/jdbcdriver-config.apt +++ b/src/site/apt/admin/jdbcdriver-config.apt @@ -70,7 +70,7 @@ Jdbc driver configuration *--+--+---+--+ |23|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set| *--+--+---+--+ -|24|lens.driver.jdbc.get.connection.timeout|1|Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket| +|24|lens.driver.jdbc.get.connection.timeout|1|The number of milliseconds a client calling getConnection() will wait for a Connection to be checked-in or acquired when the pool is exhausted. Zero means wait indefinitely. Setting any positive value will cause the getConnection () call to time-out and break with an SQLException after the specified number of milliseconds. The default value of this property is 10 secs.| *--+--+---+--+ |25|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed| *--+--+---+--+
