Initial commit
Project: http://git-wip-us.apache.org/repos/asf/drill-site/repo Commit: http://git-wip-us.apache.org/repos/asf/drill-site/commit/c4de0f83 Tree: http://git-wip-us.apache.org/repos/asf/drill-site/tree/c4de0f83 Diff: http://git-wip-us.apache.org/repos/asf/drill-site/diff/c4de0f83 Branch: refs/heads/asf-site Commit: c4de0f83d7247fc9a106a6a1b1068cdf34028afd Parents: Author: Tomer Shiran <[email protected]> Authored: Mon May 4 12:25:56 2015 -0700 Committer: Tomer Shiran <[email protected]> Committed: Mon May 4 12:25:56 2015 -0700 ---------------------------------------------------------------------- README.md | 51 + architecture/index.html | 219 + blog/2014/11/19/sql-on-mongodb/index.html | 323 + .../12/02/drill-top-level-project/index.html | 211 + .../running-sql-queries-on-amazon-s3/index.html | 241 + .../index.html | 215 + blog/2014/12/16/whats-coming-in-2015/index.html | 294 + blog/2014/12/23/drill-0.7-released/index.html | 187 + .../index.html | 223 + .../index.html | 178 + blog/2015/03/31/drill-0.8-released/index.html | 195 + .../index.html | 180 + blog/2015/05/04/drill-0.9-released/index.html | 202 + blog/index.html | 169 + community-resources/index.html | 157 + css/arrows.css | 86 + css/breadcrumbs.css | 40 + css/code.css | 69 + css/docpage.css | 389 + css/download.css | 33 + css/main.css | 449 + css/responsive.css | 252 + css/search.css | 9 + css/style.css | 826 + css/syntax.css | 60 + data/index.html | 1088 ++ doap_Drill.rdf | 57 + docs/2014-q1-drill-report/index.html | 1013 ++ docs/about-sql-function-examples/index.html | 844 ++ docs/about-the-mapr-sandbox/index.html | 855 ++ .../adding-custom-functions-to-drill/index.html | 864 ++ docs/advanced-properties/index.html | 857 ++ .../index.html | 979 ++ docs/alter-session-command/index.html | 911 ++ docs/alter-system-command/index.html | 937 ++ .../index.html | 1004 ++ .../index.html | 1239 ++ docs/aol-search/index.html | 888 ++ .../apache-drill-0-4-0-release-notes/index.html | 877 ++ .../apache-drill-0-5-0-release-notes/index.html | 862 ++ .../apache-drill-0-6-0-release-notes/index.html | 868 ++ .../apache-drill-0-7-0-release-notes/index.html | 882 ++ .../apache-drill-0-8-0-release-notes/index.html | 864 ++ .../index.html | 1073 ++ docs/apache-drill-contribution-ideas/index.html | 1025 ++ .../index.html | 877 ++ docs/architectural-highlights/index.html | 849 ++ docs/architecture-introduction/index.html | 891 ++ docs/architecture/index.html | 858 ++ docs/archived-pages/index.html | 847 ++ docs/case/index.html | 887 ++ docs/compiling-drill-from-source/index.html | 883 ++ .../index.html | 1247 ++ docs/configuration-options/index.html | 855 ++ .../index.html | 864 ++ .../index.html | 852 ++ .../index.html | 1066 ++ .../index.html | 983 ++ .../index.html | 869 ++ .../index.html | 924 ++ .../index.html | 901 ++ .../index.html | 874 ++ docs/connect-a-data-source/index.html | 872 ++ docs/connecting-to-odbc-data-sources/index.html | 861 ++ docs/contribute-to-drill/index.html | 849 ++ docs/core-modules/index.html | 869 ++ docs/create-table-as-ctas-command/index.html | 964 ++ docs/create-view-command/index.html | 1035 ++ docs/custom-function-interfaces/index.html | 974 ++ .../index.html | 863 ++ docs/data-sources-and-file-formats/index.html | 856 ++ docs/data-type-conversion/index.html | 1762 +++ docs/data-types/index.html | 852 ++ docs/date-time-and-timestamp/index.html | 956 ++ .../index.html | 1302 ++ docs/deploying-and-using-a-hive-udf/index.html | 878 ++ docs/deploying-drill-in-a-cluster/index.html | 935 ++ docs/describe-command/index.html | 934 ++ docs/design-docs/index.html | 858 ++ docs/develop-a-simple-function/index.html | 884 ++ .../index.html | 876 ++ docs/develop-custom-functions/index.html | 859 ++ docs/develop-drill/index.html | 849 ++ docs/developer-information/index.html | 880 ++ .../developing-an-aggregate-function/index.html | 894 ++ docs/drill-default-input-format/index.html | 911 ++ docs/drill-in-10-minutes/index.html | 1145 ++ docs/drill-introduction/index.html | 879 ++ docs/drill-patch-review-tool/index.html | 1009 ++ docs/drill-plan-syntax/index.html | 863 ++ docs/driver-configuration-options/index.html | 856 ++ docs/embedded-mode-prerequisites/index.html | 862 ++ docs/enron-emails/index.html | 856 ++ docs/explain-commands/index.html | 987 ++ docs/file-system-storage-plugin/index.html | 908 ++ docs/flatten/index.html | 934 ++ docs/flexibility/index.html | 925 ++ docs/functions-for-handling-nulls/index.html | 896 ++ docs/getting-started/index.html | 847 ++ .../index.html | 982 ++ docs/handling-different-data-types/index.html | 912 ++ docs/hbase-storage-plugin/index.html | 877 ++ docs/hive-storage-plugin/index.html | 923 ++ docs/hive-to-drill-data-type-mapping/index.html | 1012 ++ docs/how-to-run-the-drill-demo/index.html | 1127 ++ docs/img/11.png | Bin 0 -> 5224 bytes docs/img/18.png | Bin 0 -> 22253 bytes docs/img/19.png | Bin 0 -> 22248 bytes docs/img/21.png | Bin 0 -> 23385 bytes docs/img/30.png | Bin 0 -> 50936 bytes docs/img/4.png | Bin 0 -> 41555 bytes docs/img/40.png | Bin 0 -> 25898 bytes docs/img/42.png | Bin 0 -> 153938 bytes docs/img/46.png | Bin 0 -> 3597 bytes docs/img/51.png | Bin 0 -> 64520 bytes docs/img/52.png | Bin 0 -> 21243 bytes docs/img/53.png | Bin 0 -> 52269 bytes docs/img/54.png | Bin 0 -> 10704 bytes docs/img/58.png | Bin 0 -> 35404 bytes docs/img/7.png | Bin 0 -> 30755 bytes docs/img/BI_to_Drill_2.png | Bin 0 -> 46126 bytes docs/img/DrillWebUI.png | Bin 0 -> 53187 bytes docs/img/DrillbitModules.png | Bin 0 -> 54907 bytes docs/img/HbaseViewCreation0.png | Bin 0 -> 22945 bytes docs/img/HbaseViewDSN.png | Bin 0 -> 32284 bytes docs/img/Hbase_Browse.png | Bin 0 -> 148451 bytes docs/img/Hive_DSN.png | Bin 0 -> 31302 bytes docs/img/ODBC_CustomSQL.png | Bin 0 -> 41405 bytes docs/img/ODBC_HbasePreview2.png | Bin 0 -> 130202 bytes docs/img/ODBC_HbaseView.png | Bin 0 -> 36774 bytes docs/img/ODBC_HiveConnection.png | Bin 0 -> 33385 bytes docs/img/ODBC_to_Drillbit.png | Bin 0 -> 6694 bytes docs/img/ODBC_to_Quorum.png | Bin 0 -> 11684 bytes docs/img/Overview.png | Bin 0 -> 165981 bytes docs/img/Parquet_DSN.png | Bin 0 -> 31356 bytes docs/img/Parquet_Preview.png | Bin 0 -> 78339 bytes docs/img/RegionParquet_table.png | Bin 0 -> 90698 bytes docs/img/SelectHbaseView.png | Bin 0 -> 28721 bytes docs/img/VoterContributions_hbaseview.png | Bin 0 -> 84225 bytes docs/img/connect-plugin.png | Bin 0 -> 36731 bytes docs/img/data-sources-schemachg.png | Bin 0 -> 8071 bytes docs/img/datasources-json-bracket.png | Bin 0 -> 15160 bytes docs/img/datasources-json.png | Bin 0 -> 16364 bytes docs/img/drill-bin.png | Bin 0 -> 85017 bytes docs/img/drill-directory.png | Bin 0 -> 87651 bytes docs/img/drill-runtime.png | Bin 0 -> 78592 bytes docs/img/drill2.png | Bin 0 -> 22806 bytes docs/img/example_query.png | Bin 0 -> 63294 bytes docs/img/get2kno_plugin.png | Bin 0 -> 55794 bytes docs/img/image_1.png | Bin 0 -> 33528 bytes docs/img/image_10.png | Bin 0 -> 53212 bytes docs/img/image_11.png | Bin 0 -> 73527 bytes docs/img/image_12.png | Bin 0 -> 114497 bytes docs/img/image_13.png | Bin 0 -> 92812 bytes docs/img/image_14.png | Bin 0 -> 18639 bytes docs/img/image_15.png | Bin 0 -> 32379 bytes docs/img/image_16.png | Bin 0 -> 45973 bytes docs/img/image_17.png | Bin 0 -> 67424 bytes docs/img/image_2.png | Bin 0 -> 30123 bytes docs/img/image_3.png | Bin 0 -> 81685 bytes docs/img/image_4.png | Bin 0 -> 169567 bytes docs/img/image_5.png | Bin 0 -> 37764 bytes docs/img/image_6.png | Bin 0 -> 57145 bytes docs/img/image_7.png | Bin 0 -> 29885 bytes docs/img/image_8.png | Bin 0 -> 64920 bytes docs/img/image_9.png | Bin 0 -> 43775 bytes docs/img/json-workaround.png | Bin 0 -> 27547 bytes docs/img/loginSandBox.png | Bin 0 -> 67090 bytes docs/img/ngram_plugin.png | Bin 0 -> 51922 bytes docs/img/ngram_plugin2.png | Bin 0 -> 55418 bytes docs/img/plugin-default.png | Bin 0 -> 56412 bytes docs/img/queryFlow.png | Bin 0 -> 37031 bytes docs/img/queuing.png | Bin 0 -> 8343 bytes docs/img/settings.png | Bin 0 -> 3094 bytes docs/img/slide-15-638.png | Bin 0 -> 58477 bytes docs/img/sqlline1.png | Bin 0 -> 23074 bytes docs/img/storageplugin.png | Bin 0 -> 7589 bytes docs/img/student_hive.png | Bin 0 -> 134755 bytes docs/img/value1.png | Bin 0 -> 10017 bytes docs/img/value2.png | Bin 0 -> 13935 bytes docs/img/value3.png | Bin 0 -> 14694 bytes docs/img/value4.png | Bin 0 -> 20297 bytes docs/img/value5.png | Bin 0 -> 44110 bytes docs/img/value6.png | Bin 0 -> 13854 bytes docs/img/value7.png | Bin 0 -> 21719 bytes docs/img/vbApplSettings.png | Bin 0 -> 115803 bytes docs/img/vbEthernet.png | Bin 0 -> 31861 bytes docs/img/vbGenSettings.png | Bin 0 -> 56642 bytes docs/img/vbImport.png | Bin 0 -> 85744 bytes docs/img/vbMaprSetting.png | Bin 0 -> 56436 bytes docs/img/vbNetwork.png | Bin 0 -> 30826 bytes docs/img/vbloginSandBox.png | Bin 0 -> 65477 bytes docs/img/vmLibrary.png | Bin 0 -> 85632 bytes docs/img/vmShare.png | Bin 0 -> 22898 bytes docs/img/vmWelcome.png | Bin 0 -> 39255 bytes docs/index.html | 841 + docs/install-drill-introduction/index.html | 848 ++ docs/install-drill/index.html | 865 ++ .../index.html | 901 ++ .../index.html | 855 ++ docs/installing-drill-on-linux/index.html | 863 ++ docs/installing-drill-on-mac-os-x/index.html | 870 ++ docs/installing-drill-on-windows/index.html | 899 ++ .../index.html | 996 ++ docs/installing-the-driver-on-linux/index.html | 988 ++ .../index.html | 928 ++ .../installing-the-driver-on-windows/index.html | 908 ++ docs/interfaces-introduction/index.html | 891 ++ docs/json-data-model/index.html | 1354 ++ docs/kvgen/index.html | 1043 ++ .../index.html | 864 ++ .../index.html | 1209 ++ .../index.html | 1211 ++ .../index.html | 1192 ++ docs/lexical-structure/index.html | 1005 ++ docs/manage-drill-introduction/index.html | 844 ++ docs/manage-drill/index.html | 887 ++ docs/mapr-db-format/index.html | 867 ++ docs/math-and-trig/index.html | 1194 ++ docs/mongodb-plugin-for-apache-drill/index.html | 1004 ++ .../index.html | 864 ++ docs/nested-data-functions/index.html | 858 ++ docs/nested-data-limitations/index.html | 844 ++ docs/odbc-jdbc-interfaces/index.html | 890 ++ .../005-odbc-on-windows-introduction.md | 38 + .../index.html | 849 ++ docs/operators/index.html | 938 ++ docs/parquet-format/index.html | 1118 ++ docs/partition-pruning/index.html | 906 ++ docs/performance/index.html | 894 ++ .../persistent-configuration-storage/index.html | 925 ++ docs/planning-and-execution-options/index.html | 898 ++ .../index.html | 976 ++ docs/ports-used-by-drill/index.html | 845 ++ docs/progress-reports/index.html | 844 ++ docs/project-bylaws/index.html | 1008 ++ docs/query-data-introduction/index.html | 882 ++ docs/query-data/index.html | 892 ++ docs/query-directory-functions/index.html | 893 ++ docs/query-stages/index.html | 880 ++ .../index.html | 880 ++ docs/querying-a-file-system/index.html | 858 ++ .../index.html | 897 ++ docs/querying-complex-data/index.html | 861 ++ docs/querying-directories/index.html | 927 ++ docs/querying-hbase/index.html | 1006 ++ docs/querying-hive/index.html | 881 ++ docs/querying-json-files/index.html | 878 ++ docs/querying-parquet-files/index.html | 941 ++ docs/querying-plain-text-files/index.html | 953 ++ docs/querying-system-tables/index.html | 997 ++ docs/querying-the-information-schema/index.html | 946 ++ docs/release-notes/index.html | 862 ++ docs/repeated-contains/index.html | 927 ++ docs/repeated-count/index.html | 882 ++ docs/reserved-keywords/index.html | 852 ++ docs/rpc-overview/index.html | 857 ++ docs/sample-data-donuts/index.html | 879 ++ docs/sample-datasets/index.html | 850 ++ docs/select-statements/index.html | 1040 ++ docs/selecting-flat-data/index.html | 857 ++ .../index.html | 862 ++ .../index.html | 882 ++ .../index.html | 899 ++ docs/show-files-command/index.html | 901 ++ docs/show-tables-command/index.html | 966 ++ docs/sql-commands/index.html | 879 ++ docs/sql-conditional-expressions/index.html | 846 ++ docs/sql-extensions/index.html | 910 ++ docs/sql-functions/index.html | 864 ++ docs/sql-reference-introduction/index.html | 852 ++ docs/sql-reference/index.html | 950 ++ docs/start-up-options/index.html | 906 ++ docs/starting-stopping-drill/index.html | 915 ++ docs/storage-plugin-configuration/index.html | 861 ++ docs/storage-plugin-registration/index.html | 862 ++ docs/string-manipulation/index.html | 1286 ++ docs/summary/index.html | 851 ++ docs/supported-data-types/index.html | 1460 ++ docs/supported-sql-commands/index.html | 848 ++ docs/tableau-examples/index.html | 1116 ++ docs/testing-the-odbc-connection/index.html | 880 ++ docs/tutorials-introduction/index.html | 861 ++ docs/tutorials/index.html | 874 ++ docs/use-command/index.html | 993 ++ docs/useful-research/index.html | 898 ++ docs/using-a-connection-string/index.html | 859 ++ .../index.html | 888 ++ docs/using-drill-explorer-on-windows/index.html | 892 ++ docs/using-jdbc/index.html | 994 ++ .../index.html | 1011 ++ .../using-odbc-on-linux-and-mac-os-x/index.html | 867 ++ docs/using-odbc-on-windows/index.html | 858 ++ .../index.html | 894 ++ docs/value-vectors/index.html | 1000 ++ docs/what-is-apache-drill/index.html | 881 ++ docs/why-drill/index.html | 924 ++ docs/wikipedia-edit-history/index.html | 945 ++ docs/workspaces/index.html | 916 ++ download/index.html | 176 + faq/index.html | 317 + favicon.ico | Bin 0 -> 1150 bytes feed.xml | 546 + fonts/icomoon/icomoon.eot | Bin 0 -> 1144 bytes fonts/icomoon/icomoon.svg | 12 + fonts/icomoon/icomoon.ttf | Bin 0 -> 980 bytes fonts/icomoon/icomoon.woff | Bin 0 -> 1056 bytes images/agility-w.png | Bin 0 -> 1337 bytes images/agility.png | Bin 0 -> 1627 bytes images/al.png | Bin 0 -> 1517 bytes images/apachedrill.png | Bin 0 -> 8262 bytes images/ar.png | Bin 0 -> 1419 bytes images/arc-1.jpg | Bin 0 -> 109057 bytes images/arc-2.jpg | Bin 0 -> 137590 bytes images/architecture.png | Bin 0 -> 1360 bytes images/btn-lens.png | Bin 0 -> 1369 bytes images/code-block-bg.png | Bin 0 -> 943 bytes images/community.png | Bin 0 -> 2131 bytes images/docs_intro.png | Bin 0 -> 7433 bytes images/dot-sprite.png | Bin 0 -> 1352 bytes images/download-ico.png | Bin 0 -> 1150 bytes images/download.png | Bin 0 -> 1316 bytes images/drillcompiler.png | Bin 0 -> 25890 bytes images/familiarity-w.png | Bin 0 -> 1537 bytes images/familiarity.png | Bin 0 -> 2356 bytes images/flexibility-w.png | Bin 0 -> 1141 bytes images/flexibility.png | Bin 0 -> 1413 bytes images/home-img1.jpg | Bin 0 -> 94388 bytes images/home-img2.jpg | Bin 0 -> 132354 bytes images/home-img3.jpg | Bin 0 -> 22377 bytes images/intro-bg.gif | Bin 0 -> 1658 bytes images/len.png | Bin 0 -> 1374 bytes images/overview-img1.png | Bin 0 -> 58147 bytes images/reel-bg.png | Bin 0 -> 63054 bytes images/speed.png | Bin 0 -> 2563 bytes images/twitter_32_26_white.png | Bin 0 -> 1526 bytes images/twitterbw.png | Bin 0 -> 545 bytes index.html | 209 + js/drill.js | 252 + js/lib/jquery-1.11.1.min.js | 4 + js/lib/jquery.easing.1.3.js | 205 + js/modernizr.custom.js | 4 + js/script.js | 97 + mailinglists/index.html | 160 + overview/index.html | 168 + search/index.html | 150 + static/sql-on-mongodb/empinfo.json | 18 + static/sql-on-mongodb/replicated.png | Bin 0 -> 26389 bytes static/sql-on-mongodb/sharded.png | Bin 0 -> 37181 bytes static/sql-on-mongodb/standalone.png | Bin 0 -> 9432 bytes team/index.html | 221 + toc/index.html | 13464 +++++++++++++++++ why/index.html | 203 + 353 files changed, 191502 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill-site/blob/c4de0f83/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..caa2d75 --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +The Apache Drill website is built using [Jekyll](http://jekyllrb.com/). + +# Developing and Previewing the Website + +To preview the website on your local machine: + +```bash +jekyll build --config _config.yml,_config-prod.yml +_tools/createdatadocs.py +jekyll serve --config _config.yml,_config-prod.yml +``` +Note that you can skip the first two commands (and only run `jekyll serve`) if you haven't changed the title or path of any of the documentation pages. + +# Compiling the Website + +Once the website is ready, you'll need to compile the site to static HTML so that it can then be published to Apache. This is as simple as running the `jekyll build` command. The _config-prod.yml configuration file causes a few changes to the site: + +* The `noindex` meta tag is removed. We want the production site to be indexed by search engines, but we don't want the staging site to be indexed. +* The base URL is set to `/`. The production site is at `/`, whereas the staging site is at `/drill` (convenient for previewing on GitHub Pages: <http://apache.github.io/drill>). + +```bash +jekyll build --config _config.yml,_config-prod.yml +_tools/createdatadocs.py +jekyll build --config _config.yml,_config-prod.yml +``` + +# Uploading to the Apache Website (Drill Committers Only) + +Apache project websites use a system called svnpubsub for publishing. Basically, the static HTML needs to be pushed by one of the committers into the Apache SVN. + +```bash +svn co https://svn.apache.org/repos/asf/drill/site/trunk/content/drill ../_site-apache +cp -R _site/* ../_site-apache/ +cd ../_site-apache +``` + +Then `svn add` and `svn rm` as needed, and commit the changes via `svn commit -m "Website update"`. Note that once changes are committed via `svn commit`, they will immediately be visible on the live site: <http://drill.apache.org>. + +# Documentation Guidelines + +The documentation pages are under `_docs`. Each file, in its YAML front matter, has two important parameters: + +* title - This is the title of the page. Each page must have a *unique* title +* parent - This is the title of the page's parent page. It should be empty for top-level sections/guides, and be identical to the title attribute of another page in all other cases. + +The name of the file itself doesn't matter except for the alphanumeric order of the filenames. Files that share the same parent are ordered alphanumerically. Note that the content of parent files is ignored, so add an overview/introduction child when needed. + +Best practices: + +* Prefix the filenames with `010-foo.md`, `020-bar.md`, `030-baz.md`, etc. This allows room to add files in-between (eg, `005-qux.md`). +* Use the slug of the title as the filename. For example, if the title is "Getting Started with Drill", name the file `...-getting-started-with-drill.md`. If you're not sure what the slug is, you should be able to see it in the URL and then adjust (the URLs are auto-generated based on the title attribute). http://git-wip-us.apache.org/repos/asf/drill-site/blob/c4de0f83/architecture/index.html ---------------------------------------------------------------------- diff --git a/architecture/index.html b/architecture/index.html new file mode 100644 index 0000000..a149adb --- /dev/null +++ b/architecture/index.html @@ -0,0 +1,219 @@ +<!DOCTYPE html> +<html> + +<head> + +<meta charset="UTF-8"> +<meta name=viewport content="width=device-width, initial-scale=1"> + + +<title>Architecture - Apache Drill</title> + +<link href="/css/syntax.css" rel="stylesheet" type="text/css"> +<link href="/css/style.css" rel="stylesheet" type="text/css"> +<link href="/css/arrows.css" rel="stylesheet" type="text/css"> +<link href="/css/breadcrumbs.css" rel="stylesheet" type="text/css"> +<link href="/css/code.css" rel="stylesheet" type="text/css"> +<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css"> +<link href="/css/responsive.css" rel="stylesheet" type="text/css"> + +<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> +<link rel="icon" href="/favicon.ico" type="image/x-icon"> + +<script language="javascript" type="text/javascript" src="/js/lib/jquery-1.11.1.min.js"></script> +<script language="javascript" type="text/javascript" src="/js/lib/jquery.easing.1.3.js"></script> +<script language="javascript" type="text/javascript" src="/js/modernizr.custom.js"></script> +<script language="javascript" type="text/javascript" src="/js/script.js"></script> +<script language="javascript" type="text/javascript" src="/js/drill.js"></script> + + +</head> + +<body onResize="resized();"> + <div class="page-wrap"> + <div class="bui"></div> + +<div id="menu" class="mw"> +<ul> + <li class='toc-categories'> + <a class="expand-toc-icon" href="javascript:void(0);"><i class="fa fa-bars"></i></a> + </li> + <li class="logo"><a href="/"></a></li> + <li class='expand-menu'> + <a href="javascript:void(0);"><span class='menu-text'>Menu</span><span class='expand-icon'><i class="fa fa-bars"></i></span></a> + </li> + <li class='clear-float'></li> + <li class="documentation-menu"> + <a href="/docs/">Documentation</a> + <ul> + + <li><a href="/docs/getting-started/">Getting Started</a></li> + + <li><a href="/docs/architecture/">Architecture</a></li> + + <li><a href="/docs/tutorials/">Tutorials</a></li> + + <li><a href="/docs/install-drill/">Install Drill</a></li> + + <li><a href="/docs/connect-a-data-source/">Connect a Data Source</a></li> + + <li><a href="/docs/odbc-jdbc-interfaces/">ODBC/JDBC Interfaces</a></li> + + <li><a href="/docs/query-data/">Query Data</a></li> + + <li><a href="/docs/sql-reference/">SQL Reference</a></li> + + <li><a href="/docs/data-sources-and-file-formats/">Data Sources and File Formats</a></li> + + <li><a href="/docs/develop-custom-functions/">Develop Custom Functions</a></li> + + <li><a href="/docs/manage-drill/">Manage Drill</a></li> + + <li><a href="/docs/developer-information/">Developer Information</a></li> + + <li><a href="/docs/release-notes/">Release Notes</a></li> + + <li><a href="/docs/sample-datasets/">Sample Datasets</a></li> + + <li><a href="/docs/archived-pages/">Archived Pages</a></li> + + <li><a href="/docs/progress-reports/">Progress Reports</a></li> + + <li><a href="/docs/project-bylaws/">Project Bylaws</a></li> + + </ul> + </li> + <li class='nav'> + <a href="/community-resources/">Community</a> + <ul> + <li><a href="/team/">Team</a></li> + <li><a href="/mailinglists/">Mailing Lists</a></li> + <li><a href="/community-resources/">Community Resources</a></li> + </ul> + </li> + <li class='nav'><a href="/faq/">FAQ</a></li> + <li class='nav'><a href="/blog/">Blog</a></li> + <li id="twitter-menu-item"><a href="https://twitter.com/apachedrill"; title="apachedrill on twitter" target="_blank"><img src="/images/twitter_32_26_white.png" alt="twitter logo" align="center"></a> </li> + <li class='search-bar'> + <form id="drill-search-form"> + <input type="text" placeholder="Search Apache Drill" id="drill-search-term" /> + <button type="submit"> + <i class="fa fa-search"></i> + </button> + </form> + </li> + <li class="d"> + <a href="/download/"> + <i class="fa fa-cloud-download"></i> Download + </a> + </li> +</ul> +</div> + + <div class="int_title"> +<h1>Architecture</h1> + <p>Apache Drill is a low latency distributed query engine for large-scale datasets, including structured and semi-structured/nested data. Inspired by Google's Dremel, Drill is designed to scale to several thousands of nodes and query petabytes of data at interactive speeds that BI/Analytics environments require.</p> +</div> + +<div class="int_text" align="left"> <h2>High-Level Architecture<h2> + + <p>Drill includes a distributed execution environment, purpose built for large-scale data processing. At the core of Apache Drill is the 'Drillbit' service, which is responsible for accepting requests from the client, processing the queries, and returning results to the client.</p> + <p>A Drillbit service can be installed and run on all of the required nodes in a Hadoop cluster to form a distributed cluster environment. When a Drillbit runs on each data node in the cluster, Drill can maximize data locality during query execution without moving data over the network or between nodes. Drill uses ZooKeeper to maintain cluster membership and health-check information.</p> + + <p>Note that though Drill works in a Hadoop cluster environment, Drill is not tied to Hadoop and can run in any distributed cluster environment. The only pre-requisite for Drill is Zookeeper.</p> + + <h2>Query Flow in Drill</h2> + + <p> + The following image represents the flow of a Drill query: + <img src="/images/arc-1.jpg" alt="The flow of a Drill query" width="550"> + The flow of a Drill query typically involves the following steps: + </p> + <ul class="num"> + <li>The Drill client issues a query. A Drill client is a JDBC, ODBC, command line interface or a REST API. Any Drillbit in the cluster can accept queries from the clients. There is no master-slave concept.</li> + <li>The Drillbit then parses the query, optimizes it, and generates a distributed query plan that is optimized for fast and efficient execution.</li> + <li>The Drillbit that accepts the query becomes the driving Drillbit node for the request. It gets a list of available Drillbit nodes in the cluster from ZooKeeper. The driving Drillbit determines the appropriate nodes to execute various query plan fragments to maximize data locality.</li> + <li> The Drillbit schedules the execution of query fragments on individual nodes according to the execution plan.</li> + <li>The individual nodes finish their execution and return data to the driving Drillbit.</li> + <li>The driving Drillbit streams results back to the client.</li> + </ul> + + <h2>Core Modules within a Drillbit</h2> + <p>The following image represents Drillbit components:</p> + + <img src="/images/arc-2.jpg" alt="Drillbit components" width="525"> + + <p>⢠RPC end point: Drill exposes a low overhead protobuf-based RPC protocol to communicate with the clients. Additionally, a C++ and Java API layers are also available for the client applications to interact with Drill. Clients can communicate to a specific Drillbit directly or go through a ZooKeeper quorum to discover the available Drillbits before submitting queries. It is recommended that the clients always go through ZooKeeper to shield clients from the intricacies of cluster management, such as the addition or removal of nodes.</p> + <p>⢠SQL parser: Drill uses Optiq, the open source framework, to parse incoming queries. The output of the parser component is a language agnostic, computer-friendly logical plan that represents the query.</p> + <p>⢠Optimizer: Drill uses various standard database optimizations such as rule based/cost based, as well as data locality and other optimization rules exposed by the storage engine to re-write and split the query. The output of the optimizer is a distributed physical query plan that represents the most efficient and fastest way to execute the query across different nodes in the cluster.</p> + <p>⢠Execution engine: Drill provides a MPP execution engine built to perform distributed query processing across the various nodes in the cluster. </p> + <p>⢠Storage plugin interfaces: Drill serves as a query layer on top of several data sources. Storage plugins in Drill represent the abstractions that Drill uses to interact with the data sources. Storage plugins provide Drill with the following information:</p> + + <p class="l1"> + ⢠Metadata available in the source<br> + ⢠Interfaces for Drill to read from and write to data sources<br> + ⢠Location of data and a set of optimization rules to help with efficient and faster execution of Drill queries on a specific data source + </p> + <p>In the context of Hadoop, Drill provides storage plugins for files and HBase/M7. Drill also integrates with Hive as a storage plugin since Hive provides a metadata abstraction layer on top of files, HBase/M7, and provides libraries to read data and operate on these sources (SerDes and UDFs).</p> + <p>When users query files and HBase/M7 with Drill, they can do it directly or go through Hive if they have metadata defined there. Drill integration with Hive is only for metadata. Drill does not invoke the Hive execution engine for any requests. </p> + <p>⢠Distributed cache: Drill uses a distributed cache to manage metadata (not the data) and configuration information across various nodes. Sample metadata information that is stored in the cache includes query plan fragments, intermediate state of the query execution, and statistics. Drill uses Infinispan as its cache technology.</p> + <h2>Architectural Highlights</h2> + + <p>The goal for Drill is to bring the SQL ecosystem and performance of the relational systems to Hadoop scale data WITHOUT compromising on the flexibility of Hadoop/NoSQL systems. There are several core architectural elements in Apache Drill that make it a highly flexible and efficient query engine.</p> + + <h3>Flexibility</h3> + <ul> + <li><p><strong>Dynamic schema discovery</strong>: Drill does not require schema or type specification for the data in order to start the query execution process. Instead, Drill starts processing the data in units called record-batches and discovers the schema on the fly during processing. Self-describing data formats such as Parquet, JSON, AVRO and NoSQL databases have schema specified as part of the data itself, which will be leveraged by Drill dynamically at the query time. Schema can change over the course of a Drill query, so all of the Drill operators are designed to reconfigure themselves when such schema changing events occur. + <li><p><strong>Flexible data model</strong>: Drill is purpose-built from ground up for complex/multi-structured data commonly seen in Hadoop/NoSQL applications such as social/mobile, clickstream, logs and sensor equipped IOT. From a user point of view, Drill allows access to nested data attributes, just like SQL columns, and provides intuitive extensions to easily operate on them. From an architecture point of view, Drill provides a flexible hierarchical columnar data model that can represent complex, highly dynamic and evolving data models and allows efficient processing of it without need to flatten or materialize it either design time or at execution time. Relational data in Drill is treated as a special or simplified case of complex/multi-structured data. + <li><p><strong>Decentralized metadata</strong>: Unlike other SQL-on-Hadoop technologies or any traditional relational database, Drill does not have a centralized metadata requirement. In order to query data through Drill, users do not need to create and manage tables/views in a metadata repository, or rely on a database administrator group for such a function. <br><br> + Drill metadata is derived from the storage plugins that correspond to data sources. Drill supports a varied set of storage plugins that provide a spectrum of metadata ranging from full metadata such as for Hive, partial metadata such as for HBase, or no central metadata such as for files. <br><br> + De-centralized metadata also means that Drill is NOT tied to a single Hive repository either. Users can query from multiple Hive repositories in a single query and then combine data with information from HBase tables or a file in the distributed file system. <br><br> + Users also have the ability to create metadata (tables/views/databases) within Drill using the SQL DDL syntax. De-centralized metadata is applicable during metadata creation. Drill allows persisting metadata in one of the underlying data sources. <br><br> + From a client access perspective, Drill metadata is organized just like a traditional DB <br>(Databases->Tables/Views->Columns). The metadata is accessible through the ANSI standard INFORMATION_SCHEMA database. + <li><p><strong>Extensibility</strong>: Drill provides an extensible architecture at all layers, including storage plugin layer, query layer, query optimization/execution, and client APIs. You can customize any layer for the specific needs of an organization or you can extend the layer to a broader array of use cases. <br><br> + Drill provides a built in classpath scanning and plugin concept to add additional storage plugins, functions, and operators with minimal configuration. + <br><br> + The following list provides a few examples of Drill's extensible architectural capabilities: + <ul> + <li>A high performance Java API to implement custom UDFs/UDAFs + <li>Ability to go beyond Hadoop by implementing custom storage plugins to other data sources such as Oracle/MySQL or NoSQL stores, such as Mongo or Cassandra + <li>An API to implement custom operators + <li>Support for direct execution of strongly specified JSON based logical and physical plans to help with the simplification of testing, and to enable integration of alternative query languages other than SQL. + </ul> + </ul> + + <h3>Performance</h3> + + <p>Drill is designed from the ground up for high performance on large datasets. Few core elements of Drill processing that help Drill achieve its performance include:</p> + <ul> + <li><p><strong>Distributed engine</strong>: Drill provides a powerful distributed execution engine for processing queries. Users can submit requests to any node in the cluster. You can simply add new nodes to the cluster to scale for larger volumes of data, support more users or to improve performance. + <li><p><strong>Columnar execution</strong>: Drill optimizes for both columnar storage and execution by using an in-memory data model that is hierarchical and columnar. When working with data stored in columnar formats such as Parquet, Drill avoids disk access for columns that are not involved in an analytic query. Drill also provides an execution layer that performs SQL processing directly on columnar data without row materialization. The combination of optimizations for columnar storage and direct columnar execution significantly lowers memory footprints and provides faster execution of BI/Analytic type of workloads. + <li><p><strong>Vectorization</strong>: Rather than operating on single values from a single table record at one time, vectorization in Drill allows the CPU to operate on vectors, referred to as a Record Batches. Record Batches are arrays of values from many different records. The technical basis for efficiency of vectorized processing is modern chip technology with deep-pipelined CPU designs. Keeping all pipelines full to achieve efficiency near peak performance is something impossible to achieve in traditional database engines, primarily due to code complexity. + <li><p><strong>Runtime compilation</strong>: + Runtime compilation is faster compared to the interpreted execution. Drill generates highly efficient custom code for every single query for every single operator. Here is a quick overview of the Drill compilation/code generation process at a glance. + <br><img src="/images/drillcompiler.png" width="500"> + <li><p><strong>Optimistic and pipelined query execution</strong>: Drill adopts an optimistic execution model to process queries. Drill assumes that failures are infrequent within the short span of a query and therefore does not spend time creating boundaries or checkpoints to minimize recovery time. Failures at node level are handled gracefully. In the instance of a single query failure, the query is rerun. Drill execution uses a pipeline model where all tasks are scheduled at once. The query execution happens in-memory as much as possible to move data through task pipelines, persisting to disk only if there is memory overflow. + </ul> +</div> + + </div> + <p class="push"></p> +<div id="footer" class="mw"> +<div class="wrapper"> +Copyright © 2012-2014 The Apache Software Foundation, licensed under the Apache License, Version 2.0.<br> +Apache and the Apache feather logo are trademarks of The Apache Software Foundation. Other names appearing on the site may be trademarks of their respective owners.<br/><br/> +</div> +</div> + + <script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + +ga('create', 'UA-53379651-1', 'auto'); +ga('send', 'pageview'); +</script> + +</body> +</html> http://git-wip-us.apache.org/repos/asf/drill-site/blob/c4de0f83/blog/2014/11/19/sql-on-mongodb/index.html ---------------------------------------------------------------------- diff --git a/blog/2014/11/19/sql-on-mongodb/index.html b/blog/2014/11/19/sql-on-mongodb/index.html new file mode 100644 index 0000000..83573bb --- /dev/null +++ b/blog/2014/11/19/sql-on-mongodb/index.html @@ -0,0 +1,323 @@ +<!DOCTYPE html> +<html> + +<head> + +<meta charset="UTF-8"> +<meta name=viewport content="width=device-width, initial-scale=1"> + + +<title>SQL on MongoDB - Apache Drill</title> + +<link href="/css/syntax.css" rel="stylesheet" type="text/css"> +<link href="/css/style.css" rel="stylesheet" type="text/css"> +<link href="/css/arrows.css" rel="stylesheet" type="text/css"> +<link href="/css/breadcrumbs.css" rel="stylesheet" type="text/css"> +<link href="/css/code.css" rel="stylesheet" type="text/css"> +<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css"> +<link href="/css/responsive.css" rel="stylesheet" type="text/css"> + +<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> +<link rel="icon" href="/favicon.ico" type="image/x-icon"> + +<script language="javascript" type="text/javascript" src="/js/lib/jquery-1.11.1.min.js"></script> +<script language="javascript" type="text/javascript" src="/js/lib/jquery.easing.1.3.js"></script> +<script language="javascript" type="text/javascript" src="/js/modernizr.custom.js"></script> +<script language="javascript" type="text/javascript" src="/js/script.js"></script> +<script language="javascript" type="text/javascript" src="/js/drill.js"></script> + + +</head> + +<body onResize="resized();"> + <div class="page-wrap"> + <div class="bui"></div> + +<div id="menu" class="mw"> +<ul> + <li class='toc-categories'> + <a class="expand-toc-icon" href="javascript:void(0);"><i class="fa fa-bars"></i></a> + </li> + <li class="logo"><a href="/"></a></li> + <li class='expand-menu'> + <a href="javascript:void(0);"><span class='menu-text'>Menu</span><span class='expand-icon'><i class="fa fa-bars"></i></span></a> + </li> + <li class='clear-float'></li> + <li class="documentation-menu"> + <a href="/docs/">Documentation</a> + <ul> + + <li><a href="/docs/getting-started/">Getting Started</a></li> + + <li><a href="/docs/architecture/">Architecture</a></li> + + <li><a href="/docs/tutorials/">Tutorials</a></li> + + <li><a href="/docs/install-drill/">Install Drill</a></li> + + <li><a href="/docs/connect-a-data-source/">Connect a Data Source</a></li> + + <li><a href="/docs/odbc-jdbc-interfaces/">ODBC/JDBC Interfaces</a></li> + + <li><a href="/docs/query-data/">Query Data</a></li> + + <li><a href="/docs/sql-reference/">SQL Reference</a></li> + + <li><a href="/docs/data-sources-and-file-formats/">Data Sources and File Formats</a></li> + + <li><a href="/docs/develop-custom-functions/">Develop Custom Functions</a></li> + + <li><a href="/docs/manage-drill/">Manage Drill</a></li> + + <li><a href="/docs/developer-information/">Developer Information</a></li> + + <li><a href="/docs/release-notes/">Release Notes</a></li> + + <li><a href="/docs/sample-datasets/">Sample Datasets</a></li> + + <li><a href="/docs/archived-pages/">Archived Pages</a></li> + + <li><a href="/docs/progress-reports/">Progress Reports</a></li> + + <li><a href="/docs/project-bylaws/">Project Bylaws</a></li> + + </ul> + </li> + <li class='nav'> + <a href="/community-resources/">Community</a> + <ul> + <li><a href="/team/">Team</a></li> + <li><a href="/mailinglists/">Mailing Lists</a></li> + <li><a href="/community-resources/">Community Resources</a></li> + </ul> + </li> + <li class='nav'><a href="/faq/">FAQ</a></li> + <li class='nav'><a href="/blog/">Blog</a></li> + <li id="twitter-menu-item"><a href="https://twitter.com/apachedrill"; title="apachedrill on twitter" target="_blank"><img src="/images/twitter_32_26_white.png" alt="twitter logo" align="center"></a> </li> + <li class='search-bar'> + <form id="drill-search-form"> + <input type="text" placeholder="Search Apache Drill" id="drill-search-term" /> + <button type="submit"> + <i class="fa fa-search"></i> + </button> + </form> + </li> + <li class="d"> + <a href="/download/"> + <i class="fa fa-cloud-download"></i> Download + </a> + </li> +</ul> +</div> + + <div class="post int_text"> + + <header class="post-header"> + <div class="int_title"> + <h1 class="post-title">SQL on MongoDB</h1> + </div> + <p class="post-meta"> + + <strong>Authors:</strong> + <ul> + + + <li>Anil Kumar Batchu (SSE, Intuit India (IDC))</li> + + + <li>Kamesh Bhallamudi (SSE, Intuit India (IDC))</li> + + </ul> + +<br/><strong>Date:</strong> Nov 19, 2014 +</p> + </header> + <div class="addthis_sharing_toolbox"></div> + + <article class="post-content"> + <p>One of the many interesting and unique features about Drill is its ability to support multiple query languages, data formats, and data sources, as well as cross-platform querying (such as joining HBase tables with your Mongo collections) using ANSI SQL.</p> + +<p>As of now, Drill supports multiple storage plugins, including HDFS, HBase, Hive, and LocalFileSystem. Since Drill is optimized for nested data, we realized that a Mongo storage plugin would be a useful feature.</p> + +<p>So, recently Kamesh and I (we have an avid interest in all things Mongo) contributed the MongoDB storage plugin feature to the Apache Drill project. As part of this blog post, we would like to provide instructions on how to use this plugin, which has been included in the <a href="http://incubator.apache.org/drill/download/";>Drill 0.6 release</a>.</p> + +<p>The instructions are divided into the following subtopics:</p> + +<ul> +<li>Drill and Mongo setup (standalone/replicated/sharded)</li> +<li>Running queries</li> +<li>Securely accessing MongoDB</li> +<li>Optimizations</li> +</ul> + +<h2 id="drill-and-mongodb-setup-(standalone/replicated/sharded)">Drill and MongoDB Setup (Standalone/Replicated/Sharded)</h2> + +<h3 id="standalone">Standalone</h3> + +<ul> +<li>Start <code>mongod</code> process (<a href="http://docs.mongodb.org/manual/installation/";>Install MongoDB</a>)</li> +<li>Start Drill in embedded mode (<a href="https://cwiki.apache.org/confluence/display/DRILL/Installing+Drill+in+Embedded+Mode";>Installing Drill in Embedded Mode</a> & <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=44994063";>Starting/Stopping Drill</a>) </li> +<li>Access the Web UI through the local drillbit: <a href="http://localhost:8047/";>http://localhost:8047/</a></li> +<li><p>Enable the Mongo storage plugin and update its configuration:</p> +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span> + <span class="nt">"type"</span><span class="p">:</span> <span class="s2">"mongo"</span><span class="p">,</span> + <span class="nt">"connection"</span><span class="p">:</span> <span class="s2">"mongodb://localhost:27017"</span><span class="p">,</span> + <span class="nt">"enabled"</span><span class="p">:</span> <span class="kc">true</span> +<span class="p">}</span> +</code></pre></div></li> +</ul> + +<p>By default, <code>mongod</code> listens on port 27017.</p> + +<p><img src="/static/sql-on-mongodb/standalone.png" alt="Drill on MongoDB in standalone mode"></p> + +<h3 id="replica-set">Replica Set</h3> + +<ul> +<li>Start <code>mongod</code> processes in replication mode</li> +<li>Start Drill in distributed mode (<a href="https://cwiki.apache.org/confluence/display/DRILL/Installing+Drill+in+Distributed+Mode";>Installing Drill in Distributed Mode</a> & <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=44994063";>Starting/Stopping Drill</a>)</li> +<li>Access the Web UI through any drillbit: <a href="http://drillbit2:8047";>http://drillbit2:8047</a></li> +<li><p>Enable the Mongo storage plugin and update its configuration:</p> +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span> + <span class="nt">"type"</span><span class="p">:</span> <span class="s2">"mongo"</span><span class="p">,</span> + <span class="nt">"connection"</span><span class="p">:</span> <span class="s2">"mongodb://<host1>:<port1>,<host2>:<port2>"</span><span class="p">,</span> + <span class="nt">"enabled"</span><span class="p">:</span> <span class="kc">true</span> +<span class="p">}</span> +</code></pre></div> +<p>Where <code>host1</code> and <code>host2</code> are <code>mongod</code> hostnames in the replica set.</p></li> +</ul> + +<p><img src="/static/sql-on-mongodb/replicated.png" alt="Drill on MongoDB in replicated mode"></p> + +<p>In replicated mode, whichever drillbit receives the query connects to the nearest <code>mongod</code> (local <code>mongod</code>) to read the data.</p> + +<h3 id="sharded/sharded-with-replica-set">Sharded/Sharded with Replica Set</h3> + +<ul> +<li>Start Mongo processes in sharded mode</li> +<li>Start Drill in distributed mode (<a href="https://cwiki.apache.org/confluence/display/DRILL/Installing+Drill+in+Distributed+Mode";>Installing Drill in Distributed Mode</a> & <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=44994063";>Starting/Stopping Drill</a>)</li> +<li>Access the Web UI through any drillbit: <a href="http://drillbit3:8047";>http://drillbit3:8047</a></li> +<li><p>Enable the Mongo storage plugin and update its configuration:</p> +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span> + <span class="nt">"type"</span><span class="p">:</span> <span class="s2">"mongo"</span><span class="p">,</span> + <span class="nt">"connection"</span><span class="p">:</span> <span class="s2">"mongodb://<host1>:<port1>,<host2>:<port2>"</span><span class="p">,</span> + <span class="nt">"enabled"</span><span class="p">:</span> <span class="kc">true</span> +<span class="p">}</span> +</code></pre></div> +<p>Where <code>host1</code> and <code>host2</code> are the <code>mongos</code> hostnames.</p></li> +</ul> + +<p><img src="/static/sql-on-mongodb/sharded.png" alt="Drill on MongoDB in sharded mode"></p> + +<p>In sharded mode, drillbit first connects to the <code>mongos</code> server to get the shard information.</p> + +<h2 id="endpoint-assignments">Endpoint Assignments</h2> + +<p>Drill is designed to maximize data locality:</p> + +<ul> +<li>When drillbits and shards run together on the same machines, each drillbit (endpoint) will read the chunks from the local shard. That is, all the chunks from a shard will be assigned to its local drillbit. This is known as data locality, and is the ideal scenario.</li> +<li>When all drillbits and shards are running on different machines, chunks will be assigned to drillbits in a round-robin fashion. In this case there is no data locality.</li> +<li>When some of drillbits and shards are colocated, and some of them are running on different machines, partial data locality is achieved.</li> +</ul> + +<h2 id="running-queries">Running Queries</h2> + +<p>Here is a simple exercise that provides steps for creating an <code>empinfo</code> collection in an <code>employee</code> database in Mongo that you can query using Drill:</p> + +<ol> +<li>Download <a href="http://media.mongodb.org/zips.json";>zips.json</a> and the <a href="/static/sql-on-mongodb/empinfo.json">empinfo.json</a> dataset referenced at the end of blog.</li> +<li><p>Import the zips.json and empinfo.json files into Mongo using the following command: </p> +<div class="highlight"><pre><code class="language-bash" data-lang="bash">mongoimport --host localhost --db <span class="nb">test</span> --collection zips < zips.json +mongoimport --host localhost --db employee --collection empinfo < empinfo.json +</code></pre></div></li> +<li><p>Issue the following queries either from sqlline (Drillâs shell) or from the Drill Web UI to get corresponding results from the Mongo collection. </p> + +<ul> +<li>To issue queries from the web UI, open the Drill web UI and go to Query tab. </li> +<li><p>To issue queries from sqlline, connect to sqlline using the following command: </p> +<div class="highlight"><pre><code class="language-bash" data-lang="bash"><DRILLHOME>/bin/sqlline -u jdbc:drill:zk<span class="o">=</span>zkhost:2181 -n admin -p admin +</code></pre></div></li> +</ul></li> +<li><p>Queries:</p> +<div class="highlight"><pre><code class="language-sql" data-lang="sql"><span class="k">SELECT</span> <span class="n">first_name</span><span class="p">,</span> <span class="n">last_name</span><span class="p">,</span> <span class="n">position_id</span> +<span class="k">FROM</span> <span class="n">mongo</span><span class="p">.</span><span class="n">employee</span><span class="p">.</span><span class="o">`</span><span class="n">empinfo</span><span class="o">`</span> +<span class="k">WHERE</span> <span class="n">employee_id</span> <span class="o">=</span> <span class="mi">1107</span> <span class="k">AND</span> <span class="n">position_id</span> <span class="o">=</span> <span class="mi">17</span> <span class="k">AND</span> <span class="n">last_name</span> <span class="o">=</span> <span class="s1">'Yonce'</span><span class="p">;</span> + +<span class="k">SELECT</span> <span class="n">city</span><span class="p">,</span> <span class="k">sum</span><span class="p">(</span><span class="n">pop</span><span class="p">)</span> +<span class="k">FROM</span> <span class="n">mongo</span><span class="p">.</span><span class="n">test</span><span class="p">.</span><span class="o">`</span><span class="n">zips</span><span class="o">`</span> <span class="n">zipcodes</span> +<span class="k">WHERE</span> <span class="k">state</span> <span class="k">IS</span> <span class="k">NOT</span> <span class="k">NULL</span> <span class="k">GROUP</span> <span class="k">BY</span> <span class="n">city</span> +<span class="k">ORDER</span> <span class="k">BY</span> <span class="k">sum</span><span class="p">(</span><span class="n">pop</span><span class="p">)</span> <span class="k">DESC</span> <span class="k">LIMIT</span> <span class="mi">1</span><span class="p">;</span> +</code></pre></div></li> +</ol> + +<p><em>Note</em>: If a field contains a mixture of different data types across different records, such as both int and decimal values, then queries fail unless <code>store.mongo.all_text_mode = true</code> and aggregations fail in that case. For more information refer to <a href="https://issues.apache.org/jira/browse/DRILL-1475";>DRILL-1475</a> and <a href="https://issues.apache.org/jira/browse/DRILL-1460";>DRILL-1460</a>.</p> + +<p>To set <code>store.mongo.all_text_mode = true</code>, execute the following command in sqlline:</p> +<div class="highlight"><pre><code class="language-sql" data-lang="sql"><span class="k">alter</span> <span class="k">session</span> <span class="k">set</span> <span class="n">store</span><span class="p">.</span><span class="n">mongo</span><span class="p">.</span><span class="n">all_text_mode</span> <span class="o">=</span> <span class="k">true</span> +</code></pre></div> +<h2 id="securely-accessing-mongodb">Securely Accessing MongoDB</h2> + +<p>Create two databases, emp and zips. For each database, create a user with read privileges. As an example, for the zips database, create a user âapacheâ with read privileges. For the emp database, create a user âdrillâ with read privileges.</p> + +<p>The apache user will be able to query the zips database by using the following storage plugin configuration:</p> +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span> + <span class="nt">"type"</span><span class="p">:</span> <span class="s2">"mongo"</span><span class="p">,</span> + <span class="nt">"connection"</span><span class="p">:</span> <span class="s2">"mongodb://apache:apache@localhost:27017/zips"</span><span class="p">,</span> + <span class="nt">"enabled"</span><span class="p">:</span> <span class="kc">true</span> +<span class="p">}</span> +</code></pre></div> +<p>The <code>drill</code> user will be able to query the <code>emp</code> database by using the following storage plugin configuration:</p> +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span> + <span class="nt">"type"</span><span class="p">:</span> <span class="s2">"mongo"</span><span class="p">,</span> + <span class="nt">"connection"</span><span class="p">:</span> <span class="s2">"mongodb://drill:drill@localhost:27017/emp"</span><span class="p">,</span> + <span class="nt">"enabled"</span><span class="p">:</span> <span class="kc">true</span> +<span class="p">}</span> +</code></pre></div> +<p><em>Note</em>: The security patch may be included in next release. Check <a href="https://issues.apache.org/jira/browse/DRILL-1502";>DRILL-1502</a> for status.</p> + +<h2 id="optimizations">Optimizations</h2> + +<p>The MongoDB storage plugin supports predicate pushdown and projection pushdown. As of now, predicate pushdown is implemented for the following filters: <code>></code>, <code>>=</code>, <code><</code>, <code><=</code>, <code>==</code>, <code>!=</code>, <code>isNull</code> and <code>isNotNull</code>.</p> + +<p>We are excited about the release of the MongoDB storage plugin, and we believe that Drill is the perfect SQL query tool for MongoDB.</p> + + </article> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'drill'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> + +</div> +<script type="text/javascript" src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-548b2caa33765e8d" async="async"></script> + + </div> + <p class="push"></p> +<div id="footer" class="mw"> +<div class="wrapper"> +Copyright © 2012-2014 The Apache Software Foundation, licensed under the Apache License, Version 2.0.<br> +Apache and the Apache feather logo are trademarks of The Apache Software Foundation. Other names appearing on the site may be trademarks of their respective owners.<br/><br/> +</div> +</div> + + <script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + +ga('create', 'UA-53379651-1', 'auto'); +ga('send', 'pageview'); +</script> + +</body> +</html> http://git-wip-us.apache.org/repos/asf/drill-site/blob/c4de0f83/blog/2014/12/02/drill-top-level-project/index.html ---------------------------------------------------------------------- diff --git a/blog/2014/12/02/drill-top-level-project/index.html b/blog/2014/12/02/drill-top-level-project/index.html new file mode 100644 index 0000000..6a97da5 --- /dev/null +++ b/blog/2014/12/02/drill-top-level-project/index.html @@ -0,0 +1,211 @@ +<!DOCTYPE html> +<html> + +<head> + +<meta charset="UTF-8"> +<meta name=viewport content="width=device-width, initial-scale=1"> + + +<title>Apache Drill Graduates to a Top-Level Project - Apache Drill</title> + +<link href="/css/syntax.css" rel="stylesheet" type="text/css"> +<link href="/css/style.css" rel="stylesheet" type="text/css"> +<link href="/css/arrows.css" rel="stylesheet" type="text/css"> +<link href="/css/breadcrumbs.css" rel="stylesheet" type="text/css"> +<link href="/css/code.css" rel="stylesheet" type="text/css"> +<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css"> +<link href="/css/responsive.css" rel="stylesheet" type="text/css"> + +<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> +<link rel="icon" href="/favicon.ico" type="image/x-icon"> + +<script language="javascript" type="text/javascript" src="/js/lib/jquery-1.11.1.min.js"></script> +<script language="javascript" type="text/javascript" src="/js/lib/jquery.easing.1.3.js"></script> +<script language="javascript" type="text/javascript" src="/js/modernizr.custom.js"></script> +<script language="javascript" type="text/javascript" src="/js/script.js"></script> +<script language="javascript" type="text/javascript" src="/js/drill.js"></script> + + +</head> + +<body onResize="resized();"> + <div class="page-wrap"> + <div class="bui"></div> + +<div id="menu" class="mw"> +<ul> + <li class='toc-categories'> + <a class="expand-toc-icon" href="javascript:void(0);"><i class="fa fa-bars"></i></a> + </li> + <li class="logo"><a href="/"></a></li> + <li class='expand-menu'> + <a href="javascript:void(0);"><span class='menu-text'>Menu</span><span class='expand-icon'><i class="fa fa-bars"></i></span></a> + </li> + <li class='clear-float'></li> + <li class="documentation-menu"> + <a href="/docs/">Documentation</a> + <ul> + + <li><a href="/docs/getting-started/">Getting Started</a></li> + + <li><a href="/docs/architecture/">Architecture</a></li> + + <li><a href="/docs/tutorials/">Tutorials</a></li> + + <li><a href="/docs/install-drill/">Install Drill</a></li> + + <li><a href="/docs/connect-a-data-source/">Connect a Data Source</a></li> + + <li><a href="/docs/odbc-jdbc-interfaces/">ODBC/JDBC Interfaces</a></li> + + <li><a href="/docs/query-data/">Query Data</a></li> + + <li><a href="/docs/sql-reference/">SQL Reference</a></li> + + <li><a href="/docs/data-sources-and-file-formats/">Data Sources and File Formats</a></li> + + <li><a href="/docs/develop-custom-functions/">Develop Custom Functions</a></li> + + <li><a href="/docs/manage-drill/">Manage Drill</a></li> + + <li><a href="/docs/developer-information/">Developer Information</a></li> + + <li><a href="/docs/release-notes/">Release Notes</a></li> + + <li><a href="/docs/sample-datasets/">Sample Datasets</a></li> + + <li><a href="/docs/archived-pages/">Archived Pages</a></li> + + <li><a href="/docs/progress-reports/">Progress Reports</a></li> + + <li><a href="/docs/project-bylaws/">Project Bylaws</a></li> + + </ul> + </li> + <li class='nav'> + <a href="/community-resources/">Community</a> + <ul> + <li><a href="/team/">Team</a></li> + <li><a href="/mailinglists/">Mailing Lists</a></li> + <li><a href="/community-resources/">Community Resources</a></li> + </ul> + </li> + <li class='nav'><a href="/faq/">FAQ</a></li> + <li class='nav'><a href="/blog/">Blog</a></li> + <li id="twitter-menu-item"><a href="https://twitter.com/apachedrill"; title="apachedrill on twitter" target="_blank"><img src="/images/twitter_32_26_white.png" alt="twitter logo" align="center"></a> </li> + <li class='search-bar'> + <form id="drill-search-form"> + <input type="text" placeholder="Search Apache Drill" id="drill-search-term" /> + <button type="submit"> + <i class="fa fa-search"></i> + </button> + </form> + </li> + <li class="d"> + <a href="/download/"> + <i class="fa fa-cloud-download"></i> Download + </a> + </li> +</ul> +</div> + + <div class="post int_text"> + + <header class="post-header"> + <div class="int_title"> + <h1 class="post-title">Apache Drill Graduates to a Top-Level Project</h1> + </div> + <p class="post-meta"> + + + + <strong>Author:</strong> Tomer Shiran (Founder, PMC Member and Committer, Apache Drill) + +<br/><strong>Date:</strong> Dec 2, 2014 +</p> + </header> + <div class="addthis_sharing_toolbox"></div> + + <article class="post-content"> + <p>The Apache Software Foundation has just announced that it has promoted Drill to a top-level project at Apache, similar to other well-known projects like Apache Hadoop and httpd (the world's most popular Web server). This marks a significant accomplishment for the Drill community, and I wanted to personally thank everyone who has contributed to the project. It takes many people, and countless hours, to develop something as complex and innovative as Drill.</p> + +<p>In this post I wanted to reflect on the past and future of Drill.</p> + +<h2 id="why-we-started-drill">Why We Started Drill</h2> + +<h3 id="the-evolution-of-application-development-and-data">The Evolution of Application Development and Data</h3> + +<p>Over the last decade, organizations have been striving to become more agile and data-driven, seeking to gain competitive advantage in their markets. This trend has led to dramatic changes in the way applications are built and delivered, and in the type and volume of data that is being leveraged.</p> + +<p><strong>Applications</strong>: In previous decades, software development was a carefully orchestrated and planned process. The release cycles were often measured in years, and upgrades were infrequent. Today, Web and mobile applications are developed in a much more iterative fashion. The release cycles are measured in days or weeks, and upgrades are a non-issue. (What version of Salesforce.com or Google Maps are you using?)</p> + +<p><strong>Data</strong>: In previous decades, data was measured in MBs or GBs, and it was highly structured and denormalized. Today's data is often measured in TBs or PBs, and it tends to be multi-structured â a combination of unstructured, semi-structured and structured. The data comes from many different sources, including a variety of applications, devices and services, and its structure changes much more frequently.</p> + +<h3 id="a-new-generation-of-datastores">A New Generation of Datastores</h3> + +<p>The relational database, which was invented in 1970, was not designed for these new processes and data volumes and structures. As a result, a new generation of datastores has emerged, including HDFS, NoSQL (HBase, MongoDB, etc.) and search (Elasticsearch, Solr). These systems are schema-free (also known as "dynamic schema"). Applications, as opposed to DBAs, control the data structure, enabling more agility and flexibility. For example, an application developer can independently evolve the data structure with each application release (which could be daily or weekly) without filing a ticket with IT and waiting for the schema of the databae to be modified.</p> + +<h2 id="the-need-for-a-new-query-engine">The Need for a New Query Engine</h2> + +<p>With data increasingly being stored in schema-free datastores (HDFS, HBase, MongoDB, etc.) and a variety of cloud services, users need a way to explore and analyze this data, and a way to visualize it with BI tools (reports, dashboards, etc.). In 2012 we decided to embark on a journey to create the world's next-generation SQL engine. We had several high-level requirements in mind:</p> + +<ul> +<li><strong>A schema-free data model.</strong> Schema-free datastores (HDFS, NoSQL, search) need a schema-free SQL engine. These datastores became popular for a reason, and we shouldn't expect organizations to sacrifice those advantages in order to enjoy SQL-based analytics and BI. Today's organizations need agility and flexibility to cope with the volume, variety and velocity associated with modern applications and data.<br></li> +<li><strong>A standalone query engine that supports multiple data sources.</strong> Most companies now use a variety of best-of-breed datastores and services to store data. This is true not just for large Global 2000 companies, but also for small startups. For example, it is not uncommon for a startup to have data in MySQL, MongoDB, HBase and HDFS, as well as a variety of online services. ETL was hard even 10 years ago when data was static and 100x smaller than it is today, and in today's era of Big Data it's often impractical or impossible to ingest all the data into a single system.</li> +<li><strong>Ease of use.</strong> The SQL engine can't be hard to setup and use. Analysts and developers should be able to download and use it without deploying any complex infrastructure such as Hadoop.</li> +<li><strong>Scalability and performance.</strong> The SQL engine must support interactive queries. It can't be batch-oriented like Hive. In addition, it must be able to scale linearly from a small laptop or virtual machine to a large cluster with hundreds or thousands of powerful servers.</li> +</ul> + +<p>With these requirements in mind, we decided to incubate a new project in 2012 in the Apache Software Foundation so that a community of vendors and developers could come together and develop the technology. (One little known fact is that the name "Drill" was actually suggested by Google engineers due to its inspiration from Google's Dremel execution engine.)</p> + +<p>After almost two years of research and development, we released Drill 0.4 in August, and continued with monthly releases since then.</p> + +<h2 id="what's-next">What's Next</h2> + +<p>Graduating to a top-level project is a significant milestone, but it's really just the beginning of the journey. In fact, we're currently wrapping up Drill 0.7, which includes hundreds of fixes and enhancements, and we expect to release that in the next couple weeks.</p> + +<p>Drill is currently being used by dozens of organizations, ranging from small startups to some of the largest Fortune 100s. These organizations are already gaining tremendous business value with Drill. As we march towards a 1.0 release early next year, these organizations are helping us shape the project and ensure that it meets the needs of a broad range of organizations as well as users (business analysts, technical analysts, data scientists and application developers). I would like to encourage you to join the ride today by <a href="http://drill.apache.org/download/";>downloading Drill</a> and <a href="mailto:[email protected]";>letting us know</a> what you think.</p> + +<p>Happy Drilling!<br> +Tomer Shiran</p> + + </article> + <div id="disqus_thread"></div> + <script type="text/javascript"> + /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ + var disqus_shortname = 'drill'; // required: replace example with your forum shortname + + /* * * DON'T EDIT BELOW THIS LINE * * */ + (function() { + var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; + dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; + (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); + })(); + </script> + <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> + +</div> +<script type="text/javascript" src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-548b2caa33765e8d" async="async"></script> + + </div> + <p class="push"></p> +<div id="footer" class="mw"> +<div class="wrapper"> +Copyright © 2012-2014 The Apache Software Foundation, licensed under the Apache License, Version 2.0.<br> +Apache and the Apache feather logo are trademarks of The Apache Software Foundation. Other names appearing on the site may be trademarks of their respective owners.<br/><br/> +</div> +</div> + + <script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + +ga('create', 'UA-53379651-1', 'auto'); +ga('send', 'pageview'); +</script> + +</body> +</html>
