http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/release-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/release-guide.html b/src/main/webapp/release-guide.html new file mode 100644 index 0000000..fb51368 --- /dev/null +++ b/src/main/webapp/release-guide.html @@ -0,0 +1,572 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>CarbonData</title> + <style> + + </style> + <!-- Bootstrap --> + + <link rel="stylesheet" href="css/bootstrap.min.css"> + <link href="css/style.css" rel="stylesheet"> + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="js/jquery.min.js"></script> + <script src="js/bootstrap.min.js"></script> + <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> + + +</head> +<body> +<header> + <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> + <div class="container"> + <div class="navbar-header"> + <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" + class="navbar-toggle collapsed" type="button"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="index.html" class="logo"> + <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> + </a> + </div> + <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> + <ul class="nav navbar-nav navbar-right navlist-custom"> + <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> + </li> + <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false"> Download <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; + target="_blank">Apache CarbonData 1.4.0</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; + target="_blank">Apache CarbonData 1.3.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; + target="_blank">Release Archive</a></li> + </ul> + </li> + <li><a href="documentation.html" class="active">Documentation</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; + target="_blank">Contributing to CarbonData</a></li> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; + target="_blank">Release Guide</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; + target="_blank">Project PMC and Committers</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; + target="_blank">CarbonData Meetups</a></li> + <li><a href="security.html">Apache CarbonData Security</a></li> + <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache + Jira</a></li> + <li><a href="videogallery.html">CarbonData Videos </a></li> + </ul> + </li> + <li class="dropdown"> + <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li> + <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> + + </li> + + </ul> + </div><!--/.nav-collapse --> + <div id="search-box"> + <form method="get" action="http://www.google.com/search"; target="_blank"> + <div class="search-block"> + <table border="0" cellpadding="0" width="100%"> + <tr> + <td style="width:80%"> + <input type="text" name="q" size=" 5" maxlength="255" value="" + class="search-input" placeholder="Search...." required/> + </td> + <td style="width:20%"> + <input type="submit" value="Search"/></td> + </tr> + <tr> + <td align="left" style="font-size:75%" colspan="2"> + <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> + <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> + </td> + </tr> + </table> + </div> + </form> + </div> + </div> + </nav> +</header> <!-- end Header part --> + +<div class="fixed-padding"></div> <!-- top padding with fixde header --> + +<section><!-- Dashboard nav --> + <div class="container-fluid q"> + <div class="col-sm-12 col-md-12 maindashboard"> + <div class="verticalnavbar"> + <nav class="b-sticky-nav"> + <div class="nav-scroller"> + <div class="nav__inner"> + <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> + <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> + <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> + <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> + <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> + <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> + <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> + <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> + <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> + </div> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> + <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> + <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> + <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> + <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> + </div> + + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> + <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> + <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> + <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> + <a class="b-nav__security nav__item" href="./security.html">Security</a> + <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> + </div> + </div> + <div class="navindicator"> + <div class="b-nav__intro navindicator__item"></div> + <div class="b-nav__quickstart navindicator__item"></div> + <div class="b-nav__uses navindicator__item"></div> + <div class="b-nav__docs navindicator__item"></div> + <div class="b-nav__datamap navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> + <div class="b-nav__api navindicator__item"></div> + <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__faq navindicator__item"></div> + <div class="b-nav__contri navindicator__item"></div> + <div class="b-nav__security navindicator__item"></div> + </div> + </nav> + </div> + <div class="mdcontent"> + <section> + <div style="padding:10px 15px;"> + <div id="viewpage" name="viewpage"> + <div class="row"> + <div class="col-sm-12 col-md-12"> + <div> +<h1> +<a id="apache-carbondata-release-guide" class="anchor" href="#apache-carbondata-release-guide" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Apache CarbonData Release Guide</h1> +<p>Apache CarbonData periodically declares and publishes releases.</p> +<p>Each release is executed by a <em>Release Manager</em>, who is selected among the CarbonData committers. +This document describes the process that the Release Manager follows to perform a release. Any +changes to this process should be discussed and adopted on the +<a href="mailto:[email protected]";>dev@ mailing list</a>.</p> +<p>Please remember that publishing software has legal consequences. This guide complements the +foundation-wide <a href="http://www.apache.org/dev/release.html"; target=_blank rel="nofollow">Product Release Policy</a> and <a href="http://www.apache.org/dev/release-distribution"; target=_blank rel="nofollow">Release +Distribution Policy</a>.</p> +<h2> +<a id="decide-to-release" class="anchor" href="#decide-to-release" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Decide to release</h2> +<p>Deciding to release and selecting a Release Manager is the first step of the release process. +This is a consensus-based decision of the entire community.</p> +<p>Anybody can propose a release on the dev@ mailing list, giving a solid argument and nominating a +committer as the Release Manager (including themselves). There's no formal process, no vote +requirements, and no timing requirements. Any objections should be resolved by consensus before +starting the release.</p> +<p><em>Checklist to proceed to next step:</em></p> +<ol> +<li>Community agrees to release</li> +<li>Community selects a Release Manager</li> +</ol> +<h2> +<a id="prepare-for-the-release" class="anchor" href="#prepare-for-the-release" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Prepare for the release</h2> +<p>Before your first release, you should perform one-time configuration steps. This will set up your +security keys for signing the artifacts and access release repository.</p> +<p>To prepare for each release, you should audit the project status in the Jira, and do necessary +bookkeeping. Finally, you should tag a release.</p> +<h3> +<a id="one-time-setup-instructions" class="anchor" href="#one-time-setup-instructions" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>One-time setup instructions</h3> +<h4> +<a id="gpg-key" class="anchor" href="#gpg-key" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>GPG Key</h4> +<p>You need to have a GPG key to sign the release artifacts. Please be aware of the ASF-wide +<a href="https://www.apache.org/dev/release-signing.html"; target=_blank rel="nofollow">release signing guidelines</a>. If you don't have +a GPG key associated with your Apache account, please create one according to the guidelines.</p> +<p>Determine your Apache GPG key and key ID, as follows:</p> +<pre><code>gpg --list-keys +</code></pre> +<p>This will list your GPG keys. One of these should reflect your Apache account, for example:</p> +<pre><code>pub 2048R/845E6689 2016-02-23 +uid Nomen Nescio <[email protected]> +sub 2048R/BA4D50BE 2016-02-23 +</code></pre> +<p>Here, the key ID is the 8-digit hex string in the <code>pub</code> line: <code>845E6689</code>.</p> +<p>Now, add your Apache GPG key to the CarbonData's <code>KEYS</code> file in <code>dev</code> and <code>release</code> repositories +at <code>dist.apache.org</code>. Follow the instructions listed at the top of these files.</p> +<p>Configure <code>git</code> to use this key when signing code by giving it your key ID, as follows:</p> +<pre><code>git config --global user.signingkey 845E6689 +</code></pre> +<p>You may drop the <code>--global</code> option if you'd prefer to use this key for the current repository only.</p> +<p>You may wish to start <code>gpg-agent</code> to unlock your GPG key only once using your passphrase. +Otherwise, you may need to enter this passphrase several times. The setup of <code>gpg-agent</code> varies +based on operating system, but may be something like this:</p> +<pre><code>eval $(gpg-agent --daemon --no-grab --write-env-file $HOME/.gpg-agent-info) +export GPG_TTY=$(tty) +export GPG_AGENT_INFO +</code></pre> +<h4> +<a id="access-to-apache-nexus" class="anchor" href="#access-to-apache-nexus" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Access to Apache Nexus</h4> +<p>Configure access to the <a href="https://repository.apache.org"; target=_blank rel="nofollow">Apache Nexus repository</a>, used for +staging repository and promote the artifacts to Maven Central.</p> +<ol> +<li>You log in with your Apache account.</li> +<li>Confirm you have appropriate access by finding <code>org.apache.carbondata</code> under <code>Staging Profiles</code>.</li> +<li>Navigate to your <code>Profile</code> (top right dropdown menu of the page).</li> +<li>Choose <code>User Token</code> from the dropdown, then click <code>Access User Token</code>. Copy a snippet of the +Maven XML configuration block.</li> +<li>Insert this snippet twice into your global Maven <code>settings.xml</code> file, typically <code>${HOME]/ .m2/settings.xml</code>. The end result should look like this, where <code>TOKEN_NAME</code> and <code>TOKEN_PASSWORD</code> +are your secret tokens:</li> +</ol> +<pre><code> <settings> + <servers> + <server> + <id>apache.releases.https</id> + <username>TOKEN_NAME</username> + <password>TOKEN_PASSWORD</password> + </server> + <server> + <id>apache.snapshots.https</id> + <username>TOKEN_NAME</username> + <password>TOKEN_PASSWORD</password> + </server> + </servers> + </settings> +</code></pre> +<h4> +<a id="create-a-new-version-in-jira" class="anchor" href="#create-a-new-version-in-jira" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Create a new version in Jira</h4> +<p>When contributors resolve an issue in Jira, they are tagging it with a release that will contain +their changes. With the release currently underway, new issues should be resolved against a +subsequent future release. Therefore, you should create a release item for this subsequent +release, as follows:</p> +<ol> +<li>In Jira, navigate to <code>CarbonData > Administration > Versions</code>.</li> +<li>Add a new release: choose the next minor version number compared to the one currently +underway, select today's date as the <code>Start Date</code>, and choose <code>Add</code>.</li> +</ol> +<h4> +<a id="triage-release-blocking-issues-in-jira" class="anchor" href="#triage-release-blocking-issues-in-jira" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Triage release-blocking issues in Jira</h4> +<p>There could be outstanding release-blocking issues, which should be triaged before proceeding to +build the release. We track them by assigning a specific <code>Fix Version</code> field even before the +issue is resolved.</p> +<p>The list of release-blocking issues is available at the <a href="https://issues.apache.org/jira/browse/CARBONDATA/?selectedTab=com.atlassian.jira.jira-projects-plugin:versions-panel"; rel="nofollow">version status page</a>. +Triage each unresolved issue with one of the following resolutions:</p> +<ul> +<li>If the issue has been resolved and Jira was not updated, resolve it accordingly.</li> +<li>If the issue has not been resolved and it is acceptable to defer until the next release, update +the <code>Fix Version</code> field to the new version you just created. Please consider discussing this +with stakeholders and the dev@ mailing list, as appropriate.</li> +<li>If the issue has not been resolved and it is not acceptable to release until it is fixed, the +release cannot proceed. Instead, work with the CarbonData community to resolve the issue.</li> +</ul> +<h4> +<a id="review-release-notes-in-jira" class="anchor" href="#review-release-notes-in-jira" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Review Release Notes in Jira</h4> +<p>Jira automatically generates Release Notes based on the <code>Fix Version</code> applied to the issues. +Release Notes are intended for CarbonData users (not CarbonData committers/contributors). You +should ensure that Release Notes are informative and useful.</p> +<p>Open the release notes from the <a href="https://issues.apache.org/jira/browse/CARBONDATA/?selectedTab=com.atlassian.jira.jira-projects-plugin:versions-panel"; rel="nofollow">version status page</a> +by choosing the release underway and clicking Release Notes.</p> +<p>You should verify that the issues listed automatically by Jira are appropriate to appear in the +Release Notes. Specifically, issues should:</p> +<ul> +<li>Be appropriate classified as <code>Bug</code>, <code>New Feature</code>, <code>Improvement</code>, etc.</li> +<li>Represent noteworthy user-facing changes, such as new functionality, backward-incompatible +changes, or performance improvements.</li> +<li>Have occurred since the previous release; an issue that was introduced and fixed between +releases should not appear in the Release Notes.</li> +<li>Have an issue title that makes sense when read on its own.</li> +</ul> +<p>Adjust any of the above properties to the improve clarity and presentation of the Release Notes.</p> +<h4> +<a id="verify-that-a-release-build-works" class="anchor" href="#verify-that-a-release-build-works" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Verify that a Release Build works</h4> +<p>Run <code>mvn clean install -Prelease</code> to ensure that the build processes that are specific to that +profile are in good shape.</p> +<p><em>Checklist to proceed to the next step:</em></p> +<ol> +<li>Release Manager's GPG key is published to <code>dist.apache.org</code>.</li> +<li>Release Manager's GPG key is configured in <code>git</code> configuration.</li> +<li>Release Manager has <code>org.apache.carbondata</code> listed under <code>Staging Profiles</code> in Nexus.</li> +<li>Release Manager's Nexus User Token is configured in <code>settings.xml</code>.</li> +<li>Jira release item for the subsequent release has been created.</li> +<li>There are no release blocking Jira issues.</li> +<li>Release Notes in Jira have been audited and adjusted.</li> +</ol> +<h3> +<a id="build-a-release" class="anchor" href="#build-a-release" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Build a release</h3> +<p>Use Maven release plugin to tag and build release artifacts, as follows:</p> +<pre><code>mvn release:prepare +</code></pre> +<p>Use Maven release plugin to stage these artifacts on the Apache Nexus repository, as follows:</p> +<pre><code>mvn release:perform +</code></pre> +<p>Review all staged artifacts. They should contain all relevant parts for each module, including +<code>pom.xml</code>, jar, test jar, source, etc. Artifact names should follow +<a href="https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.carbondata%22"; rel="nofollow">the existing format</a> +in which artifact name mirrors directory structure. Carefully review any new artifacts.</p> +<p>Close the staging repository on Nexus. When prompted for a description, enter "Apache CarbonData +x.x.x release".</p> +<h3> +<a id="stage-source-release-on-distapacheorg" class="anchor" href="#stage-source-release-on-distapacheorg" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Stage source release on dist.apache.org</h3> +<p>Copy the source release to dev repository on <code>dist.apache.org</code>.</p> +<ol> +<li>If you have not already, check out the section of the <code>dev</code> repository on <code>dist.apache.org</code> via Subversion. In a fresh directory:</li> +</ol> +<pre><code>svn co https://dist.apache.org/repos/dist/dev/carbondata +</code></pre> +<ol start="2"> +<li>Make a directory for the new release:</li> +</ol> +<pre><code>mkdir x.x.x +</code></pre> +<ol start="3"> +<li>Copy the CarbonData source distribution, hash, and GPG signature:</li> +</ol> +<pre><code>cp apache-carbondata-x.x.x-source-release.zip x.x.x +</code></pre> +<ol start="4"> +<li>Add and commit the files:</li> +</ol> +<pre><code>svn add x.x.x +svn commit +</code></pre> +<ol start="5"> +<li>Verify the files are <a href="https://dist.apache.org/repos/dist/dev/carbondata"; target=_blank rel="nofollow">present</a>.</li> +</ol> +<h3> +<a id="propose-a-pull-request-for-website-updates" class="anchor" href="#propose-a-pull-request-for-website-updates" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Propose a pull request for website updates</h3> +<p>The final step of building a release candidate is to propose a website pull request.</p> +<p>This pull request should update the following page with the new release:</p> +<ul> +<li><code>src/main/webapp/index.html</code></li> +<li><code>src/main/webapp/docs/latest/mainpage.html</code></li> +</ul> +<p><em>Checklist to proceed to the next step:</em></p> +<ol> +<li>Maven artifacts deployed to the staging repository of +<a href="https://repository.apache.org"; target=_blank rel="nofollow">repository.apache.org</a> +</li> +<li>Source distribution deployed to the dev repository of +<a href="https://dist.apache.org/repos/dist/dev/carbondata/"; target=_blank rel="nofollow">dist.apache.org</a> +</li> +<li>Website pull request to list the release.</li> +</ol> +<h2> +<a id="vote-on-the-release-candidate" class="anchor" href="#vote-on-the-release-candidate" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Vote on the release candidate</h2> +<p>Once you have built and individually reviewed the release candidate, please share it for the +community-wide review. Please review foundation-wide <a href="http://www.apache.org/foundation/voting.html"; target=_blank rel="nofollow">voting guidelines</a> +for more information.</p> +<p>Start the review-and-vote thread on the dev@ mailing list. Here's an email template; please +adjust as you see fit:</p> +<pre><code>From: Release Manager +To: [email protected] +Subject: [VOTE] Apache CarbonData Release x.x.x + +Hi everyone, +Please review and vote on the release candidate for the version x.x.x, as follows: + +[ ] +1, Approve the release +[ ] -1, Do not approve the release (please provide specific comments) + +The complete staging area is available for your review, which includes: +* JIRA release notes [1], +* the official Apache source release to be deployed to dist.apache.org [2], which is signed with the key with fingerprint FFFFFFFF [3], +* all artifacts to be deployed to the Maven Central Repository [4], +* source code tag "x.x.x" [5], +* website pull request listing the release [6]. + +The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes. + +Thanks, +Release Manager + +[1] link +[2] link +[3] https://dist.apache.org/repos/dist/dist/carbondata/KEYS +[4] link +[5] link +[6] link +</code></pre> +<p>If there are any issues found in the release candidate, reply on the vote thread to cancel the vote. +There?s no need to wait 72 hours. Proceed to the <code>Cancel a Release (Fix Issues)</code> step below and +address the problem. +However, some issues don?t require cancellation. +For example, if an issue is found in the website pull request, just correct it on the spot and the +vote can continue as-is.</p> +<p>If there are no issues, reply on the vote thread to close the voting. Then, tally the votes in a +separate email. Here?s an email template; please adjust as you see fit.</p> +<pre><code>From: Release Manager +To: [email protected] +Subject: [RESULT][VOTE] Apache CarbonData Release x.x.x + +I'm happy to announce that we have unanimously approved this release. + +There are XXX approving votes, XXX of which are binding: +* approver 1 +* approver 2 +* approver 3 +* approver 4 + +There are no disapproving votes. + +Thanks everyone! +</code></pre> +<p>While in incubation, the Apache Incubator PMC must also vote on each release, using the same +process as above. Start the review and vote thread on the <code>[email protected]</code> list.</p> +<p><em>Checklist to proceed to the final step:</em></p> +<ol> +<li>Community votes to release the proposed release</li> +<li>While in incubation, Apache Incubator PMC votes to release the proposed release</li> +</ol> +<h2> +<a id="cancel-a-release-fix-issues" class="anchor" href="#cancel-a-release-fix-issues" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Cancel a Release (Fix Issues)</h2> +<p>Any issue identified during the community review and vote should be fixed in this step.</p> +<p>To fully cancel a vote:</p> +<ul> +<li>Cancel the current release and verify the version is back to the correct SNAPSHOT:</li> +</ul> +<pre><code>mvn release:cancel +</code></pre> +<ul> +<li>Drop the release tag:</li> +</ul> +<pre><code>git tag -d x.x.x +git push --delete apache x.x.x +</code></pre> +<ul> +<li>Drop the staging repository on Nexus (<a href="https://repository.apache.org"; target=_blank rel="nofollow">repository.apache.org</a>)</li> +</ul> +<p>Verify the version is back to the correct SNAPSHOT.</p> +<p>Code changes should be proposed as standard pull requests and merged.</p> +<p>Once all issues have been resolved, you should go back and build a new release candidate with +these changes.</p> +<h2> +<a id="finalize-the-release" class="anchor" href="#finalize-the-release" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Finalize the release</h2> +<p>Once the release candidate has been reviewed and approved by the community, the release should be +finalized. This involves the final deployment of the release to the release repositories, +merging the website changes, and announce the release.</p> +<h3> +<a id="deploy-artifacts-to-maven-central-repository" class="anchor" href="#deploy-artifacts-to-maven-central-repository" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Deploy artifacts to Maven Central repository</h3> +<p>On Nexus, release the staged artifacts to Maven Central repository. In the <code>Staging Repositories</code> +section, find the relevant release candidate <code>orgapachecarbondata-XXX</code> entry and click <code>Release</code>.</p> +<h3> +<a id="deploy-source-release-to-distapacheorg" class="anchor" href="#deploy-source-release-to-distapacheorg" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Deploy source release to dist.apache.org</h3> +<p>Copy the source release from the <code>dev</code> repository to <code>release</code> repository at <code>dist.apache.org</code> +using Subversion.</p> +<h3> +<a id="merge-website-pull-request" class="anchor" href="#merge-website-pull-request" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Merge website pull request</h3> +<p>Merge the website pull request to list the release created earlier.</p> +<h3> +<a id="mark-the-version-as-released-in-jira" class="anchor" href="#mark-the-version-as-released-in-jira" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Mark the version as released in Jira</h3> +<p>In Jira, inside <a href="https://issues.apache.org/jira/plugins/servlet/project-config/CARBONDATA/versions"; target=_blank rel="nofollow">version management</a> +, hover over the current release and a settings menu will appear. Click <code>Release</code>, and select +today's state.</p> +<p><em>Checklist to proceed to the next step:</em></p> +<ol> +<li>Maven artifacts released and indexed in the +<a href="https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.carbondata%22"; rel="nofollow">Maven Central repository</a> +</li> +<li>Source distribution available in the release repository of +<a href="https://dist.apache.org/repos/dist/release/carbondata/"; target=_blank rel="nofollow">dist.apache.org</a> +</li> +<li>Website pull request to list the release merged</li> +<li>Release version finalized in Jira</li> +</ol> +<h2> +<a id="promote-the-release" class="anchor" href="#promote-the-release" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Promote the release</h2> +<p>Once the release has been finalized, the last step of the process is to promote the release +within the project and beyond.</p> +<h3> +<a id="apache-mailing-lists" class="anchor" href="#apache-mailing-lists" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Apache mailing lists</h3> +<p>Announce on the dev@ mailing list that the release has been finished.</p> +<p>Announce on the user@ mailing list that the release is available, listing major improvements and +contributions.</p> +<p>While in incubation, announce the release on the Incubator's general@ mailing list.</p> +<p><em>Checklist to declare the process completed:</em></p> +<ol> +<li>Release announced on the user@ mailing list.</li> +<li>Release announced on the Incubator's general@ mailing list.</li> +<li>Completion declared on the dev@ mailing list.</li> +</ol> +<script> +// Show selected style on nav item +$(function() { $('.b-nav__release').addClass('selected'); }); +</script> +</div> +</div> +</div> +</div> +<div class="doc-footer"> + <a href="#top" class="scroll-top">Top</a> +</div> +</div> +</section> +</div> +</div> +</div> +</section><!-- End systemblock part --> +<script src="js/custom.js"></script> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/s3-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/s3-guide.html b/src/main/webapp/s3-guide.html index d9edeaf..9042b19 100644 --- a/src/main/webapp/s3-guide.html +++ b/src/main/webapp/s3-guide.html @@ -22,6 +22,7 @@ <![endif]--> <script src="js/jquery.min.js"></script> <script src="js/bootstrap.min.js"></script> + <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> </head> @@ -67,7 +68,7 @@ target="_blank">Release Archive</a></li> </ul> </li> - <li><a href="mainpage.html" class="active">Documentation</a></li> + <li><a href="documentation.html" class="active">Documentation</a></li> <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Community <span class="caret"></span></a> @@ -152,14 +153,65 @@ <section><!-- Dashboard nav --> <div class="container-fluid q"> <div class="col-sm-12 col-md-12 maindashboard"> - <div class="row"> + <div class="verticalnavbar"> + <nav class="b-sticky-nav"> + <div class="nav-scroller"> + <div class="nav__inner"> + <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> + <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> + <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> + <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> + <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> + <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> + <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> + <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> + <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> + </div> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> + <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> + <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> + <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> + <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> + </div> + + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> + <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> + <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> + <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> + <a class="b-nav__security nav__item" href="./security.html">Security</a> + <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> + </div> + </div> + <div class="navindicator"> + <div class="b-nav__intro navindicator__item"></div> + <div class="b-nav__quickstart navindicator__item"></div> + <div class="b-nav__uses navindicator__item"></div> + <div class="b-nav__docs navindicator__item"></div> + <div class="b-nav__datamap navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> + <div class="b-nav__api navindicator__item"></div> + <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__faq navindicator__item"></div> + <div class="b-nav__contri navindicator__item"></div> + <div class="b-nav__security navindicator__item"></div> + </div> + </nav> + </div> + <div class="mdcontent"> <section> <div style="padding:10px 15px;"> <div id="viewpage" name="viewpage"> <div class="row"> <div class="col-sm-12 col-md-12"> <div> -<p>#S3 Guide (Alpha Feature 1.4.1)</p> +<h1> +<a id="s3-guide-alpha-feature-141" class="anchor" href="#s3-guide-alpha-feature-141" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>S3 Guide (Alpha Feature 1.4.1)</h1> <p>Object storage is the recommended storage format in cloud as it can support storing large data files. S3 APIs are widely used for accessing object stores. This can be used to store or retrieve data on Amazon cloud, Huawei Cloud(OBS) or on any other object @@ -168,7 +220,8 @@ Storing data in cloud is advantageous as there are no restrictions on the size o data and the data can be accessed from anywhere at any time. Carbondata can support any Object Storage that conforms to Amazon S3 API. Carbondata relies on Hadoop provided S3 filesystem APIs to access Object stores.</p> -<p>#Writing to Object Storage</p> +<h1> +<a id="writing-to-object-storage" class="anchor" href="#writing-to-object-storage" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Writing to Object Storage</h1> <p>To store carbondata files onto Object Store, <code>carbon.storelocation</code> property will have to be configured with Object Store path in CarbonProperties file.</p> <p>For example:</p> @@ -180,8 +233,9 @@ table DDL command.</p> <p>For example:</p> <pre><code>CREATE TABLE IF NOT EXISTS db1.table1(col1 string, col2 int) STORED AS carbondata LOCATION 's3a://mybucket/carbonstore' </code></pre> -<p>For more details on create table, Refer <a href="./data-management-on-carbondata.html#create-table">data-management-on-carbondata</a></p> -<p>#Authentication</p> +<p>For more details on create table, Refer <a href="ddl-of-carbondata.html#create-table">DDL of CarbonData</a></p> +<h1> +<a id="authentication" class="anchor" href="#authentication" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Authentication</h1> <p>Authentication properties will have to be configured to store the carbondata files on to S3 location.</p> <p>Authentication properties can be set in any of the following ways:</p> <ol> @@ -211,16 +265,19 @@ spark.hadoop.fs.s3a.access.key=456 <pre><code>sparkSession.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "123") sparkSession.sparkContext.hadoopConfiguration.set("fs.s3a.access.key","456") </code></pre> -<p>#Recommendations</p> +<h1> +<a id="recommendations" class="anchor" href="#recommendations" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Recommendations</h1> <ol> <li>Object Storage like S3 does not support file leasing mechanism(supported by HDFS) that is required to take locks which ensure consistency between concurrent operations therefore, it is -recommended to set the configurable lock path property(<a href="https://github.com/apache/carbondata/blob/master/docs/configuration-parameters.html#miscellaneous-configuration";>carbon.lock.path</a>) +recommended to set the configurable lock path property(<a href="./configuration-parameters.html#system-configuration">carbon.lock.path</a>) to a HDFS directory.</li> -<li>Concurrent data manipulation operations are not supported. Object stores follow eventual -consistency semantics, i.e., any put request might take some time to reflect when trying to list -.This behaviour causes not to ensure the data read is always consistent or latest.</li> +<li>Concurrent data manipulation operations are not supported. Object stores follow eventual consistency semantics, i.e., any put request might take some time to reflect when trying to list. This behaviour causes the data read is always not consistent or not the latest.</li> </ol> +<script> +// Show selected style on nav item +$(function() { $('.b-nav__s3').addClass('selected'); }); +</script> </div> </div> </div> @@ -236,4 +293,4 @@ consistency semantics, i.e., any put request might take some time to reflect whe </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> \ No newline at end of file +</html> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/sdk-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/sdk-guide.html b/src/main/webapp/sdk-guide.html index cc55dc0..95ae214 100644 --- a/src/main/webapp/sdk-guide.html +++ b/src/main/webapp/sdk-guide.html @@ -22,6 +22,7 @@ <![endif]--> <script src="js/jquery.min.js"></script> <script src="js/bootstrap.min.js"></script> + <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> </head> @@ -67,7 +68,7 @@ target="_blank">Release Archive</a></li> </ul> </li> - <li><a href="mainpage.html" class="active">Documentation</a></li> + <li><a href="documentation.html" class="active">Documentation</a></li> <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Community <span class="caret"></span></a> @@ -152,7 +153,57 @@ <section><!-- Dashboard nav --> <div class="container-fluid q"> <div class="col-sm-12 col-md-12 maindashboard"> - <div class="row"> + <div class="verticalnavbar"> + <nav class="b-sticky-nav"> + <div class="nav-scroller"> + <div class="nav__inner"> + <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> + <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> + <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> + <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> + <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> + <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> + <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> + <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> + <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> + </div> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> + <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> + <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> + <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> + <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> + </div> + + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> + <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> + <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> + <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> + <a class="b-nav__security nav__item" href="./security.html">Security</a> + <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> + </div> + </div> + <div class="navindicator"> + <div class="b-nav__intro navindicator__item"></div> + <div class="b-nav__quickstart navindicator__item"></div> + <div class="b-nav__uses navindicator__item"></div> + <div class="b-nav__docs navindicator__item"></div> + <div class="b-nav__datamap navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> + <div class="b-nav__api navindicator__item"></div> + <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__faq navindicator__item"></div> + <div class="b-nav__contri navindicator__item"></div> + <div class="b-nav__security navindicator__item"></div> + </div> + </nav> + </div> + <div class="mdcontent"> <section> <div style="padding:10px 15px;"> <div id="viewpage" name="viewpage"> @@ -502,6 +553,22 @@ public CarbonWriterBuilder taskNo(long taskNo); public CarbonWriterBuilder withLoadOptions(Map<String, String> options); </code></pre> <pre><code>/** + * To support the table properties for sdk writer + * + * @param options key,value pair of create table properties. + * supported keys values are + * a. blocksize -- [1-2048] values in MB. Default value is 1024 + * b. blockletsize -- values in MB. Default value is 64 MB + * c. localDictionaryThreshold -- positive value, default is 10000 + * d. enableLocalDictionary -- true / false. Default is false + * e. sortcolumns -- comma separated column. "c1,c2". Default all dimensions are sorted. + * + * @return updated CarbonWriterBuilder + */ +public CarbonWriterBuilder withTableProperties(Map<String, String> options); +</code></pre> +<pre><code>/** +* this writer is not thread safe, use buildThreadSafeWriterForCSVInput in multi thread environment * Build a {@link CarbonWriter}, which accepts row in CSV format object * @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema} * @return CSVCarbonWriter @@ -511,6 +578,19 @@ public CarbonWriterBuilder withLoadOptions(Map<String, String> options); public CarbonWriter buildWriterForCSVInput(org.apache.carbondata.sdk.file.Schema schema) throws IOException, InvalidLoadOptionException; </code></pre> <pre><code>/** +* Can use this writer in multi-thread instance. +* Build a {@link CarbonWriter}, which accepts row in CSV format +* @param schema carbon Schema object {org.apache.carbondata.sdk.file.Schema} +* @param numOfThreads number of threads() in which .write will be called. +* @return CSVCarbonWriter +* @throws IOException +* @throws InvalidLoadOptionException +*/ +public CarbonWriter buildThreadSafeWriterForCSVInput(Schema schema, short numOfThreads) + throws IOException, InvalidLoadOptionException; +</code></pre> +<pre><code>/** +* this writer is not thread safe, use buildThreadSafeWriterForAvroInput in multi thread environment * Build a {@link CarbonWriter}, which accepts Avro format object * @param avroSchema avro Schema object {org.apache.avro.Schema} * @return AvroCarbonWriter @@ -520,6 +600,19 @@ public CarbonWriter buildWriterForCSVInput(org.apache.carbondata.sdk.file.Schema public CarbonWriter buildWriterForAvroInput(org.apache.avro.Schema schema) throws IOException, InvalidLoadOptionException; </code></pre> <pre><code>/** +* Can use this writer in multi-thread instance. +* Build a {@link CarbonWriter}, which accepts Avro object +* @param avroSchema avro Schema object {org.apache.avro.Schema} +* @param numOfThreads number of threads() in which .write will be called. +* @return AvroCarbonWriter +* @throws IOException +* @throws InvalidLoadOptionException +*/ +public CarbonWriter buildThreadSafeWriterForAvroInput(org.apache.avro.Schema avroSchema, short numOfThreads) + throws IOException, InvalidLoadOptionException +</code></pre> +<pre><code>/** +* this writer is not thread safe, use buildThreadSafeWriterForJsonInput in multi thread environment * Build a {@link CarbonWriter}, which accepts Json object * @param carbonSchema carbon Schema object * @return JsonCarbonWriter @@ -528,6 +621,17 @@ public CarbonWriter buildWriterForAvroInput(org.apache.avro.Schema schema) throw */ public JsonCarbonWriter buildWriterForJsonInput(Schema carbonSchema); </code></pre> +<pre><code>/** +* Can use this writer in multi-thread instance. +* Build a {@link CarbonWriter}, which accepts Json object +* @param carbonSchema carbon Schema object +* @param numOfThreads number of threads() in which .write will be called. +* @return JsonCarbonWriter +* @throws IOException +* @throws InvalidLoadOptionException +*/ +public JsonCarbonWriter buildThreadSafeWriterForJsonInput(Schema carbonSchema, short numOfThreads) +</code></pre> <h3> <a id="class-orgapachecarbondatasdkfilecarbonwriter" class="anchor" href="#class-orgapachecarbondatasdkfilecarbonwriter" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Class org.apache.carbondata.sdk.file.CarbonWriter</h3> <pre><code>/** @@ -536,7 +640,7 @@ public JsonCarbonWriter buildWriterForJsonInput(Schema carbonSchema); * which is one row of data. * If CSVCarbonWriter, object is of type String[], which is one row of data * If JsonCarbonWriter, object is of type String, which is one row of json -* Note: This API is not thread safe +* Note: This API is not thread safe if writer is not built with number of threads argument. * @param object * @throws IOException */ @@ -780,7 +884,6 @@ External client can make use of this reader to read CarbonData files without Car * * @param dataFilePath complete path including carbondata file name * @return Schema object - * @throws IOException */ public static Schema readSchemaInDataFile(String dataFilePath); </code></pre> @@ -882,7 +985,11 @@ public String getProperty(String key); */ public String getProperty(String key, String defaultValue); </code></pre> -<p>Reference : <a href="http://carbondata.apache.org/configuration-parameters.html"; target=_blank rel="nofollow">list of carbon properties</a></p> +<p>Reference : <a href="./configuration-parameters.html">list of carbon properties</a></p> +<script> +// Show selected style on nav item +$(function() { $('.b-nav__api').addClass('selected'); }); +</script> </div> </div> </div> @@ -898,4 +1005,4 @@ public String getProperty(String key, String defaultValue); </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> \ No newline at end of file +</html> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/sdk-writer-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/sdk-writer-guide.html b/src/main/webapp/sdk-writer-guide.html index 36bb9ad..5553194 100644 --- a/src/main/webapp/sdk-writer-guide.html +++ b/src/main/webapp/sdk-writer-guide.html @@ -546,4 +546,4 @@ public static org.apache.carbondata.sdk.file.Schema getCarbonSchemaFromAvroSchem </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> \ No newline at end of file +</html> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/security.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/security.html b/src/main/webapp/security.html index 9168fa3..2aa4b8e 100644 --- a/src/main/webapp/security.html +++ b/src/main/webapp/security.html @@ -205,4 +205,4 @@ <script src="js/custom.js"></script> <script src="js/mdNavigation.js" type="text/javascript"></script> </body> -</html> \ No newline at end of file +</html> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/segment-management-on-carbondata.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/segment-management-on-carbondata.html b/src/main/webapp/segment-management-on-carbondata.html new file mode 100644 index 0000000..528ee9d --- /dev/null +++ b/src/main/webapp/segment-management-on-carbondata.html @@ -0,0 +1,326 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> + <title>CarbonData</title> + <style> + + </style> + <!-- Bootstrap --> + + <link rel="stylesheet" href="css/bootstrap.min.css"> + <link href="css/style.css" rel="stylesheet"> + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js";></script> + <script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="js/jquery.min.js"></script> + <script src="js/bootstrap.min.js"></script> + <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> + + +</head> +<body> +<header> + <nav class="navbar navbar-default navbar-custom cd-navbar-wrapper"> + <div class="container"> + <div class="navbar-header"> + <button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse" + class="navbar-toggle collapsed" type="button"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="index.html" class="logo"> + <img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/> + </a> + </div> + <div class="navbar-collapse collapse cd_navcontnt" id="navbar"> + <ul class="nav navbar-nav navbar-right navlist-custom"> + <li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a> + </li> + <li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false"> Download <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.1/"; + target="_blank">Apache CarbonData 1.4.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.4.0/"; + target="_blank">Apache CarbonData 1.4.0</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.1/"; + target="_blank">Apache CarbonData 1.3.1</a></li> + <li> + <a href="https://dist.apache.org/repos/dist/release/carbondata/1.3.0/"; + target="_blank">Apache CarbonData 1.3.0</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"; + target="_blank">Release Archive</a></li> + </ul> + </li> + <li><a href="documentation.html" class="active">Documentation</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" + aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/How-to-contribute-to-Apache-CarbonData.md"; + target="_blank">Contributing to CarbonData</a></li> + <li> + <a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"; + target="_blank">Release Guide</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"; + target="_blank">Project PMC and Committers</a></li> + <li> + <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"; + target="_blank">CarbonData Meetups</a></li> + <li><a href="security.html">Apache CarbonData Security</a></li> + <li><a href="https://issues.apache.org/jira/browse/CARBONDATA"; target="_blank">Apache + Jira</a></li> + <li><a href="videogallery.html">CarbonData Videos </a></li> + </ul> + </li> + <li class="dropdown"> + <a href="http://www.apache.org/"; class="apache_link hidden-xs dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="http://www.apache.org/"; class="hidden-lg hidden-md hidden-sm dropdown-toggle" + data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/"; target="_blank">Apache Homepage</a></li> + <li><a href="http://www.apache.org/licenses/"; target="_blank">License</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; + target="_blank">Sponsorship</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + </ul> + </li> + + <li> + <a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a> + + </li> + + </ul> + </div><!--/.nav-collapse --> + <div id="search-box"> + <form method="get" action="http://www.google.com/search"; target="_blank"> + <div class="search-block"> + <table border="0" cellpadding="0" width="100%"> + <tr> + <td style="width:80%"> + <input type="text" name="q" size=" 5" maxlength="255" value="" + class="search-input" placeholder="Search...." required/> + </td> + <td style="width:20%"> + <input type="submit" value="Search"/></td> + </tr> + <tr> + <td align="left" style="font-size:75%" colspan="2"> + <input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/> + <span style=" position: relative; top: -3px;"> Only search for CarbonData</span> + </td> + </tr> + </table> + </div> + </form> + </div> + </div> + </nav> +</header> <!-- end Header part --> + +<div class="fixed-padding"></div> <!-- top padding with fixde header --> + +<section><!-- Dashboard nav --> + <div class="container-fluid q"> + <div class="col-sm-12 col-md-12 maindashboard"> + <div class="verticalnavbar"> + <nav class="b-sticky-nav"> + <div class="nav-scroller"> + <div class="nav__inner"> + <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> + <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> + <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> + <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> + <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> + <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> + <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> + <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> + <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> + </div> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> + <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> + <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> + <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> + <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> + </div> + + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> + <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> + <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> + <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> + <a class="b-nav__security nav__item" href="./security.html">Security</a> + <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> + </div> + </div> + <div class="navindicator"> + <div class="b-nav__intro navindicator__item"></div> + <div class="b-nav__quickstart navindicator__item"></div> + <div class="b-nav__uses navindicator__item"></div> + <div class="b-nav__docs navindicator__item"></div> + <div class="b-nav__datamap navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> + <div class="b-nav__api navindicator__item"></div> + <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__faq navindicator__item"></div> + <div class="b-nav__contri navindicator__item"></div> + <div class="b-nav__security navindicator__item"></div> + </div> + </nav> + </div> + <div class="mdcontent"> + <section> + <div style="padding:10px 15px;"> + <div id="viewpage" name="viewpage"> + <div class="row"> + <div class="col-sm-12 col-md-12"> + <div> +<h2> +<a id="segment-management" class="anchor" href="#segment-management" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SEGMENT MANAGEMENT</h2> +<p>Each load into CarbonData is written into a separate folder called Segment.Segments is a powerful +concept which helps to maintain consistency of data and easy transaction management.CarbonData provides DML (Data Manipulation Language) commands to maintain the segments.</p> +<ul> +<li><a href="#show-segment">Show Segments</a></li> +<li><a href="#delete-segment-by-id">Delete Segment by ID</a></li> +<li><a href="#delete-segment-by-date">Delete Segment by Date</a></li> +<li><a href="#query-data-with-specified-segments">Query Data with Specified Segments</a></li> +</ul> +<h3> +<a id="show-segment" class="anchor" href="#show-segment" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>SHOW SEGMENT</h3> +<p>This command is used to list the segments of CarbonData table.</p> +<pre><code>SHOW [HISTORY] SEGMENTS FOR TABLE [db_name.]table_name LIMIT number_of_segments +</code></pre> +<p>Example: +Show visible segments</p> +<pre><code>SHOW SEGMENTS FOR TABLE CarbonDatabase.CarbonTable LIMIT 4 +</code></pre> +<p>Show all segments, include invisible segments</p> +<pre><code>SHOW HISTORY SEGMENTS FOR TABLE CarbonDatabase.CarbonTable LIMIT 4 +</code></pre> +<h3> +<a id="delete-segment-by-id" class="anchor" href="#delete-segment-by-id" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>DELETE SEGMENT BY ID</h3> +<p>This command is used to delete segment by using the segment ID. Each segment has a unique segment ID associated with it. +Using this segment ID, you can remove the segment.</p> +<p>The following command will get the segmentID.</p> +<pre><code>SHOW SEGMENTS FOR TABLE [db_name.]table_name LIMIT number_of_segments +</code></pre> +<p>After you retrieve the segment ID of the segment that you want to delete, execute the following command to delete the selected segment.</p> +<pre><code>DELETE FROM TABLE [db_name.]table_name WHERE SEGMENT.ID IN (segment_id1, segments_id2, ...) +</code></pre> +<p>Example:</p> +<pre><code>DELETE FROM TABLE CarbonDatabase.CarbonTable WHERE SEGMENT.ID IN (0) +DELETE FROM TABLE CarbonDatabase.CarbonTable WHERE SEGMENT.ID IN (0,5,8) +</code></pre> +<h3> +<a id="delete-segment-by-date" class="anchor" href="#delete-segment-by-date" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>DELETE SEGMENT BY DATE</h3> +<p>This command will allow to delete the CarbonData segment(s) from the store based on the date provided by the user in the DML command. +The segment created before the particular date will be removed from the specific stores.</p> +<pre><code>DELETE FROM TABLE [db_name.]table_name WHERE SEGMENT.STARTTIME BEFORE DATE_VALUE +</code></pre> +<p>Example:</p> +<pre><code>DELETE FROM TABLE CarbonDatabase.CarbonTable WHERE SEGMENT.STARTTIME BEFORE '2017-06-01 12:05:06' +</code></pre> +<h3> +<a id="query-data-with-specified-segments" class="anchor" href="#query-data-with-specified-segments" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>QUERY DATA WITH SPECIFIED SEGMENTS</h3> +<p>This command is used to read data from specified segments during CarbonScan.</p> +<p>Get the Segment ID:</p> +<pre><code>SHOW SEGMENTS FOR TABLE [db_name.]table_name LIMIT number_of_segments +</code></pre> +<p>Set the segment IDs for table</p> +<pre><code>SET carbon.input.segments.<database_name>.<table_name> = <list of segment IDs> +</code></pre> +<p><strong>NOTE:</strong> +carbon.input.segments: Specifies the segment IDs to be queried. This property allows you to query specified segments of the specified table. The CarbonScan will read data from specified segments only.</p> +<p>If user wants to query with segments reading in multi threading mode, then CarbonSession. threadSet can be used instead of SET query.</p> +<pre><code>CarbonSession.threadSet ("carbon.input.segments.<database_name>.<table_name>","<list of segment IDs>"); +</code></pre> +<p>Reset the segment IDs</p> +<pre><code>SET carbon.input.segments.<database_name>.<table_name> = *; +</code></pre> +<p>If user wants to query with segments reading in multi threading mode, then CarbonSession. threadSet can be used instead of SET query.</p> +<pre><code>CarbonSession.threadSet ("carbon.input.segments.<database_name>.<table_name>","*"); +</code></pre> +<p><strong>Examples:</strong></p> +<ul> +<li>Example to show the list of segment IDs,segment status, and other required details and then specify the list of segments to be read.</li> +</ul> +<pre><code>SHOW SEGMENTS FOR carbontable1; + +SET carbon.input.segments.db.carbontable1 = 1,3,9; +</code></pre> +<ul> +<li>Example to query with segments reading in multi threading mode:</li> +</ul> +<pre><code>CarbonSession.threadSet ("carbon.input.segments.db.carbontable_Multi_Thread","1,3"); +</code></pre> +<ul> +<li>Example for threadset in multithread environment (following shows how it is used in Scala code):</li> +</ul> +<pre><code>def main(args: Array[String]) { +Future { + CarbonSession.threadSet ("carbon.input.segments.db.carbontable_Multi_Thread","1") + spark.sql("select count(empno) from carbon.input.segments.db.carbontable_Multi_Thread").show(); + } + } +</code></pre> +<script> +$(function() { + // Show selected style on nav item + $('.b-nav__docs').addClass('selected'); + // Display docs subnav items + if (!$('.b-nav__docs').parent().hasClass('nav__item__with__subs--expanded')) { + $('.b-nav__docs').parent().toggleClass('nav__item__with__subs--expanded'); + } +}); +</script> +</div> +</div> +</div> +</div> +<div class="doc-footer"> + <a href="#top" class="scroll-top">Top</a> +</div> +</div> +</section> +</div> +</div> +</div> +</section><!-- End systemblock part --> +<script src="js/custom.js"></script> +</body> +</html> http://git-wip-us.apache.org/repos/asf/carbondata-site/blob/44eed099/src/main/webapp/streaming-guide.html ---------------------------------------------------------------------- diff --git a/src/main/webapp/streaming-guide.html b/src/main/webapp/streaming-guide.html index 6e2e7ee..c6d8391 100644 --- a/src/main/webapp/streaming-guide.html +++ b/src/main/webapp/streaming-guide.html @@ -22,6 +22,7 @@ <![endif]--> <script src="js/jquery.min.js"></script> <script src="js/bootstrap.min.js"></script> + <script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js";></script> </head> @@ -67,7 +68,7 @@ target="_blank">Release Archive</a></li> </ul> </li> - <li><a href="mainpage.html" class="active">Documentation</a></li> + <li><a href="documentation.html" class="active">Documentation</a></li> <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Community <span class="caret"></span></a> @@ -152,7 +153,57 @@ <section><!-- Dashboard nav --> <div class="container-fluid q"> <div class="col-sm-12 col-md-12 maindashboard"> - <div class="row"> + <div class="verticalnavbar"> + <nav class="b-sticky-nav"> + <div class="nav-scroller"> + <div class="nav__inner"> + <a class="b-nav__intro nav__item" href="./introduction.html">introduction</a> + <a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a> + <a class="b-nav__uses nav__item" href="./usescases.html">use cases</a> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a> + <a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a> + <a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a> + <a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a> + <a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a> + <a class="nav__item nav__sub__item" href="./datamap-developer-guide.html">Datamaps</a> + <a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a> + </div> + + <div class="nav__item nav__item__with__subs"> + <a class="b-nav__datamap nav__item nav__sub__anchor" href="./datamap-management.html">DataMaps</a> + <a class="nav__item nav__sub__item" href="./bloomfilter-datamap-guide.html">Bloom Filter</a> + <a class="nav__item nav__sub__item" href="./lucene-datamap-guide.html">Lucene</a> + <a class="nav__item nav__sub__item" href="./preaggregate-datamap-guide.html">Pre-Aggregate</a> + <a class="nav__item nav__sub__item" href="./timeseries-datamap-guide.html">Time Series</a> + </div> + + <a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Support</a> + <a class="b-nav__api nav__item" href="./sdk-guide.html">API</a> + <a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a> + <a class="b-nav__faq nav__item" href="./faq.html">FAQ</a> + <a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a> + <a class="b-nav__security nav__item" href="./security.html">Security</a> + <a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a> + </div> + </div> + <div class="navindicator"> + <div class="b-nav__intro navindicator__item"></div> + <div class="b-nav__quickstart navindicator__item"></div> + <div class="b-nav__uses navindicator__item"></div> + <div class="b-nav__docs navindicator__item"></div> + <div class="b-nav__datamap navindicator__item"></div> + <div class="b-nav__s3 navindicator__item"></div> + <div class="b-nav__api navindicator__item"></div> + <div class="b-nav__perf navindicator__item"></div> + <div class="b-nav__faq navindicator__item"></div> + <div class="b-nav__contri navindicator__item"></div> + <div class="b-nav__security navindicator__item"></div> + </div> + </nav> + </div> + <div class="mdcontent"> <section> <div style="padding:10px 15px;"> <div id="viewpage" name="viewpage"> @@ -432,6 +483,17 @@ streaming table using following DDL.</p> <li>block delete "streaming" segment while the streaming ingestion is running.</li> <li>block drop the streaming table while the streaming ingestion is running.</li> </ol> +<script> +$(function() { + // Show selected style on nav item + $('.b-nav__docs').addClass('selected'); + + // Display docs subnav items + if (!$('.b-nav__docs').parent().hasClass('nav__item__with__subs--expanded')) { + $('.b-nav__docs').parent().toggleClass('nav__item__with__subs--expanded'); + } +}); +</script> </div> </div> </div> @@ -447,4 +509,4 @@ streaming table using following DDL.</p> </section><!-- End systemblock part --> <script src="js/custom.js"></script> </body> -</html> \ No newline at end of file +</html>
