Repository: incubator-tephra Updated Branches: refs/heads/site ab91a88bb -> a764587ef (forced update)
WIP: Adding site Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/a764587e Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/a764587e Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/a764587e Branch: refs/heads/site Commit: a764587efc0665f137c56e2db4785f50005e83d8 Parents: d1a340a Author: Terence Yim <[email protected]> Authored: Tue Jun 21 00:11:12 2016 -0700 Committer: Terence Yim <[email protected]> Committed: Tue Jun 21 15:08:32 2016 -0700 ---------------------------------------------------------------------- pom.xml | 96 +++- src/site/markdown/GettingStarted.md | 463 ++++++++++++++++++++ src/site/markdown/HowToContribute.md | 71 +++ src/site/markdown/ReleaseGuide.md | 220 ++++++++++ src/site/markdown/index.md | 116 +++++ src/site/markdown/releases/0.8.0-incubating.md | 28 ++ src/site/resources/images/tephra_logotype.png | Bin 0 -> 12676 bytes src/site/site.xml | 105 +++++ 8 files changed, 1098 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4455efb..9c5b12a 100644 --- a/pom.xml +++ b/pom.xml @@ -392,7 +392,7 @@ <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> - <version>0.10</version> + <version>0.11</version> <executions> <execution> <id>rat-check</id> @@ -614,6 +614,100 @@ </plugins> </build> </profile> + + <profile> + <id>site</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-site-plugin</artifactId> + <version>3.4</version> + <dependencies> + <dependency> + <groupId>org.apache.maven.doxia</groupId> + <artifactId>doxia-core</artifactId> + <version>1.6</version> + </dependency> + <dependency> + <groupId>org.apache.maven.doxia</groupId> + <artifactId>doxia-module-markdown</artifactId> + <version>1.6</version> + </dependency> + <dependency> + <groupId>lt.velykis.maven.skins</groupId> + <artifactId>reflow-velocity-tools</artifactId> + <version>1.1.1</version> + </dependency> + <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + <version>1.7</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </build> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-project-info-reports-plugin</artifactId> + <version>2.7</version> + <reportSets> + <reportSet> + <reports/> + </reportSet> + <reportSet> + <id>aggregate</id> + <inherited>false</inherited> + <reports> + <report>index</report> + <report>mailing-list</report> + <report>scm</report> + <report>issue-tracking</report> + <report>project-team</report> + </reports> + </reportSet> + </reportSets> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.9.1</version> + <configuration> + <failOnError>false</failOnError> + <links> + <link>http://download.oracle.com/javase/7/docs/api/</link> + </links> + <bottom> + <![CDATA[Copyright © 2016 <a href="http://www.apache.org";>The Apache Software Foundation</a>. All rights reserved.]]> + </bottom> + </configuration> + <reportSets> + <!--<reportSet>--> + <!--<reports>--> + <!--<report>javadoc</report>--> + <!--</reports>--> + <!--</reportSet>--> + <reportSet> + <id>aggregate</id> + <inherited>false</inherited> + <reports> + <report>aggregate</report> + </reports> + </reportSet> + </reportSets> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>2.14.1</version> + </plugin> + </plugins> + </reporting> + </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/markdown/GettingStarted.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/GettingStarted.md b/src/site/markdown/GettingStarted.md new file mode 100644 index 0000000..ff03018 --- /dev/null +++ b/src/site/markdown/GettingStarted.md @@ -0,0 +1,463 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<head> + <title>Getting Started</title> +</head> + +## Getting Started +You can get started with Tephra by building directly from the latest source code: + +```sh + git clone https://git-wip-us.apache.org/repos/asf/incubator-tephra.git + cd incubator-tephra + mvn clean package +``` + +After the build completes, you will have a full binary distribution of Tephra under the +`tephra-distribution/target/` directory. Take the `tephra-<version>.tar.gz` file and install +it on your systems. + +For any client applications, add the following dependencies to any Apache Maven POM files (or your +build system's equivalent configuration), in order to make use of Tephra classes: + +```xml + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-api</artifactId> + <version>${tephra.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-core</artifactId> + <version>${tephra.version}</version> + </dependency> +``` + +Since the HBase APIs have changed between versions, you will need to select the +appropriate HBase compatibility library. + +For HBase 0.96.x: + +```xml + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-hbase-compat-0.96</artifactId> + <version>${tephra.version}</version> + </dependency> +``` + +For HBase 0.98.x: + +```xml + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-hbase-compat-0.98</artifactId> + <version>${tephra.version}</version> + </dependency> +``` + +For HBase 1.0.x: + +```xml + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-hbase-compat-1.0</artifactId> + <version>${tephra.version}</version> + </dependency> +``` + +If you are running the CDH 5.4, 5.5, or 5.6 version of HBase 1.0.x (this version contains API incompatibilities +with Apache HBase 1.0.x): + +```xml + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-hbase-compat-1.0-cdh</artifactId> + <version>${tephra.version}</version> + </dependency> +``` + +For HBase 1.1.x or HBase 1.2.x: + +```xml + <dependency> + <groupId>org.apache.tephra</groupId> + <artifactId>tephra-hbase-compat-1.1</artifactId> + <version>${tephra.version}</version> + </dependency> +``` + +## Deployment and Configuration +Tephra makes use of a central transaction server to assign unique transaction IDs for data +modifications and to perform conflict detection. Only a single transaction server can actively +handle client requests at a time, however, additional transaction server instances can be run +simultaneously, providing automatic failover if the active server becomes unreachable. + +### Transaction Server Configuration +The Tephra transaction server can be deployed on the same cluster nodes running the HBase HMaster +process. The transaction server requires that the HBase libraries be available on the server's +Java `CLASSPATH`. + +The transaction server supports the following configuration properties. All configuration +properties can be added to the `hbase-site.xml` file on the server's `CLASSPATH`: + + +| Name | Default | Description | +|-----------------------------|------------|-----------------------------------------------------------------| +| `data.tx.bind.port` | 15165 | Port to bind to | +| `data.tx.bind.address` | 0.0.0.0 | Server address to listen on | +| `data.tx.server.io.threads` | 2 | Number of threads for socket IO | +| `data.tx.server.threads` | 20 | Number of handler threads | +| `data.tx.timeout` | 30 | Timeout for a transaction to complete (seconds) | +| `data.tx.long.timeout` | 86400 | Timeout for a long running transaction to complete (seconds) | +| `data.tx.cleanup.interval` | 10 | Frequency to check for timed out transactions (seconds) | +| `data.tx.snapshot.dir` | | HDFS directory used to store snapshots of tx state | +| `data.tx.snapshot.interval` | 300 | Frequency to write new snapshots | +| `data.tx.snapshot.retain` | 10 | Number of old transaction snapshots to retain | +| `data.tx.metrics.period` | 60 | Frequency for metrics reporting (seconds) | + +To run the Transaction server, execute the following command in your Tephra installation: + +```sh + ./bin/tephra start +``` + +Any environment-specific customizations can be made by editing the `bin/tephra-env.sh` script. + + +### Client Configuration +Since Tephra clients will be communicating with HBase, the HBase client libraries and the HBase cluster +configuration must be available on the client's Java `CLASSPATH`. + +Client API usage is described in the Client APIs section. + +The transaction service client supports the following configuration properties. All configuration +properties can be added to the `hbase-site.xml` file on the client's `CLASSPATH`: + +| Name | Default | Description | +|----------------------------------------|-----------|-----------------------------------------------| +| `data.tx.client.timeout` | 30000 | Client socket timeout (milliseconds) | +| `data.tx.client.provider` | pool | Client provider strategy: <ul><li>"pool" uses a pool of clients</li><li>"thread-local" a client per thread</li></ul> Note that "thread-local" provider can have a resource leak if threads are recycled | +| `data.tx.client.count` | 50 | Max number of clients for "pool" provider | +| `data.tx.client.obtain.timeout` | 3000 | Timeout (milliseconds) to wait when obtaining clients from the "pool" provider | +| `data.tx.client.retry.strategy` | backoff | Client retry strategy: "backoff" for back off between attempts; "n-times" for fixed number of tries | +| `data.tx.client.retry.attempts` | 2 | Number of times to retry ("n-times" strategy) | +| `data.tx.client.retry.backoff.initial` | 100 | Initial sleep time ("backoff" strategy) | +| `data.tx.client.retry.backoff.factor` | 4 | Multiplication factor for sleep time | +| `data.tx.client.retry.backoff.limit` | 30000 | Exit when sleep time reaches this limit | + + +### HBase Coprocessor Configuration + +In addition to the transaction server, Tephra requires an HBase coprocessor to be installed on all +tables where transactional reads and writes will be performed. + +To configure the coprocessor on all HBase tables, add the following to `hbase-site.xml`: + +```xml + <property> + <name>hbase.coprocessor.region.classes</name> + <value>org.apache.tephra.hbase.coprocessor.TransactionProcessor</value> + </property> +``` + +You may configure the `TransactionProcessor` to be loaded only on HBase tables that you will +be using for transaction reads and writes. However, you must ensure that the coprocessor is +available on all impacted tables in order for Tephra to function correctly. + +### Using Existing HBase Tables Transactionally + +Tephra overrides HBase cell timestamps with transaction IDs, and uses these transaction +IDs to filter out cells older than the TTL (Time-To-Live). Transaction IDs are at a higher +scale than cell timestamps. When a regular HBase table that has existing data is +converted to a transactional table, existing data may be filtered out during reads. To +allow reading of existing data from a transactional table, you will need to set the +property `data.tx.read.pre.existing` as `true` on the table's table descriptor. + +Note that even without the property `data.tx.read.pre.existing` being set to `true`, +any existing data will not be removed during compactions. Existing data simply won't be +visible during reads. + +### Metrics Reporting + +Tephra ships with built-in support for reporting metrics via JMX and a log file, using the +[Dropwizard Metrics](http://metrics.dropwizard.io) library. + +To enable JMX reporting for metrics, you will need to enable JMX in the Java runtime +arguments. Edit the `bin/tephra-env.sh` script and uncomment the following lines, making any +desired changes to configuration for port used, SSL, and JMX authentication: + +```sh + export JMX_OPTS="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=13001" + export OPTS="$OPTS $JMX_OPTS" +``` + +To enable file-based reporting for metrics, edit the `conf/logback.xml` file and uncomment the +following section, replacing the `FILE-PATH` placeholder with a valid directory on the local +filesystem: + +```xml + <appender name="METRICS" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>/FILE-PATH/metrics.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>metrics.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder> + <pattern>%d{ISO8601} %msg%n</pattern> + </encoder> + </appender> + <logger name="tephra-metrics" level="TRACE" additivity="false"> + <appender-ref ref="METRICS" /> + </logger> +``` + +The frequency of metrics reporting may be configured by setting the `data.tx.metrics.period` +configuration property to the report frequency in seconds. + + +## Client APIs + +The `TransactionAwareHTable` class implements HBase's `HTableInterface`, thus providing the same APIs +that a standard HBase `HTable` instance provides. Only certain operations are supported +transactionally. These are: + +| Methods Supported In Transactions | +|---------------------------------------------------------------------------------------------------| +| `exists(Get get)` | +| `exists(List<Get> gets)` | +| `get(Get get)` | +| `get(List<Get> gets)` | +| `batch(List<? extends Row> actions, Object[] results)` | +| `batch(List<? extends Row> actions)` | +| `batchCallback(List<? extends Row> actions, Object[] results, Batch.Callback<R> callback)` [0.96] | +| `batchCallback(List<? extends Row> actions, Batch.Callback<R> callback)` [0.96] | +| `getScanner(byte[] family)` | +| `getScanner(byte[] family, byte[] qualifier)` | +| `put(Put put)` | +| `put(List<Put> puts)` | +| `delete(Delete delete)` | +| `delete(List<Delete> deletes)` | + +Other operations are not supported transactionally and will throw an ``UnsupportedOperationException`` if invoked. +To allow use of these non-transactional operations, call ``setAllowNonTransactional(true)``. This +allows you to call the following methods non-transactionally: + +| Methods Supported Outside of Transactions | +|---------------------------------------------------------------------------------------------------------| +| `getRowOrBefore(byte[] row, byte[], family)` | +| `checkAndPut(byte[] row, byte[] family, byte[] qualifier, byte[] value, Put put)` | +| `checkAndDelete(byte[] row, byte[] family, byte[] qualifier, byte[] value, Delete delete)` | +| `mutateRow(RowMutations rm)` | +| `append(Append append)` | +| `increment(Increment increment)` | +| `incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount)` | +| `incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, Durability durability)` | +| `incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL)` | + +Note that for `batch` operations, only certain supported operations (`get`, `put`, and `delete`) +are applied transactionally. + +### Usage + +To use a `TransactionalAwareHTable`, you need an instance of `TransactionContext`. +`TransactionContext` provides the basic contract for client use of transactions. At each point +in the transaction lifecycle, it provides the necessary interactions with the Tephra Transaction +Server in order to start, commit, and rollback transactions. Basic usage of +`TransactionContext` is handled using the following pattern: + +```java + TransactionContext context = new TransactionContext(client, transactionAwareHTable); + try { + context.start(); + transactionAwareHTable.put(new Put(Bytes.toBytes("row")); + // ... + context.finish(); + } catch (TransactionFailureException e) { + context.abort(); + } +``` + +1. First, a new transaction is started using `TransactionContext.start()`. +1. Next, any data operations are performed within the context of the transaction. +1. After data operations are complete, `TransactionContext.finish()` is called to commit the + transaction. +1. If an exception occurs, `TransactionContext.abort()` can be called to rollback the + transaction. + +`TransactionAwareHTable` handles the details of performing data operations transactionally, and +implements the necessary hooks in order to commit and rollback the data changes (see +`TransactionAware`). + +### Example + +To demonstrate how you might use `TransactionAwareHTable`\s, below is a basic implementation of a +`SecondaryIndexTable`. This class encapsulates the usage of a `TransactionContext` and provides a simple interface +to a user: + +```java + /** + * A Transactional SecondaryIndexTable. + */ + public class SecondaryIndexTable { + private byte[] secondaryIndex; + private TransactionAwareHTable transactionAwareHTable; + private TransactionAwareHTable secondaryIndexTable; + private TransactionContext transactionContext; + private final TableName secondaryIndexTableName; + private static final byte[] secondaryIndexFamily = + Bytes.toBytes("secondaryIndexFamily"); + private static final byte[] secondaryIndexQualifier = Bytes.toBytes('r'); + private static final byte[] DELIMITER = new byte[] {0}; + + public SecondaryIndexTable(TransactionServiceClient transactionServiceClient, + HTable hTable, byte[] secondaryIndex) { + secondaryIndexTableName = + TableName.valueOf(hTable.getName().getNameAsString() + ".idx"); + HTable secondaryIndexHTable = null; + HBaseAdmin hBaseAdmin = null; + try { + hBaseAdmin = new HBaseAdmin(hTable.getConfiguration()); + if (!hBaseAdmin.tableExists(secondaryIndexTableName)) { + hBaseAdmin.createTable(new HTableDescriptor(secondaryIndexTableName)); + } + secondaryIndexHTable = new HTable(hTable.getConfiguration(), + secondaryIndexTableName); + } catch (Exception e) { + Throwables.propagate(e); + } finally { + try { + hBaseAdmin.close(); + } catch (Exception e) { + Throwables.propagate(e); + } + } + + this.secondaryIndex = secondaryIndex; + this.transactionAwareHTable = new TransactionAwareHTable(hTable); + this.secondaryIndexTable = new TransactionAwareHTable(secondaryIndexHTable); + this.transactionContext = new TransactionContext(transactionServiceClient, + transactionAwareHTable, + secondaryIndexTable); + } + + public Result get(Get get) throws IOException { + return get(Collections.singletonList(get))[0]; + } + + public Result[] get(List<Get> gets) throws IOException { + try { + transactionContext.start(); + Result[] result = transactionAwareHTable.get(gets); + transactionContext.finish(); + return result; + } catch (Exception e) { + try { + transactionContext.abort(); + } catch (TransactionFailureException e1) { + throw new IOException("Could not rollback transaction", e1); + } + } + return null; + } + + public Result[] getByIndex(byte[] value) throws IOException { + try { + transactionContext.start(); + Scan scan = new Scan(value, Bytes.add(value, new byte[0])); + scan.addColumn(secondaryIndexFamily, secondaryIndexQualifier); + ResultScanner indexScanner = secondaryIndexTable.getScanner(scan); + + ArrayList<Get> gets = new ArrayList<Get>(); + for (Result result : indexScanner) { + for (Cell cell : result.listCells()) { + gets.add(new Get(cell.getValue())); + } + } + Result[] results = transactionAwareHTable.get(gets); + transactionContext.finish(); + return results; + } catch (Exception e) { + try { + transactionContext.abort(); + } catch (TransactionFailureException e1) { + throw new IOException("Could not rollback transaction", e1); + } + } + return null; + } + + public void put(Put put) throws IOException { + put(Collections.singletonList(put)); + } + + + public void put(List<Put> puts) throws IOException { + try { + transactionContext.start(); + ArrayList<Put> secondaryIndexPuts = new ArrayList<Put>(); + for (Put put : puts) { + List<Put> indexPuts = new ArrayList<Put>(); + Set<Map.Entry<byte[], List<KeyValue>>> familyMap = put.getFamilyMap().entrySet(); + for (Map.Entry<byte [], List<KeyValue>> family : familyMap) { + for (KeyValue value : family.getValue()) { + if (value.getQualifier().equals(secondaryIndex)) { + byte[] secondaryRow = Bytes.add(value.getQualifier(), + DELIMITER, + Bytes.add(value.getValue(), + DELIMITER, + value.getRow())); + Put indexPut = new Put(secondaryRow); + indexPut.add(secondaryIndexFamily, secondaryIndexQualifier, put.getRow()); + indexPuts.add(indexPut); + } + } + } + secondaryIndexPuts.addAll(indexPuts); + } + transactionAwareHTable.put(puts); + secondaryIndexTable.put(secondaryIndexPuts); + transactionContext.finish(); + } catch (Exception e) { + try { + transactionContext.abort(); + } catch (TransactionFailureException e1) { + throw new IOException("Could not rollback transaction", e1); + } + } + } + } +``` + +## Known Issues and Limitations + +- Currently, column family `Delete` operations are implemented by writing a cell with an empty + qualifier (empty `byte[]`) and empty value (empty `byte[]`). This is done in place of + native HBase `Delete` operations so the delete marker can be rolled back in the event of + a transaction failure -- normal HBase `Delete` operations cannot be undone. However, this + means that applications that store data in a column with an empty qualifier will not be able to + store empty values, and will not be able to transactionally delete that column. +- Column `Delete` operations are implemented by writing a empty value (empty `byte[]`) to the + column. This means that applications will not be able to store empty values to columns. +- Invalid transactions are not automatically cleared from the exclusion list. When a transaction is + invalidated, either from timing out or being invalidated by the client due to a failure to rollback + changes, its transaction ID is added to a list of excluded transactions. Data from invalidated + transactions will be dropped by the `TransactionProcessor` coprocessor on HBase region flush + and compaction operations. Currently, however, transaction IDs can only be manually removed + from the list of excluded transaction IDs, using the `org.apache.tephra.TransactionAdmin` tool. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/markdown/HowToContribute.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/HowToContribute.md b/src/site/markdown/HowToContribute.md new file mode 100644 index 0000000..381a048 --- /dev/null +++ b/src/site/markdown/HowToContribute.md @@ -0,0 +1,71 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<head> + <title>How to Contribute</title> +</head> + +## Contributing to Apache Tephra + +The Apache Tephra team welcome all types of contributions, whether they are bug reports, feature requests, +documentation, or code patches. + +### Reporting Issues + +To report bugs or request new features, please open an issue in the +[Apache Tephra JIRA](https://issues.apache.org/jira/browse/TEPHRA). You can also use the +[dev mailing list](mail-lists.html) for general questions or discussions. + +### Contributing Code + +We prefer contributions through [GitHub](https://github.com/apache/incubator-tephra) pull requests. Please follow +these steps to get your contributions in: + +1. Open a new issue or pick up an existing one in the [Apache TEPHRA JIRA](https://issues.apache.org/jira/browse/TEPHRA) + about the patch that you are going to submit. +2. If you are proposing public API changes or big changes, please attach a design document to the JIRA. You + can also use the [dev mailing list](mail-lists.html) to discuss it first. This will help us understand your needs + and best guide your solution in a way that fits the project. +3. [Fork](https://help.github.com/articles/fork-a-repo) the + [Apache Tephra GitHub repo.](https://github.com/apache/incubator-tephra) +4. Make the changes and send a [pull request](https://help.github.com/articles/using-pull-requests) from your + forked repo to the Apache Tephra repo. +5. Please prefix your pull request title with the JIRA issue ID; for example, `(TEPHRA-35) Added invalid transaction pruning`. +6. Please complete the pull request description with additional details as appropriate. +7. Once sent, code review will be done through the pull request. +8. Once all review issues are resolved, we will merge the changes into the `master` branch of the Apache Tephra repo. + +### How to Merge Code Changes + +Committer can merge code changes that are already reviewed into the `master` branch with the following steps: + +1. Make sure the GitHub pull request is squashed into one commit. If not, ask the patch contributor to help doing so. + +2. Download the patch file from GitHub. You can append `.patch` to the end of the GitHub pull request URL to get the patch file. + + curl -L -O https://github.com/apache/incubator-tephra/pull/${PR_NUMBER}.patch +3. Edit the patch file and add the following line in the commit message for closing the pull request. + + This closes #${PR_NUMBER} from GitHub. +4. Apply the patch and push it back to remote repo. Make sure you apply it on the latest `master` branch. + + git checkout master + git pull origin master + git am --signoff < ${PR_NUMBER} + git push origin master +5. Close the JIRA issue associated with the patch. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/markdown/ReleaseGuide.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/ReleaseGuide.md b/src/site/markdown/ReleaseGuide.md new file mode 100644 index 0000000..050d21c --- /dev/null +++ b/src/site/markdown/ReleaseGuide.md @@ -0,0 +1,220 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<head> + <title>Release Guide</title> +</head> + +This page describes the step-by-step process of how to perform an official Apache Tephra version release, +including deploying the release artifacts to Maven repositories and the additional administrative +steps to complete the release process. + +## Prerequisites + +### Maven Settings File +Prior to performing an Apache Tephra release, you must have an entry such as this in your +`~/.m2/settings.xml` file to authenticate when deploying the release artifacts: + +```xml + <?xml version="1.0" encoding="UTF-8"?> + <settings xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"; + xmlns="http://maven.apache.org/SETTINGS/1.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <servers> + <server> + <id>apache.snapshots.https</id> + <username>USERNAME</username> + <password>PASSWORD</password> + </server> + <server> + <id>apache.releases.https</id> + <username>USERNAME</username> + <password>PASSWORD</password> + </server> + </servers> + </settings> +``` + +Replace `USERNAME` and `PASSWORD` with the correct values for your user account. See the +[Maven Encryption Guide](http://maven.apache.org/guides/mini/guide-encryption.html) for details +on how to avoid storing the plaintext password in the `settings.xml` file. + +### PGP Key +You will also need to have created a PGP (or GPG) key pair, which will be used in signing the release +artifacts. For more information on using the Maven GPG plugin, see this +[introduction](http://blog.sonatype.com/2010/01/how-to-generate-pgp-signatures-with-maven/) from Sonatype and +the Maven GPG Plugin [usage page](https://maven.apache.org/plugins/maven-gpg-plugin/usage.html). +You may also want to run gpg-agent in order to avoid being prompted multiple times for the GPG key passphrase when +performing a release. + + +## Performing the Release + +### Ensure Local Branch is Up-to-date +First, make sure your local copy of the `master` branch is up-to-date with all changes: + +```sh + git checkout master + git pull +``` + +### Create the Release Branch +Next, create a release branch from `master`: + +```sh + git checkout -b release/N.N.N +``` + +replacing `N.N.N` with the desired release version. + +### Prepare the Release +While on the release branch, prepare the release: + +```sh + mvn clean release:prepare -P apache-release +``` + +This will prompt you for the release version and the git tag to use for the release. By +convention, we use `vN.N.N` for the release tag (ie. v0.6.0 for release 0.6.0). + +### Perform the Release +Perform the release by running: + +```sh + mvn release:perform -P apache-release +``` + +This will checkout the source code using the release tag, build the release and deploy it to the +repository.apache.org repository. Also it creates a source tarball +`apache-tephra-0.8.0-incubating-SNAPSHOT-source-release.tar.gz` under the `target` directory. + +### Prepare Release Artifacts +1. Checkin the source release tarball, together with the signature, md5 and sha512 files found in + `target/` directory + to `dist.apache.org/repos/dist/dev/incubator/tephra/${RELEASE_VERSION}-incubating-rc1/src/`. +1. Create a CHANGES.txt file to describe the changes in the release and checkin the file to + `dist.apache.org/repos/dist/dev/incubator/tephra/${RELEASE_VERSION}-incubating-rc1/CHANGES.txt`. +1. Close the staging repository at <https://repository.apache.org> + + +### Update POM Version in master +Update the POMs in `master` by: + +```sh + git checkout master + git merge release/N.N.N + git push origin master +``` + +### Vote for the Release in Dev Mailing List +Create a vote in the dev@tephra mailing list, and wait for 72 hours for the vote result. +Here is a template for the email: + +``` + Subject: [VOTE] Release of Apache Tephra-${RELEASE_VERSION}-incubating [rc1] + ============================================================================ + + Hi all, + + This is a call for a vote on releasing Apache Tephra ${RELEASE_VERSION}-incubating, release candidate 1. This + is the [Nth] release of Tephra. + + The source tarball, including signatures, digests, etc. can be found at: + https://dist.apache.org/repos/dist/dev/incubator/tephra/${RELEASE_VERSION}-incubating-rc1/src + + The tag to be voted upon is v${RELEASE_VERSION}-incubating: + https://git-wip-us.apache.org/repos/asf?p=incubator-tephra.git;a=shortlog;h=refs/tags/v${RELEASE_VERSION}-incubating + + The release hash is [REF]: + https://git-wip-us.apache.org/repos/asf?p=incubator-tephra.git;a=commit;h=[REF] + + The Nexus Staging URL: + https://repository.apache.org/content/repositories/orgapachetephra-[STAGE_ID] + + Release artifacts are signed with the following key: + [URL_TO_SIGNER_PUBLIC_KEY] + + KEYS file available: + https://dist.apache.org/repos/dist/dev/incubator/tephra/KEYS + + For information about the contents of this release, see: + https://dist.apache.org/repos/dist/dev/incubator/tephra/${RELEASE_VERSION}-incubating-rc1/CHANGES.txt + + Please vote on releasing this package as Apache Tephra ${RELEASE_VERSION}-incubating + + The vote will be open for 72 hours. + + [ ] +1 Release this package as Apache Tephra ${RELEASE_VERSION}-incubating + [ ] +0 no opinion + [ ] -1 Do not release this package because ... + + Thanks, + [YOUR_NAME] +``` + +### Consolidate Vote Result +After the vote is up for 72 hours and having at least three +1 binding votes and no -1 votes, +close the vote by replying to the voting thread. Here is a template for the reply email: + +``` + Subject: [RESULT][VOTE] Release of Apache Tephra-${RELEASE_VERSION}-incubating [rc1] + ================================================================================== + + Hi all, + + After being opened for over 72 hours, the vote for releasing Apache Tephra + ${RELEASE_VERSION}-incubating passed with n binding +1s and no 0 or -1. + + Binding +1s: + [BINDING_+1_NAMES] + + I am going to create a vote in the general@ list. + + Thanks, + [YOUR_NAME] +``` + +### Vote for the Release from IPMC +1. Create a vote in the general@ mailing list for the IPMC to vote for the release. +1. Wait for 72 hours for the vote result. Use the same template as the dev vote, with the addition + of links to the dev vote and result mail thread. +1. After the vote in general@ is completed with at least three +1 binding votes, close the vote by + replying to the voting thread. + +### Release the Staging Repository in Artifactory +Release the artifact bundle in Artifactory: + +1. Login to <https://repository.apache.org>. +1. Go to "Staging Repos". +1. Find the "orgapachetephra" repo with the Tephra release. Be sure to expand the contents of the + repo to confirm that it contains the correct Tephra artifacts. +1. Click on the "Release" button at top, and enter a brief description, such as "Apache Tephra N.N.N + release". + +## Announcing and Completing the Release +Mark the release as complete in JIRA (in the Apache Tephra Administration section): + +1. Add a release for the next version, if necessary +1. Set a release date and release the released version + +Release the source tarball: + +1. Copy the release artifacts and CHANGES.txt from the dev to release directory at + `dist.apache.org/repos/dist/release/incubator/tephra/${RELEASE_VERSION}-incubating` + +Finally, announce the release on the mailing lists: dev@tephra and announce@ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md new file mode 100644 index 0000000..7247389 --- /dev/null +++ b/src/site/markdown/index.md @@ -0,0 +1,116 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<head> + <title>Home</title> +</head> + +## What is Apache Tephra <sup>(TM)</sup> +__Transactions for Apache HBase <sup>(TM)</sup>__: Apache Tephra provides globally consistent +transactions on top of Apache HBase. While HBase provides strong consistency with row- or +region-level ACID operations, it sacrifices cross-region and cross-table consistency in favor of +scalability. This trade-off requires application developers to handle the complexity of ensuring +consistency when their modifications span region boundaries. By providing support for global +transactions that span regions, tables, or multiple RPCs, Tephra simplifies application development +on top of HBase, without a significant impact on performance or scalability for many workloads. + +## How It Works +Tephra leverages HBase's native data versioning to provide multi-versioned concurrency +control (MVCC) for transactional reads and writes. With MVCC capability, each transaction +sees its own consistent "snapshot" of data, providing +[snapshot isolation](http://en.wikipedia.org/wiki/Snapshot_isolation) of concurrent transactions. + +Tephra consists of three main components: + +- __Transaction Server__ - maintains global view of transaction state, assigns new transaction IDs + and performs conflict detection; +- __Transaction Client__ - coordinates start, commit, and rollback of transactions; and +- __TransactionProcessor Coprocessor__ - applies filtering to the data read (based on a + given transaction's state) and cleans up any data from old (no longer visible) transactions. + +### Transaction Server +A central transaction manager generates a globally unique, time-based transaction ID for each +transaction that is started, and maintains the state of all in-progress and recently committed +transactions for conflict detection. While multiple transaction server instances can be run +concurrently for automatic failover, only one server instance is actively serving requests at a +time. This is coordinated by performing leader election amongst the running instances through +[Apache ZooKeeper](https://zookeeper.apache.org). The active transaction server instance will +also register itself using a service discovery interface in ZooKeeper, allowing clients to +discover the currently active server instance without additional configuration. + +### Transaction Client +A client makes a call to the active transaction server in order to start a new transaction. This +returns a new transaction instance to the client, with a unique transaction ID (used to identify +writes for the transaction), as well as a list of transaction IDs to exclude for reads (from +in-progress or invalidated transactions). When performing writes, the client overrides the +timestamp for all modified HBase cells with the transaction ID. When reading data from HBase, the +client skips cells associated with any of the excluded transaction IDs. The read exclusions are +applied through a server-side filter injected by the `TransactionProcessor` coprocessor. + +### TransactionProcessor Coprocessor +The `TransactionProcessor` coprocessor is loaded on all HBase tables where transactional reads +and writes are performed. When clients read data, it coordinates the server-side filtering +performed based on the client transaction's snapshot. Data cells from any transactions that are +currently in-progress or those that have failed and could not be rolled back ("invalid" +transactions) will be skipped on these reads. In addition, the `TransactionProcessor` cleans +up any data versions that are no longer visible to any running transactions, either because the +transaction that the cell is associated with failed or a write from a newer transaction was +successfully committed to the same column. + +More details on how Tephra transactions work and the interactions between these components can be +found in our [Transactions over HBase](http://www.slideshare.net/alexbaranau/transactions-over-hbase) +presentation. + +## Is It Building? +Status of CI build at Travis CI: [](https://travis-ci.org/apache/incubator-tephra) + +## Requirements +### Java Runtime +The latest [JDK or JRE version 1.7.xx or 1.8.xx](http://www.java.com/en/download/manual.jsp) +for Linux, Windows, or Mac OS X must be installed in your environment; we recommend the Oracle JDK. + +To check the Java version installed, run the command: + + $ java -version + +Tephra is tested with the Oracle JDKs; it may work with other JDKs such as +[Open JDK](http://openjdk.java.net), but it has not been tested with them. + +Once you have installed the JDK, you'll need to set the `JAVA_HOME` environment variable. + +### Hadoop/HBase Environment +Tephra requires a working HBase and HDFS environment in order to operate. Tephra supports these +component versions: + +| Component | Source | Supported Versions | +|---------------|---------------|---------------------------------------------------------| +| __HDFS__ | Apache Hadoop | 2.0.2-alpha through 2.7.x | +| | CDH or HDP | (CDH) 5.0.0 through 5.7.0 or (HDP) 2.0, 2.1, 2.2 or 2.3 | +| | MapR | 4.1 (with MapR-FS) | +| __HBase__ | Apache | 0.96.x, 0.98.x, 1.0.x, 1.1.x and 1.2.x | +| | CDH or HDP | (CDH) 5.0.0 through 5.7.0 or (HDP) 2.0, 2.1, 2.2 or 2.3 | +| | MapR | 4.1 (with Apache HBase) | +| __ZooKeeper__ | Apache | Version 3.4.3 through 3.4.5 | +| | CDH or HDP | (CDH) 5.0.0 through 5.7.0 or (HDP) 2.0, 2.1, 2.2 or 2.3 | +| | MapR | 4.1 | + +__Note:__ Components versions shown in this table are those that we have tested and are +confident of their suitability and compatibility. Later versions of components may work, +but have not necessarily have been either tested or confirmed compatible. + +Ready to try out Apache Tephra? Checkout the [Getting Started Guide](GettingStarted.html) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/markdown/releases/0.8.0-incubating.md ---------------------------------------------------------------------- diff --git a/src/site/markdown/releases/0.8.0-incubating.md b/src/site/markdown/releases/0.8.0-incubating.md new file mode 100644 index 0000000..68fc26f --- /dev/null +++ b/src/site/markdown/releases/0.8.0-incubating.md @@ -0,0 +1,28 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<head> + <title>Apache Tephra Release 0.8.0-incubating</title> +</head> + +### Release Notes - Apache Tephra - 0.8.0-incubating + +This is the first Apache Incubator release of Tephra. + +### Source tar-ball download +[Source and signatures](http://www.apache.org/dyn/closer.cgi/incubator/tephra/0.8.0-incubating/src) http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/resources/images/tephra_logotype.png ---------------------------------------------------------------------- diff --git a/src/site/resources/images/tephra_logotype.png b/src/site/resources/images/tephra_logotype.png new file mode 100644 index 0000000..bfcd174 Binary files /dev/null and b/src/site/resources/images/tephra_logotype.png differ http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a764587e/src/site/site.xml ---------------------------------------------------------------------- diff --git a/src/site/site.xml b/src/site/site.xml new file mode 100644 index 0000000..ec74a49 --- /dev/null +++ b/src/site/site.xml @@ -0,0 +1,105 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project name="Apache Tephra"> + + <bannerLeft> + <name>Apache Tephra</name> + <src>images/tephra_logotype.png</src> + <href>http://tephra.incubator.apache.org/index.html</href> + <alt>Apache Tephra</alt> + </bannerLeft> + <bannerRight> + <src>http://incubator.apache.org/images/egg-logo.png</src> + <href>http://incubator.apache.org/</href> + </bannerRight> + + <publishDate position="none"/> + <version position="none"/> + + <skin> + <groupId>lt.velykis.maven.skins</groupId> + <artifactId>reflow-maven-skin</artifactId> + <version>1.1.1</version> + </skin> + + <body> + <head> + <link rel="stylesheet" href="http://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.4.0/styles/default.min.css"/> + </head> + + <breadcrumbs position="left"> + <item name="Apache Tephra" href="http://tephra.incubator.apache.org/index.html"/> + </breadcrumbs> + + <menu name="Documentation"> + <item name="Home" href="./index.html"/> + <item name="Getting Started" href="./GettingStarted.html"/> + <item name="API (0.9.0-incubating-SNAPSHOT)" href="./apidocs/index.html"/> + </menu> + + <menu name="Releases"> + <item name="0.8.0-incubating" href="releases/0.8.0-incubating.html"/> + </menu> + + <menu name="Get Involved"> + <item name="How to Contribute" href="HowToContribute.html"/> + <item name="Release Guide" href="ReleaseGuide.html"/> + <item name="Mailing Lists" href="mail-lists.html"/> + <item name="Sources" href="source-repository.html"/> + <item name="Issues" href="issue-tracking.html"/> + <item name="Team" href="team-list.html"/> + </menu> + + <menu name="ASF"> + <item name="Apache Incubator" href="http://incubator.apache.org/"/> + <item name="Apache Foundation" href="http://www.apache.org/foundation/"/> + <item name="How Apache Works" href="http://www.apache.org/foundation/how-it-works.html"/> + <item name="The Apache License" href="http://www.apache.org/licenses/LICENSE-2.0.html"/> + <item name="Sponsoring Apache" href="http://www.apache.org/foundation/sponsorship.html"/> + <item name="Thanks" href="http://www.apache.org/foundation/thanks.html"/> + </menu> + + <footer> + <div> + Apache Tephra, Apache, the Apache feather logo, + and the Apache Tephra project logos are trademarks of The Apache Software Foundation. + All other marks mentioned may be trademarks or registered trademarks of their respective owners. + </div> + <script src="http://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.4.0/highlight.min.js";></script> + </footer> + + </body> + + <custom> + <reflowSkin> + <theme>bootswatch-simplex</theme> + <smoothScroll>true</smoothScroll> + <markPageHeader>false</markPageHeader> + <navbarInverse>true</navbarInverse> + <bottomNav maxSpan="9" > + <column>Documentation</column> + <column>Releases</column> + <column>Get Involved</column> + <column>ASF</column> + </bottomNav> + <bottomDescription></bottomDescription> + </reflowSkin> + </custom> + +</project>
