JAMES-2588 Effectively remove HBase
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/81d65f1c Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/81d65f1c Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/81d65f1c Branch: refs/heads/master Commit: 81d65f1cf676b223e07decb86c102eb2c6052c67 Parents: 546d63f Author: Raphael Ouazana <raphael.ouaz...@linagora.com> Authored: Wed Nov 7 17:40:16 2018 +0100 Committer: Raphael Ouazana <raphael.ouaz...@linagora.com> Committed: Thu Nov 8 11:42:43 2018 +0100 ---------------------------------------------------------------------- mailbox/README.md | 11 - mailbox/hbase/pom.xml | 148 ---- mailbox/hbase/src/main/config/hbase-site.xml | 49 -- .../james/mailbox/hbase/FlagConvertor.java | 134 --- .../org/apache/james/mailbox/hbase/HBaseId.java | 86 -- .../hbase/HBaseMailboxIdDeserializer.java | 37 - .../mailbox/hbase/HBaseMailboxManager.java | 107 --- .../hbase/HBaseMailboxSessionMapperFactory.java | 177 ---- .../mailbox/hbase/HBaseMessageManager.java | 70 -- .../apache/james/mailbox/hbase/HBaseNames.java | 84 -- .../hbase/HBaseNonTransactionalMapper.java | 35 - .../apache/james/mailbox/hbase/HBaseUtils.java | 466 ---------- .../james/mailbox/hbase/PropertyConvertor.java | 74 -- .../mailbox/hbase/io/ChunkInputStream.java | 100 --- .../mailbox/hbase/io/ChunkOutputStream.java | 108 --- .../mailbox/hbase/mail/HBaseMailboxMapper.java | 357 -------- .../mailbox/hbase/mail/HBaseMailboxMessage.java | 364 -------- .../mailbox/hbase/mail/HBaseMessageMapper.java | 772 ----------------- .../mailbox/hbase/mail/HBaseModSeqProvider.java | 113 --- .../mailbox/hbase/mail/HBaseUidProvider.java | 94 --- .../mailbox/hbase/mail/model/HBaseMailbox.java | 198 ----- .../hbase/user/HBaseSubscriptionMapper.java | 159 ---- .../resources/META-INF/spring/mailbox-hbase.xml | 68 -- mailbox/hbase/src/reporting-site/site.xml | 29 - .../images/james-hbase-mailbox-schema.mm | 130 --- .../images/james-hbase-mailbox-schema.png | Bin 118995 -> 0 bytes .../images/james-hbase-mailbox-schema.svg | 842 ------------------- .../mailbox/hbase/HBaseClusterSingleton.java | 175 ---- .../hbase/HBaseMailboxIdDeserializerTest.java | 53 -- .../hbase/HBaseMailboxManagerStressTest.java | 111 --- .../mailbox/hbase/HBaseMailboxManagerTest.java | 112 --- .../HBaseMailboxSessionMapperFactoryTest.java | 152 ---- .../james/mailbox/hbase/HBaseUtilsTest.java | 159 ---- .../hbase/mail/HBaseMailboxMapperTest.java | 353 -------- .../mail/HBaseMailboxMessageMapperTest.java | 241 ------ .../mail/HBaseUidAndModSeqProviderTest.java | 196 ----- .../hbase/mail/model/HBaseMailboxTest.java | 155 ---- .../hbase/user/HBaseSubscriptionMapperTest.java | 211 ----- .../test/resources/hadoop-metrics2.properties | 8 - mailbox/hbase/src/test/resources/hdfs-site.xml | 33 - mailbox/pom.xml | 1 - mailbox/spring/pom.xml | 25 - .../META-INF/spring/spring-mailbox.xml | 2 - .../spring/src/test/resources/hbase-site.xml | 49 -- mpt/impl/imap-mailbox/hbase/LICENSE.txt | 176 ---- mpt/impl/imap-mailbox/hbase/NOTICE.txt | 11 - mpt/impl/imap-mailbox/hbase/pom.xml | 69 -- .../hbase/src/reporting-site/site.xml | 28 - .../hbase/HBaseAuthenticatePlainTest.java | 52 -- .../hbase/HBaseAuthenticatedStateTest.java | 52 -- .../hbase/HBaseConcurrentSessionsTest.java | 52 -- .../imapmailbox/hbase/HBaseCondstoreTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseCopyTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseEventsTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseExpungeTest.java | 52 -- .../hbase/HBaseFetchBodySectionTest.java | 52 -- .../hbase/HBaseFetchBodyStructureTest.java | 52 -- .../hbase/HBaseFetchHeadersTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseFetchTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseListingTest.java | 52 -- .../hbase/HBaseMailboxAnnotationTest.java | 52 -- .../HBaseMailboxWithLongNameErrorTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseMoveTest.java | 51 -- .../hbase/HBaseNonAuthenticatedStateTest.java | 52 -- .../hbase/HBasePartialFetchTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseQuotaTest.java | 46 - .../mpt/imapmailbox/hbase/HBaseRenameTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseSearchTest.java | 52 -- .../imapmailbox/hbase/HBaseSecurityTest.java | 52 -- .../mpt/imapmailbox/hbase/HBaseSelectTest.java | 52 -- .../hbase/HBaseSelectedInboxTest.java | 52 -- .../hbase/HBaseSelectedStateTest.java | 52 -- .../hbase/HBaseUidSearchOnIndexTest.java | 52 -- .../imapmailbox/hbase/HBaseUidSearchTest.java | 52 -- .../hbase/HBaseUserFlagsSupportTest.java | 52 -- .../imapmailbox/hbase/host/HBaseHostSystem.java | 190 ----- .../hbase/src/test/resources/logback-test.xml | 22 - mpt/impl/imap-mailbox/pom.xml | 7 - mpt/pom.xml | 1 - pom.xml | 40 - server/Overview.md | 1 - server/app/pom.xml | 31 - server/app/src/main/resources/mailbox.xml | 2 +- server/app/src/test/resources/mailbox.xml | 2 +- .../src/main/resources/mailbox.xml | 4 +- ...oxConfigurationBeanFactoryPostProcessor.java | 8 +- .../org/apache/james/spring-mailbox-hbase.xml | 61 -- server/data/data-hbase/pom.xml | 190 ----- .../james/domainlist/hbase/HBaseDomainList.java | 125 --- .../james/domainlist/hbase/def/HDomainList.java | 38 - .../rrt/hbase/HBaseRecipientRewriteTable.java | 241 ------ .../rrt/hbase/def/HRecipientRewriteTable.java | 38 - .../apache/james/system/hbase/TablePool.java | 133 --- .../james/user/hbase/HBaseUsersRepository.java | 215 ----- .../james/user/hbase/def/HUsersRepository.java | 38 - .../data/data-hbase/src/reporting-site/site.xml | 29 - .../domainlist/hbase/HBaseDomainListTest.java | 78 -- .../hbase/HBaseRecipientRewriteTableTest.java | 57 -- .../apache/james/rrt/hbase/HBaseStepdefs.java | 67 -- .../james/rrt/hbase/RewriteTablesTest.java | 32 - .../james/system/hbase/TablePoolTest.java | 61 -- .../user/hbase/HBaseUsersRepositoryTest.java | 82 -- .../test/resources/hadoop-metrics2.properties | 8 - .../src/test/resources/log4j.properties | 23 - .../features/src/main/resources/features.xml | 4 - server/pom.xml | 1 - src/site/site.xml | 1 - src/site/xdoc/mailbox/index.xml | 1 - src/site/xdoc/mailbox/mailbox-api.xml | 3 +- src/site/xdoc/mailbox/mailbox-hbase.xml | 143 ---- src/site/xdoc/mailbox/mailbox-spring.xml | 1 - src/site/xdoc/mailbox/source-code.xml | 3 +- 112 files changed, 7 insertions(+), 10503 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/README.md ---------------------------------------------------------------------- diff --git a/mailbox/README.md b/mailbox/README.md index 6b08d36..4ef2312 100644 --- a/mailbox/README.md +++ b/mailbox/README.md @@ -13,7 +13,6 @@ Apache James Mailbox has the following project (Maven) structure: ~~~ |-- api -- Mailbox API -|-- hbase -- Mailbox implementation over HBase |-- jcr -- Mailbox implementation over Java Content Repository (JCR) |-- jpa -- Database Mailbox implementation using Java Persistence API |-- lucene -- Email indexing module with Apache Lucene @@ -51,16 +50,6 @@ Implements the Maildir standard for email storage (http://en.wikipedia.org/wiki/ *Nix systems. -Mailbox HBase -============= - -Uses Apache HBase (http://hbase.apache.org/) for storing email messages. Provides a scalable email storage. To have a fully -distributed email server you will also need, among others: - -* distributed UID generation, look at Zookeeper Sequence Provider (**zoo-seq-provider**) for distributed locking and Mailbox manipulation -* distributed SMTP/IMAP access -* other - Zookeeper Sequence Provider ========================== http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/mailbox/hbase/pom.xml b/mailbox/hbase/pom.xml deleted file mode 100644 index 7c36b6c..0000000 --- a/mailbox/hbase/pom.xml +++ /dev/null @@ -1,148 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.james</groupId> - <artifactId>apache-james-mailbox</artifactId> - <version>3.2.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <artifactId>apache-james-mailbox-hbase</artifactId> - <name>Apache James :: Mailbox :: HBase</name> - <description>Apache James Mailbox implementation over HBase</description> - - <dependencies> - <dependency> - <groupId>${james.groupId}</groupId> - <artifactId>apache-james-mailbox-api</artifactId> - </dependency> - <dependency> - <groupId>${james.groupId}</groupId> - <artifactId>apache-james-mailbox-api</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>${james.groupId}</groupId> - <artifactId>apache-james-mailbox-store</artifactId> - </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>11.0.2</version><!--$NO-MVN-MAN-VER$--> - </dependency> - <dependency> - <groupId>com.sun.mail</groupId> - <artifactId>javax.mail</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> - <exclusions> - <exclusion> - <groupId>org.jruby</groupId> - <artifactId>jruby-complete</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> - <type>test-jar</type> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>org.jruby</groupId> - <artifactId>jruby-complete</artifactId> - </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.assertj</groupId> - <artifactId>assertj-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <reuseForks>true</reuseForks> - </configuration> - </plugin> - </plugins> - </build> - -</project> http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/config/hbase-site.xml ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/config/hbase-site.xml b/mailbox/hbase/src/main/config/hbase-site.xml deleted file mode 100644 index a5183f7..0000000 --- a/mailbox/hbase/src/main/config/hbase-site.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Copyright 2011 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration> -<!-- - This config file must be on the application classpath and will tell our application - where to find the HBase cluster. ---> - <property> - <name>hbase.rootdir</name> - <value>hdfs://localhost:9000/hbase</value> - </property> - - <property> - <name>hbase.master.port</name> - <value>60000</value> - </property> - - <property> - <name>hbase.regionserver.info.port</name> - <value>6030</value> - </property> - - <property> - <name>hbase.regionserver.info.bindAddress</name> - <value>0.0.0.0</value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/FlagConvertor.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/FlagConvertor.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/FlagConvertor.java deleted file mode 100644 index 5165c4a..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/FlagConvertor.java +++ /dev/null @@ -1,134 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import javax.mail.Flags; -import javax.mail.Flags.Flag; - -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Class used for converting message flags to and from byte arrays for use as - * HBase column qualifiers. - */ -public class FlagConvertor { - - public static final String PREFIX_SFLAGS = "sf:"; - public static final byte[] PREFIX_SFLAGS_B = Bytes.toBytes(PREFIX_SFLAGS); - public static final String PREFIX_UFLAGS = "uf:"; - public static final byte[] PREFIX_UFLAGS_B = Bytes.toBytes(PREFIX_UFLAGS); - /*TODO: find a way to store all flags as a single byte (HBase bitewise operations). - * this will be efficient also because we will not store the column names. - */ - public static final byte[] FLAGS_ANSWERED = Bytes.toBytes("sf:A"); - public static final byte[] FLAGS_DELETED = Bytes.toBytes("sf:DE"); - public static final byte[] FLAGS_DRAFT = Bytes.toBytes("sf:DR"); - public static final byte[] FLAGS_FLAGGED = Bytes.toBytes("sf:F"); - public static final byte[] FLAGS_RECENT = Bytes.toBytes("sf:R"); - public static final byte[] FLAGS_SEEN = Bytes.toBytes("sf:S"); - public static final byte[] FLAGS_USER = Bytes.toBytes("sf:U"); - - /** - * Converts a {@link javax.mail.Flags.Flag} to a byte array representation - * used for storing in HBase (as a column qualifier). - * @param flag - * @return a byte representation of the flag. - * - * Throws RuntimeException if Flag is not recognized. - */ - public static byte[] systemFlagToBytes(Flags.Flag flag) { - if (flag.equals(Flag.ANSWERED)) { - return FLAGS_ANSWERED; - } - if (flag.equals(Flag.DELETED)) { - return FLAGS_DELETED; - } - if (flag.equals(Flag.DRAFT)) { - return FLAGS_DRAFT; - } - if (flag.equals(Flag.FLAGGED)) { - return FLAGS_FLAGGED; - } - if (flag.equals(Flag.RECENT)) { - return FLAGS_RECENT; - } - if (flag.equals(Flag.SEEN)) { - return FLAGS_SEEN; - } - if (flag.equals(Flag.USER)) { - return FLAGS_USER; - } - throw new RuntimeException("Invalid Flag supplied"); - } - - /** - * Returns a {@link javax.mail.Flags.Flag} coresponding to the supplyed - * byte array. - * @param bytes byte array representation - * @return one of {@link javax.mail.Flags.Flag} - * @throws RuntimeException if the byte array does not match a - * suitable represetnation. - */ - public static Flag systemFlagFromBytes(byte[] bytes) { - if (Bytes.equals(bytes, FLAGS_ANSWERED)) { - return Flag.ANSWERED; - } - if (Bytes.equals(bytes, FLAGS_DELETED)) { - return Flag.DELETED; - } - if (Bytes.equals(bytes, FLAGS_DRAFT)) { - return Flag.DRAFT; - } - if (Bytes.equals(bytes, FLAGS_FLAGGED)) { - return Flag.FLAGGED; - } - if (Bytes.equals(bytes, FLAGS_RECENT)) { - return Flag.RECENT; - } - if (Bytes.equals(bytes, FLAGS_SEEN)) { - return Flag.SEEN; - } - if (Bytes.equals(bytes, FLAGS_USER)) { - return Flag.USER; - } - throw new RuntimeException("This is not a recognized system flag: " + Bytes.toString(bytes)); - } - - /** - * Converts a user flag to a byte array for use as a HBase column qualifier. - * @param flag user flag to convert - * @return a byte array representation of the user flag - */ - public static byte[] userFlagToBytes(String flag) { - return Bytes.toBytes(PREFIX_UFLAGS + flag); - } - - /** - * Converts a byte array to a user flag. - * @param bytes the user flag byte representation - * @return a {@link String} representaion of the user flag - * @throws RuntimeException if the user flag prefix is not found. - */ - public static String userFlagFromBytes(byte[] bytes) { - if (Bytes.startsWith(bytes, PREFIX_UFLAGS_B)) { - return Bytes.toString(bytes, PREFIX_UFLAGS_B.length, bytes.length - PREFIX_UFLAGS_B.length); - } - throw new RuntimeException("This is not a user flag representation: " + Bytes.toString(bytes)); - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseId.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseId.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseId.java deleted file mode 100644 index 8442522..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseId.java +++ /dev/null @@ -1,86 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import java.io.Serializable; -import java.util.UUID; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.james.mailbox.model.MailboxId; - -public class HBaseId implements MailboxId, Serializable { - - private final UUID id; - - public static HBaseId of(UUID id) { - return new HBaseId(id); - } - - private HBaseId(UUID id) { - this.id = id; - } - - public UUID getRawId() { - return id; - } - - @Override - public String serialize() { - return id.toString(); - } - - public byte[] toBytes() { - return Bytes.add( - Bytes.toBytes(id.getMostSignificantBits()), - Bytes.toBytes(id.getLeastSignificantBits())); - } - - @Override - public String toString() { - return id.toString(); - } - - @Override - public int hashCode() { - return (int) ((id == null) ? 0 : (id.getMostSignificantBits() ^ (id.getMostSignificantBits() >>> 32))); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - HBaseId other = (HBaseId) obj; - if (id == null) { - if (other.id != null) { - return false; - } - } else if (!id.equals(other.id)) { - return false; - } - return true; - } - -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxIdDeserializer.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxIdDeserializer.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxIdDeserializer.java deleted file mode 100644 index ac3f7fd..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxIdDeserializer.java +++ /dev/null @@ -1,37 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.mailbox.hbase; - -import java.util.UUID; - -import org.apache.james.mailbox.store.mail.model.MailboxIdDeserialisationException; -import org.apache.james.mailbox.store.mail.model.MailboxIdDeserializer; - -public class HBaseMailboxIdDeserializer implements MailboxIdDeserializer { - - @Override - public HBaseId deserialize(String serializedMailboxId) throws MailboxIdDeserialisationException { - try { - return HBaseId.of(UUID.fromString(serializedMailboxId)); - } catch (Exception e) { - throw new MailboxIdDeserialisationException("Error deserializing " + serializedMailboxId, e); - } - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java deleted file mode 100644 index 902b4dd..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxManager.java +++ /dev/null @@ -1,107 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import java.util.EnumSet; - -import org.apache.james.mailbox.MailboxPathLocker; -import org.apache.james.mailbox.MailboxSession; -import org.apache.james.mailbox.exception.MailboxException; -import org.apache.james.mailbox.hbase.mail.HBaseMailboxMapper; -import org.apache.james.mailbox.hbase.mail.model.HBaseMailbox; -import org.apache.james.mailbox.model.MailboxPath; -import org.apache.james.mailbox.model.MessageId; -import org.apache.james.mailbox.store.Authenticator; -import org.apache.james.mailbox.store.Authorizator; -import org.apache.james.mailbox.store.StoreMailboxAnnotationManager; -import org.apache.james.mailbox.store.StoreMailboxManager; -import org.apache.james.mailbox.store.StoreMessageManager; -import org.apache.james.mailbox.store.StoreRightManager; -import org.apache.james.mailbox.store.event.DelegatingMailboxListener; -import org.apache.james.mailbox.store.event.MailboxEventDispatcher; -import org.apache.james.mailbox.store.mail.model.Mailbox; -import org.apache.james.mailbox.store.mail.model.impl.MessageParser; -import org.apache.james.mailbox.store.transaction.Mapper; - -/** - * HBase implementation of {@link StoreMailboxManager} - * - * @Deprecated: See JAMES-2571 The HBase implementation is not enough maintained and thus will be removed from James in version 3.3.0. Please prefer using - * other MailboxManagers or contribute the HBase mailbox maintenance. - * - */ -@Deprecated -public class HBaseMailboxManager extends StoreMailboxManager { - - public static final EnumSet<MailboxCapabilities> MAILBOX_CAPABILITIES = EnumSet.of(MailboxCapabilities.Namespace); - - public HBaseMailboxManager(HBaseMailboxSessionMapperFactory mapperFactory, - Authenticator authenticator, - Authorizator authorizator, - MailboxPathLocker locker, - MessageParser messageParser, - MessageId.Factory messageIdFactory, - MailboxEventDispatcher dispatcher, - DelegatingMailboxListener delegatingMailboxListener, - StoreMailboxAnnotationManager annotationManager, - StoreRightManager storeRightManager) { - super(mapperFactory, authenticator, authorizator, locker, messageParser, messageIdFactory, - annotationManager, dispatcher, delegatingMailboxListener, storeRightManager); - } - - @Override - protected Mailbox doCreateMailbox(MailboxPath mailboxPath, MailboxSession session) { - return new HBaseMailbox(mailboxPath, randomUidValidity()); - } - - /** - * Delete all mailboxes - * - * @param mailboxSession - * @throws MailboxException - */ - public void deleteEverything(MailboxSession mailboxSession) throws MailboxException { - - HBaseMailboxMapper mapper = (HBaseMailboxMapper) getMapperFactory().getMailboxMapper(mailboxSession); - - mapper.execute(Mapper.toTransaction(mapper::deleteAllMemberships)); - mapper.execute(Mapper.toTransaction(mapper::deleteAllMailboxes)); - } - - @Override - public EnumSet<MailboxCapabilities> getSupportedMailboxCapabilities() { - return MAILBOX_CAPABILITIES; - } - - @Override - protected StoreMessageManager createMessageManager(Mailbox mailboxRow, MailboxSession session) { - return new HBaseMessageManager(getMapperFactory(), - getMessageSearchIndex(), - getEventDispatcher(), - getLocker(), - mailboxRow, - getQuotaManager(), - getQuotaRootResolver(), - getMessageParser(), - getMessageIdFactory(), - getBatchSizes(), - getImmutableMailboxMessageFactory(), - getStoreRightManager()); - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxSessionMapperFactory.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxSessionMapperFactory.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxSessionMapperFactory.java deleted file mode 100644 index e6e67b3..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMailboxSessionMapperFactory.java +++ /dev/null @@ -1,177 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOXES_TABLE; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_CF; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGES_META_CF; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGES_TABLE; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_DATA_BODY_CF; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_DATA_HEADERS_CF; -import static org.apache.james.mailbox.hbase.HBaseNames.SUBSCRIPTIONS_TABLE; -import static org.apache.james.mailbox.hbase.HBaseNames.SUBSCRIPTION_CF; - -import java.io.IOException; - -import org.apache.commons.lang.NotImplementedException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.james.mailbox.MailboxSession; -import org.apache.james.mailbox.exception.MailboxException; -import org.apache.james.mailbox.exception.SubscriptionException; -import org.apache.james.mailbox.hbase.mail.HBaseMailboxMapper; -import org.apache.james.mailbox.hbase.mail.HBaseMessageMapper; -import org.apache.james.mailbox.hbase.user.HBaseSubscriptionMapper; -import org.apache.james.mailbox.model.MessageId; -import org.apache.james.mailbox.model.MessageId.Factory; -import org.apache.james.mailbox.store.MailboxSessionMapperFactory; -import org.apache.james.mailbox.store.mail.AnnotationMapper; -import org.apache.james.mailbox.store.mail.MailboxMapper; -import org.apache.james.mailbox.store.mail.MessageIdMapper; -import org.apache.james.mailbox.store.mail.MessageMapper; -import org.apache.james.mailbox.store.mail.ModSeqProvider; -import org.apache.james.mailbox.store.mail.UidProvider; -import org.apache.james.mailbox.store.user.SubscriptionMapper; - -/** - * HBase implementation of {@link MailboxSessionMapperFactory} - * - * @Deprecated: See JAMES-2571 The HBase implementation is not enough maintained and thus will be removed from James in version 3.3.0. Please prefer using - * other MailboxManagers or contribute the HBase mailbox maintenance. - */ -@Deprecated -public class HBaseMailboxSessionMapperFactory extends MailboxSessionMapperFactory { - - private final Configuration conf; - private final UidProvider uidProvider; - private final ModSeqProvider modSeqProvider; - private Factory messageIdFactory; - - /** - * Creates the necessary tables in HBase if they do not exist. - * - * @param conf Configuration for the cluster - * @param uidProvider UID provider for mailbox uid. - * @param modSeqProvider - * @throws MasterNotRunningException - * @throws ZooKeeperConnectionException - * @throws IOException - */ - public HBaseMailboxSessionMapperFactory(Configuration conf, UidProvider uidProvider, ModSeqProvider modSeqProvider, MessageId.Factory messageIdFactory) { - this.conf = conf; - this.uidProvider = uidProvider; - this.modSeqProvider = modSeqProvider; - this.messageIdFactory = messageIdFactory; - - //TODO: add better exception handling for this - try (HBaseAdmin hbaseAdmin = new HBaseAdmin(conf)) { - HTableDescriptor desc = null; - HColumnDescriptor hColumnDescriptor = null; - - /* create the tables if it does not exist */ - - if (!hbaseAdmin.tableExists(MAILBOXES_TABLE)) { - desc = new HTableDescriptor(MAILBOXES_TABLE); - hColumnDescriptor = new HColumnDescriptor(MAILBOX_CF); - hColumnDescriptor.setMaxVersions(1); - desc.addFamily(hColumnDescriptor); - hbaseAdmin.createTable(desc); - } - - if (!hbaseAdmin.tableExists(MESSAGES_TABLE)) { - /**TODO: try to reduce the number of column families as suggested by: - * http://hbase.apache.org/book.html#number.of.cfs - * Down to three column families, striking for just two. - */ - desc = new HTableDescriptor(MESSAGES_TABLE); - hColumnDescriptor = new HColumnDescriptor(MESSAGES_META_CF); - hColumnDescriptor.setMaxVersions(1); - desc.addFamily(hColumnDescriptor); - hColumnDescriptor = new HColumnDescriptor(MESSAGE_DATA_HEADERS_CF); - hColumnDescriptor.setMaxVersions(1); - desc.addFamily(hColumnDescriptor); - hColumnDescriptor = new HColumnDescriptor(MESSAGE_DATA_BODY_CF); - hColumnDescriptor.setMaxVersions(1); - desc.addFamily(hColumnDescriptor); - hbaseAdmin.createTable(desc); - } - - - if (!hbaseAdmin.tableExists(SUBSCRIPTIONS_TABLE)) { - desc = new HTableDescriptor(SUBSCRIPTIONS_TABLE); - hColumnDescriptor = new HColumnDescriptor(SUBSCRIPTION_CF); - hColumnDescriptor.setMaxVersions(1); - desc.addFamily(hColumnDescriptor); - hbaseAdmin.createTable(desc); - } - - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public MessageMapper createMessageMapper(MailboxSession session) throws MailboxException { - return new HBaseMessageMapper(session, uidProvider, modSeqProvider, messageIdFactory, this.conf); - } - - @Override - public MessageIdMapper createMessageIdMapper(MailboxSession session) throws MailboxException { - throw new NotImplementedException(); - } - - @Override - public MailboxMapper createMailboxMapper(MailboxSession session) throws MailboxException { - return new HBaseMailboxMapper(this.conf); - } - - @Override - public SubscriptionMapper createSubscriptionMapper(MailboxSession session) throws SubscriptionException { - return new HBaseSubscriptionMapper(this.conf); - } - - /** - * Returns the configuration object for accessing the cluster. - * @return The configuration for accessing the cluster - */ - public Configuration getClusterConfiguration() { - return conf; - } - - @Override - public ModSeqProvider getModSeqProvider() { - return modSeqProvider; - } - - @Override - public UidProvider getUidProvider() { - return uidProvider; - } - - @Override - public AnnotationMapper createAnnotationMapper(MailboxSession session) - throws MailboxException { - throw new NotImplementedException(); - } - -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java deleted file mode 100644 index 7072590..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseMessageManager.java +++ /dev/null @@ -1,70 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import javax.mail.Flags; - -import org.apache.james.mailbox.MailboxPathLocker; -import org.apache.james.mailbox.MailboxSession; -import org.apache.james.mailbox.model.MessageId; -import org.apache.james.mailbox.quota.QuotaManager; -import org.apache.james.mailbox.quota.QuotaRootResolver; -import org.apache.james.mailbox.store.BatchSizes; -import org.apache.james.mailbox.store.ImmutableMailboxMessage; -import org.apache.james.mailbox.store.MailboxSessionMapperFactory; -import org.apache.james.mailbox.store.StoreMessageManager; -import org.apache.james.mailbox.store.StoreRightManager; -import org.apache.james.mailbox.store.event.MailboxEventDispatcher; -import org.apache.james.mailbox.store.mail.model.Mailbox; -import org.apache.james.mailbox.store.mail.model.impl.MessageParser; -import org.apache.james.mailbox.store.search.MessageSearchIndex; - -/** - * HBase implementation of MessageManager. - * - * @Deprecated: See JAMES-2571 The HBase implementation is not enough maintained and thus will be removed from James in version 3.3.0. Please prefer using - * other MailboxManagers or contribute the HBase mailbox maintenance. - */ -@Deprecated -public class HBaseMessageManager extends StoreMessageManager { - - public HBaseMessageManager(MailboxSessionMapperFactory mapperFactory, - MessageSearchIndex index, - MailboxEventDispatcher dispatcher, - MailboxPathLocker locker, - Mailbox mailbox, - QuotaManager quotaManager, - QuotaRootResolver quotaRootResolver, - MessageParser messageParser, - MessageId.Factory messageIdFactory, - BatchSizes batchSizes, - ImmutableMailboxMessage.Factory immutableMailboxMessageFactory, - StoreRightManager storeRightManager) { - - super(HBaseMailboxManager.DEFAULT_NO_MESSAGE_CAPABILITIES, mapperFactory, index, dispatcher, locker, mailbox, quotaManager, - quotaRootResolver, messageParser, messageIdFactory, batchSizes, immutableMailboxMessageFactory, storeRightManager); - } - - @Override - protected Flags getPermanentFlags(MailboxSession session) { - Flags flags = super.getPermanentFlags(session); - flags.add(Flags.Flag.USER); - return flags; - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNames.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNames.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNames.java deleted file mode 100644 index 69a5066..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNames.java +++ /dev/null @@ -1,84 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Contains table names, column family names, qualifier names and other constants - * for use in HBase. - * - * Each qualifier in the META column will begin with a short prefix that will - * determine it's purpose. </br> - * Qualifier prefix meaning:<br/> - * <ul> - * <li> m: meta information; </li> - * <li> sf: system flag (DELETE, RECENT, etc.) </li> - * <li> uf: user flag </li> - * <li> p: user property</li> - * </ul> - */ -public interface HBaseNames { - - /** - * The HBase table name for storing mailbox names - */ - String MAILBOXES = "JAMES_MAILBOXES"; - byte[] MAILBOXES_TABLE = Bytes.toBytes(MAILBOXES); - /** Default mailbox column family */ - byte[] MAILBOX_CF = Bytes.toBytes("D"); - /** HBase column qualifiers: field names stored as byte arrays*/ - byte[] MAILBOX_NAME = Bytes.toBytes("name"); - byte[] MAILBOX_USER = Bytes.toBytes("user"); - byte[] MAILBOX_NAMESPACE = Bytes.toBytes("namespace"); - byte[] MAILBOX_LASTUID = Bytes.toBytes("lastUID"); - byte[] MAILBOX_UIDVALIDITY = Bytes.toBytes("uidValidity"); - byte[] MAILBOX_HIGHEST_MODSEQ = Bytes.toBytes("hModSeq"); - byte[] MAILBOX_MESSAGE_COUNT = Bytes.toBytes("count"); - /** The HBase table name for storing subscriptions */ - String SUBSCRIPTIONS = "JAMES_SUBSCRIPTIONS"; - /** The HBase table name for storing subscriptions */ - byte[] SUBSCRIPTIONS_TABLE = Bytes.toBytes(SUBSCRIPTIONS); - /** Default subscription column family */ - byte[] SUBSCRIPTION_CF = Bytes.toBytes("D"); - /** The HBase table name for storing messages */ - String MESSAGES = "JAMES_MESSAGES"; - /** The HBase table name for storing messages */ - byte[] MESSAGES_TABLE = Bytes.toBytes(MESSAGES); - /** Column family for storing message meta information*/ - byte[] MESSAGES_META_CF = Bytes.toBytes("M"); - /** Column family for storing message headers*/ - byte[] MESSAGE_DATA_HEADERS_CF = Bytes.toBytes("H"); - /** Column family for storing message body*/ - byte[] MESSAGE_DATA_BODY_CF = Bytes.toBytes("B"); - String PREFIX_META = "m:"; - byte[] PREFIX_META_B = Bytes.toBytes(PREFIX_META); - /** kept sorted */ - byte[] MESSAGE_BODY_OCTETS = Bytes.toBytes(PREFIX_META + "body"); - byte[] MESSAGE_CONTENT_OCTETS = Bytes.toBytes(PREFIX_META + "content"); - byte[] MESSAGE_INTERNALDATE = Bytes.toBytes(PREFIX_META + "date"); - byte[] MESSAGE_TEXT_LINE_COUNT = Bytes.toBytes(PREFIX_META + "lcount"); - byte[] MESSAGE_MODSEQ = Bytes.toBytes(PREFIX_META + "mseq"); - byte[] MESSAGE_MEDIA_TYPE = Bytes.toBytes(PREFIX_META + "mtype"); - byte[] MESSAGE_SUB_TYPE = Bytes.toBytes(PREFIX_META + "stype"); - byte[] MARKER_PRESENT = Bytes.toBytes("X"); - byte[] MARKER_MISSING = Bytes.toBytes(" "); - // the maximum recomended HBase column size is 10 MB - int MAX_COLUMN_SIZE = 1024; //2 * 1024 * 1024; -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNonTransactionalMapper.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNonTransactionalMapper.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNonTransactionalMapper.java deleted file mode 100644 index bcf5990..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseNonTransactionalMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import org.apache.james.mailbox.store.transaction.NonTransactionalMapper; - -/** - * HBase implementation of TransactionMapper. - * I don't know if this class is thread-safe! - * Assume it is not! - * - */ -public class HBaseNonTransactionalMapper extends NonTransactionalMapper { - - @Override - public void endRequest() { - //TODO: maybe do some thing more wise here? - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseUtils.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseUtils.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseUtils.java deleted file mode 100644 index 9c24b92..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/HBaseUtils.java +++ /dev/null @@ -1,466 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_ANSWERED; -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_DELETED; -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_DRAFT; -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_FLAGGED; -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_RECENT; -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_SEEN; -import static org.apache.james.mailbox.hbase.FlagConvertor.FLAGS_USER; -import static org.apache.james.mailbox.hbase.FlagConvertor.PREFIX_SFLAGS_B; -import static org.apache.james.mailbox.hbase.FlagConvertor.PREFIX_UFLAGS_B; -import static org.apache.james.mailbox.hbase.FlagConvertor.systemFlagFromBytes; -import static org.apache.james.mailbox.hbase.FlagConvertor.userFlagFromBytes; -import static org.apache.james.mailbox.hbase.FlagConvertor.userFlagToBytes; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_CF; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_HIGHEST_MODSEQ; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_LASTUID; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_MESSAGE_COUNT; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_NAME; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_NAMESPACE; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_UIDVALIDITY; -import static org.apache.james.mailbox.hbase.HBaseNames.MAILBOX_USER; -import static org.apache.james.mailbox.hbase.HBaseNames.MARKER_MISSING; -import static org.apache.james.mailbox.hbase.HBaseNames.MARKER_PRESENT; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGES_META_CF; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_BODY_OCTETS; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_CONTENT_OCTETS; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_INTERNALDATE; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_MEDIA_TYPE; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_MODSEQ; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_SUB_TYPE; -import static org.apache.james.mailbox.hbase.HBaseNames.MESSAGE_TEXT_LINE_COUNT; -import static org.apache.james.mailbox.hbase.HBaseNames.SUBSCRIPTION_CF; -import static org.apache.james.mailbox.hbase.PropertyConvertor.PREFIX_PROP_B; -import static org.apache.james.mailbox.hbase.PropertyConvertor.getProperty; -import static org.apache.james.mailbox.hbase.PropertyConvertor.getQualifier; -import static org.apache.james.mailbox.hbase.PropertyConvertor.getValue; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.NavigableMap; -import java.util.UUID; - -import javax.mail.Flags; -import javax.mail.Flags.Flag; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.james.mailbox.MessageUid; -import org.apache.james.mailbox.hbase.io.ChunkInputStream; -import org.apache.james.mailbox.hbase.mail.HBaseMailboxMessage; -import org.apache.james.mailbox.hbase.mail.model.HBaseMailbox; -import org.apache.james.mailbox.model.MailboxPath; -import org.apache.james.mailbox.model.MessageId; -import org.apache.james.mailbox.store.mail.model.Mailbox; -import org.apache.james.mailbox.store.mail.model.MailboxMessage; -import org.apache.james.mailbox.store.mail.model.Property; -import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; -import org.apache.james.mailbox.store.user.model.Subscription; - -/** - * HBase utility classes for mailbox and message manipulation. - * @author ieugen - */ -public class HBaseUtils { - // TODO: swith to a bit wise implementation of flags. - - /** - * Creates a Mailbox object from a HBase Result object. - * @param result a result of a HBase Get operation - * @return a Mailbox object - */ - public static Mailbox mailboxFromResult(Result result) { - NavigableMap<byte[], byte[]> rawMailbox = result.getFamilyMap(MAILBOX_CF); - //TODO: should we test for null values? - MailboxPath path = new MailboxPath(Bytes.toString(rawMailbox.get(MAILBOX_NAMESPACE)), - Bytes.toString(rawMailbox.get(MAILBOX_USER)), - Bytes.toString(rawMailbox.get(MAILBOX_NAME))); - - HBaseMailbox mailbox = new HBaseMailbox(path, Bytes.toLong(rawMailbox.get(MAILBOX_UIDVALIDITY))); - mailbox.setMailboxId(hBaseIdFromRowKey(result.getRow())); - mailbox.setHighestModSeq(Bytes.toLong(rawMailbox.get(MAILBOX_HIGHEST_MODSEQ))); - mailbox.setLastUid(Bytes.toLong(rawMailbox.get(MAILBOX_LASTUID))); - mailbox.setMessageCount(Bytes.toLong(rawMailbox.get(MAILBOX_MESSAGE_COUNT))); - return mailbox; - } - - /** - * Returns a UUID from the a byte array. - * @param rowkey - * @return UUID calculated from the byte array - */ - public static HBaseId hBaseIdFromRowKey(byte[] rowkey) { - return HBaseId.of(new UUID(Bytes.toLong(rowkey, 0), Bytes.toLong(rowkey, 8))); - } - - /** - * Transforms the mailbox into a Put operation. - * @return a Put object - */ - public static Put toPut(HBaseMailbox mailbox) { - Put put = new Put(mailbox.getMailboxId().toBytes()); - // we don't store null values and we don't restore them. it's a column based store. - if (mailbox.getName() != null) { - put.add(MAILBOX_CF, MAILBOX_NAME, Bytes.toBytes(mailbox.getName())); - } - - if (mailbox.getUser() != null) { - put.add(MAILBOX_CF, MAILBOX_USER, Bytes.toBytes(mailbox.getUser())); - } - if (mailbox.getNamespace() != null) { - put.add(MAILBOX_CF, MAILBOX_NAMESPACE, Bytes.toBytes(mailbox.getNamespace())); - } - put.add(MAILBOX_CF, MAILBOX_LASTUID, Bytes.toBytes(mailbox.getLastUid())); - put.add(MAILBOX_CF, MAILBOX_UIDVALIDITY, Bytes.toBytes(mailbox.getUidValidity())); - put.add(MAILBOX_CF, MAILBOX_HIGHEST_MODSEQ, Bytes.toBytes(mailbox.getHighestModSeq())); - put.add(MAILBOX_CF, MAILBOX_MESSAGE_COUNT, Bytes.toBytes(mailbox.getMessageCount())); - return put; - } - - /** - * Transforms only the metadata into a Put object. The rest of the message will - * be transfered using multiple Puts if size requires it. - * @param message - * @return a put that contains all metadata information. - */ - public static Put metadataToPut(MailboxMessage message) { - Put put = new Put(messageRowKey(message)); - // we store the message uid and mailbox uid in the row key - // store the meta data - put.add(MESSAGES_META_CF, MESSAGE_MODSEQ, Bytes.toBytes(message.getModSeq())); - put.add(MESSAGES_META_CF, MESSAGE_INTERNALDATE, Bytes.toBytes(message.getInternalDate().getTime())); - put.add(MESSAGES_META_CF, MESSAGE_MEDIA_TYPE, Bytes.toBytes(message.getMediaType())); - put.add(MESSAGES_META_CF, MESSAGE_SUB_TYPE, Bytes.toBytes(message.getSubType())); - put.add(MESSAGES_META_CF, MESSAGE_CONTENT_OCTETS, Bytes.toBytes(message.getFullContentOctets())); - put.add(MESSAGES_META_CF, MESSAGE_BODY_OCTETS, Bytes.toBytes(message.getBodyOctets())); - if (message.getTextualLineCount() != null) { - put.add(MESSAGES_META_CF, MESSAGE_TEXT_LINE_COUNT, Bytes.toBytes(message.getTextualLineCount())); - } - // store system flags in meta and user flags in uflags to avoid name clashes - Flags flags = message.createFlags(); - // system flags - if (flags.contains(Flag.ANSWERED)) { - put.add(MESSAGES_META_CF, FLAGS_ANSWERED, MARKER_PRESENT); - } - if (flags.contains(Flag.DELETED)) { - put.add(MESSAGES_META_CF, FLAGS_DELETED, MARKER_PRESENT); - } - if (flags.contains(Flag.DRAFT)) { - put.add(MESSAGES_META_CF, FLAGS_DRAFT, MARKER_PRESENT); - } - if (flags.contains(Flag.FLAGGED)) { - put.add(MESSAGES_META_CF, FLAGS_FLAGGED, MARKER_PRESENT); - } - if (flags.contains(Flag.RECENT)) { - put.add(MESSAGES_META_CF, FLAGS_RECENT, MARKER_PRESENT); - } - if (flags.contains(Flag.SEEN)) { - put.add(MESSAGES_META_CF, FLAGS_SEEN, MARKER_PRESENT); - } - if (flags.contains(Flag.USER)) { - put.add(MESSAGES_META_CF, FLAGS_USER, MARKER_PRESENT); - } - - // user flags - for (String flag : flags.getUserFlags()) { - put.add(MESSAGES_META_CF, userFlagToBytes(flag), MARKER_PRESENT); - } - int propNumber = 0; - // add the properties - for (Property prop : message.getProperties()) { - put.add(MESSAGES_META_CF, getQualifier(propNumber++), getValue(prop)); - } - - return put; - } - - /** - * Create a row key for a message in a mailbox. The current row key is mailboxID followed by messageID. - * Both values are fixed length so no separator is needed. - * Downside: we will be storing the same message multiple times, one time for each recipient. - * @param message message to get row key from - * @return rowkey byte array that can be used with HBase API - */ - public static byte[] messageRowKey(MailboxMessage message) { - return messageRowKey((HBaseId) message.getMailboxId(), message.getUid()); - } - - /** - * Utility method to build row keys from mailbox UUID and message uid. - * The message uid's are stored in reverse order by substracting the uid value - * from Long.MAX_VALUE. - * @param mailboxUid mailbox UUID - * @param uid message uid - * @return rowkey byte array that can be used with HBase API - */ - public static byte[] messageRowKey(HBaseId mailboxUid, MessageUid uid) { - /** message uid's are stored in reverse order so we will always have the most recent messages first*/ - return Bytes.add(mailboxUid.toBytes(), Bytes.toBytes(Long.MAX_VALUE - uid.asLong())); - } - - /** - * Utility method to build row key min limit from mailbox UUID. - * The message uid's are stored in reverse order by substracting the uid value - * from Long.MAX_VALUE. - * @param mailboxUid mailbox UUID - * @return rowkey byte array that can be used with HBase API - */ - public static byte[] minMessageRowKey(HBaseId mailboxUid) { - return Bytes.add(mailboxUid.toBytes(), Bytes.toBytes(Long.MAX_VALUE)); - } - - - /** - * Utility to build row keys from mailboxUID and a value. The value is added to - * the key without any other operations. - * @param mailboxUid mailbox HBaseId - * @param value - * @return rowkey byte array that can be used with HBase API - */ - public static byte[] customMessageRowKey(HBaseId mailboxUid, MessageUid value) { - return Bytes.add(mailboxUid.toBytes(), Bytes.toBytes(value.asLong())); - } - - /** - * Creates a HBaseMailboxMessage from a Result object. This method retrieves all information - * except for body and header related bytes. The message content will be loaded on demand - * through a specialised InputStream called {@link ChunkInputStream}. - * IMPORTANT: the method expects a single version of each cell. Use setMaxVersions(1). - * @param conf configuration object for HBase cluster - * @param result the result object containing message data - * @return a HBaseMailboxMessage instance with message metadata. - */ - public static MailboxMessage messageMetaFromResult(Configuration conf, Result result, MessageId.Factory messageIdFactory) { - HBaseMailboxMessage message = null; - Flags flags = new Flags(); - List<Property> propList = new ArrayList<>(); - KeyValue[] keys = result.raw(); - String mediaType = null; - String subType = null; - MessageUid uid; - Long modSeq = null; - Long bodyOctets = null; - Long contentOctets = null; - Long textualLineCount = null; - - Date internalDate = null; - - int i = 0; - /** it is VERY IMPORTANT that the byte arrays are kept ascending */ - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_BODY_OCTETS)) { - bodyOctets = Bytes.toLong(keys[i].getValue()); - i++; - } - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_CONTENT_OCTETS)) { - contentOctets = Bytes.toLong(keys[i].getValue()); - i++; - } - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_INTERNALDATE)) { - internalDate = new Date(Bytes.toLong(keys[i].getValue())); - i++; - } - // may be null so it will probably skip - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_TEXT_LINE_COUNT)) { - textualLineCount = Bytes.toLong(keys[i].getValue()); - i++; - } - - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_MODSEQ)) { - modSeq = Bytes.toLong(keys[i].getValue()); - i++; - } - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_MEDIA_TYPE)) { - mediaType = Bytes.toString(keys[i].getValue()); - i++; - } - if (Bytes.equals(keys[i].getQualifier(), MESSAGE_SUB_TYPE)) { - subType = Bytes.toString(keys[i].getValue()); - i++; - } - // only TEXT_LINE_COUNT can be missing if message is binary - if (i < 5) { - throw new RuntimeException("HBase message column names not sorted."); - } - while (i < keys.length) { - //get message properties - if (Bytes.startsWith(keys[i].getQualifier(), PREFIX_PROP_B)) { - propList.add(getProperty(keys[i].getValue())); - } else if (Bytes.startsWith(keys[i].getQualifier(), PREFIX_SFLAGS_B)) { - // get system flags, stored as qualifiers - if (Bytes.equals(MARKER_PRESENT, keys[i].getValue())) { - flags.add(systemFlagFromBytes(keys[i].getQualifier())); - } - } else if (Bytes.startsWith(keys[i].getQualifier(), PREFIX_UFLAGS_B)) { - // get user flags, stored as value qualifier - flags.add(userFlagFromBytes(keys[i].getQualifier())); - } - i++; - } - HBaseId uuid = hBaseIdFromRowKey(result.getRow()); - uid = MessageUid.of(Long.MAX_VALUE - Bytes.toLong(result.getRow(), 16)); - PropertyBuilder props = new PropertyBuilder(propList); - props.setMediaType(mediaType); - props.setSubType(subType); - message = new HBaseMailboxMessage(conf, uuid, messageIdFactory.generate(), internalDate, flags, contentOctets, (int) (contentOctets - bodyOctets), props); - message.setUid(uid); - message.setModSeq(modSeq); - message.setTextualLineCount(textualLineCount); - return message; - } - - /** - * Creates a Put object from this subscription object - * @return Put object suitable for HBase persistence - */ - public static Put toPut(Subscription subscription) { - Put put = new Put(Bytes.toBytes(subscription.getUser())); - put.add(SUBSCRIPTION_CF, Bytes.toBytes(subscription.getMailbox()), MARKER_PRESENT); - return put; - } - - /** - * Utility method to transform message flags into a put opperation. - * @param message - * @param flags - * @return a put object with - */ - public static Put flagsToPut(MailboxMessage message, Flags flags) { - Put put = new Put(messageRowKey(message)); - //system flags - if (flags.contains(Flag.ANSWERED)) { - put.add(MESSAGES_META_CF, FLAGS_ANSWERED, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_ANSWERED, MARKER_MISSING); - } - if (flags.contains(Flag.DELETED)) { - put.add(MESSAGES_META_CF, FLAGS_DELETED, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_DELETED, MARKER_MISSING); - } - if (flags.contains(Flag.DRAFT)) { - put.add(MESSAGES_META_CF, FLAGS_DRAFT, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_DRAFT, MARKER_MISSING); - } - if (flags.contains(Flag.FLAGGED)) { - put.add(MESSAGES_META_CF, FLAGS_FLAGGED, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_FLAGGED, MARKER_MISSING); - } - if (flags.contains(Flag.RECENT)) { - put.add(MESSAGES_META_CF, FLAGS_RECENT, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_RECENT, MARKER_MISSING); - } - if (flags.contains(Flag.SEEN)) { - put.add(MESSAGES_META_CF, FLAGS_SEEN, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_SEEN, MARKER_MISSING); - } - if (flags.contains(Flag.USER)) { - put.add(MESSAGES_META_CF, FLAGS_USER, MARKER_PRESENT); - } else { - put.add(MESSAGES_META_CF, FLAGS_USER, MARKER_MISSING); - } - /**TODO: user flags are not deleted this way: store them all in a single column - * and replace that column full. - */ - // user flags - for (String flag : flags.getUserFlags()) { - put.add(MESSAGES_META_CF, userFlagToBytes(flag), MARKER_PRESENT); - } - return put; - } - - public static Delete flagsToDelete(MailboxMessage message, Flags flags) { - Delete delete = new Delete(messageRowKey(message)); - //we mark for delete flags that are not present (they will be Put'ed) - if (flags.contains(Flag.ANSWERED)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_ANSWERED); - } - if (flags.contains(Flag.DELETED)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_DELETED); - } - if (flags.contains(Flag.DRAFT)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_DRAFT); - } - if (flags.contains(Flag.FLAGGED)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_FLAGGED); - } - if (flags.contains(Flag.RECENT)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_RECENT); - } - if (flags.contains(Flag.SEEN)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_SEEN); - } - if (flags.contains(Flag.USER)) { - delete.deleteColumn(MESSAGES_META_CF, FLAGS_USER); - } - - // we delete all user flags that where not in the new configuration - for (String flag : flags.getUserFlags()) { - delete.deleteColumn(MESSAGES_META_CF, userFlagToBytes(flag)); - } - return delete; - } - - /** - * Returns a String composed of all flags in the parameter. - * @param flags - * @return a string representation of all flags - */ - public static String flagsToString(Flags flags) { - StringBuilder b = new StringBuilder(); - - if (flags.contains(Flag.ANSWERED)) { - b.append("ANSWERED "); - } - if (flags.contains(Flag.DELETED)) { - b.append("DELETED "); - } - if (flags.contains(Flag.DRAFT)) { - b.append("DRAFT "); - } - if (flags.contains(Flag.FLAGGED)) { - b.append("FLAGGED "); - } - if (flags.contains(Flag.RECENT)) { - b.append("RECENT "); - } - if (flags.contains(Flag.SEEN)) { - b.append("SEEN "); - } - if (flags.contains(Flag.USER)) { - b.append("USER "); - } - for (String flag : flags.getUserFlags()) { - b.append(flag); - b.append(" "); - } - return b.toString(); - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/PropertyConvertor.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/PropertyConvertor.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/PropertyConvertor.java deleted file mode 100644 index 8a7d039..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/PropertyConvertor.java +++ /dev/null @@ -1,74 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.james.mailbox.store.mail.model.Property; -import org.apache.james.mailbox.store.mail.model.impl.SimpleProperty; - -/** - * Class used for converting message properties to and from byte arrays for use as - * HBase column qualifiers and values. - */ -public class PropertyConvertor { - - public static final String PREFIX_PROP = "p:"; - public static final byte[] PREFIX_PROP_B = Bytes.toBytes(PREFIX_PROP); - //TODO: find a better separator. - /** The separator must not be part of the property namespace or localName */ - private static final String SEPARATOR = "%%"; - - /** - * Returns a byte array that represents a HBase column qualifier for the - * provided property. - * @param propNumber the property for storage n HBase - * @return a byte array that represents a column qualifier for the property - */ - public static byte[] getQualifier(int propNumber) { - // allow for about 1000 properties to be stored, we pad them because HBase will store them sorted - return Bytes.toBytes(PREFIX_PROP + String.format("%03d", propNumber)); - } - - /** - * Returns a byte array representation of the Property value. - * (uses Bytes.toBytes) - * @param prop - * @return a byte array of the value. - */ - public static byte[] getValue(Property prop) { - return Bytes.toBytes(prop.getNamespace() + SEPARATOR + prop.getLocalName() + SEPARATOR + prop.getValue()); - } - - /** - * Returns a Property from a qualifier byte array. - * @param value - * @return a {@link Property} - * @throws RuntimeException if property prefix or separator is not present - */ - public static Property getProperty(byte[] value) { - String ns = Bytes.toString(value); - //TODO: we assume the SEPARATOR=%% can not appear in a normal property. This may not be true. - String[] parts = ns.split(SEPARATOR); - if (parts.length != 3) { - throw new RuntimeException("Separator not found in qualifier " - + Bytes.toString(value)); - } - return new SimpleProperty(parts[0], parts[1], parts[2]); - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkInputStream.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkInputStream.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkInputStream.java deleted file mode 100644 index 76a5009..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkInputStream.java +++ /dev/null @@ -1,100 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase.io; - -import java.io.IOException; -import java.io.InputStream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Return an InputStream which retrieve columns from a row which stores chunk of - * data. See also {@link ChunkOutputStream} - * - * This implementation is not thread-safe! - * - * Bsed on Hector implementation for Cassandra. - * https://github.com/rantav/hector/blob/master/core/src/main/java/me/prettyprint/cassandra/io/ChunkInputStream.java - */ -public class ChunkInputStream extends InputStream { - - private final Configuration conf; - private final byte[] tableName; - private final byte[] cf; - private final byte[] key; - private int pos; - private long chunkPos = 1; - private byte[] chunk; - - public ChunkInputStream(Configuration conf, byte[] tableName, byte[] cf, byte[] key) { - this.key = key; - this.conf = conf; - this.tableName = tableName; - this.cf = cf; - } - - public ChunkInputStream(Configuration conf, String tableName, String cf, byte[] key) { - this(conf, Bytes.toBytes(tableName), Bytes.toBytes(cf), key); - } - - @Override - public int read() throws IOException { - if (chunk == null || pos + 1 == chunk.length) { - if (!fetchChunk()) { - return -1; - } - } - return chunk[pos++]; - } - - /** - * Fetch the next chunk. - * - * @return exists if there was a chunk to fetch. - * @throws IOException - */ - private boolean fetchChunk() throws IOException { - try (HTable messages = new HTable(conf, tableName)) { - byte[] cp = Bytes.toBytes(chunkPos); - Get get = new Get(key); - get.addColumn(cf, cp); - get.setMaxVersions(1); - Result result = messages.get(get); - if (!result.isEmpty()) { - chunk = result.getValue(cf, cp); - chunkPos++; - pos = 0; - return true; - } else { - return false; - } - } catch (IOException e) { - throw new IOException("Unable to read data", e); - } - } - - @Override - public boolean markSupported() { - return false; - } -} http://git-wip-us.apache.org/repos/asf/james-project/blob/81d65f1c/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkOutputStream.java ---------------------------------------------------------------------- diff --git a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkOutputStream.java b/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkOutputStream.java deleted file mode 100644 index 4b5eae1..0000000 --- a/mailbox/hbase/src/main/java/org/apache/james/mailbox/hbase/io/ChunkOutputStream.java +++ /dev/null @@ -1,108 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.mailbox.hbase.io; - -import java.io.IOException; -import java.io.OutputStream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * Provide an {@link OutputStream} which will write to a row. The written data - * will be split up by chunks of the given chunkSize. Each chunk we get written - * to own column which will have the chunk number (starting at 0) as column key - * (Long). - * - * This implementation is not thread-safe! - * Based on Hector implementation for Cassandra. - * https://github.com/rantav/hector/blob/master/core/src/main/java/me/prettyprint/cassandra/io/ChunkOutputStream.java - */ -public class ChunkOutputStream extends OutputStream { - - private final Configuration conf; - private final byte[] tableName; - private final byte[] cf; - private final byte[] key; - private final byte[] chunk; - private long chunkPos = 1; - private long pos = 0; - - /** - * Creates a special type of {@link OutputStream} that writes data directly to HBase. - * @param conf HBase cluster configuration - * @param tableName name of the table that writes will be made - * @param cf name of the column family where data is going to be written - * @param key the row key - * @param chunkSize the size of each column, in bytes. For HBase, max is 10MB - */ - public ChunkOutputStream(Configuration conf, byte[] tableName, byte[] cf, byte[] key, int chunkSize) { - this.conf = conf; - this.tableName = tableName; - this.cf = cf; - this.key = key; - this.chunk = new byte[chunkSize]; - } - - @Override - public void write(int b) throws IOException { - if (chunk.length - 1 == pos) { - flush(); - } - chunk[(int) pos++] = (byte) b; - } - - @Override - public void close() throws IOException { - writeData(true); - } - - /** - * Trigger a flush. This will only write the content to the column if the - * chunk size is reached - */ - @Override - public void flush() throws IOException { - writeData(false); - } - - /** - * Write the data to column if the configured chunk size is reached or if the - * stream should be closed - * - * @param close - * @throws IOException - */ - private void writeData(boolean close) throws IOException { - if (pos != 0 && (close || pos == chunk.length - 1)) { - try (HTable messages = new HTable(conf, tableName)) { - Put put = new Put(key); - put.add(cf, Bytes.toBytes(chunkPos), Bytes.head(chunk, (int) pos + 1)); - messages.put(put); - chunkPos++; - pos = 0; - - } catch (IOException e) { - throw new IOException("Unable to write data", e); - } - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
