This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 866dec231047df6c94a2762d1902a20c0ce15d9f Author: Gautier DI FOLCO <[email protected]> AuthorDate: Tue Jun 23 15:01:06 2020 +0200 JAMES-3028 Drop Swift from configuration and documentation --- JenkinsfileStressTests.groovy | 24 ++--- README.adoc | 14 +-- benchmarks/s3.properties | 23 +---- benchmarks/swift.properties | 96 ------------------ .../etc/james/templates/blobstore.properties | 17 ++-- dockerfiles/run/docker-compose.yml | 14 ++- .../destination/conf/blob.properties | 95 ++---------------- .../destination/conf/blob.properties | 95 ++---------------- .../servers/pages/distributed/architecture.adoc | 4 +- .../pages/distributed/configure/blobstore.adoc | 40 +------- .../servers/pages/distributed/run-docker.adoc | 6 +- docs/modules/servers/pages/distributed/run.adoc | 39 +++----- src/homepage/howTo/deleted-messages-vault.html | 17 +--- .../install/guice-cassandra-rabbitmq-swift.md | 29 +++--- src/site/xdoc/server/config-blobstore.xml | 109 +-------------------- 15 files changed, 93 insertions(+), 529 deletions(-) diff --git a/JenkinsfileStressTests.groovy b/JenkinsfileStressTests.groovy index 61a5bd6..79d4516 100644 --- a/JenkinsfileStressTests.groovy +++ b/JenkinsfileStressTests.groovy @@ -44,10 +44,6 @@ pipeline { sh 'cp server/protocols/jmap-draft-integration-testing/rabbitmq-jmap-draft-integration-testing/src/test/resources/keystore dockerfiles/run/guice/cassandra-rabbitmq/destination/conf' sh 'wget -O dockerfiles/run/guice/cassandra-rabbitmq/destination/glowroot.zip https://github.com/glowroot/glowroot/releases/download/v0.13.4/glowroot-0.13.4-dist.zip && unzip -u dockerfiles/run/guice/cassandra-rabbitmq/destination/glowroot.zip -d dockerfiles/run/guice/cassandra-rabbitmq/destination' - if (params.PROFILE in ["s3", "swift"]) { - sh 'cp benchmarks/' + params.PROFILE + '.properties dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/blob.properties' - } - sh 'docker build -t james_run dockerfiles/run/guice/cassandra-rabbitmq' } } @@ -55,30 +51,24 @@ pipeline { stage('Start James') { steps { script { - sh 'docker rm -f cassandra rabbitmq elasticsearch tika swift james_run || true' + sh 'docker rm -f cassandra rabbitmq elasticsearch tika s3 james_run || true' if (fileExists('/srv/bench-running-docker')) { echo 'Last build failed, cleaning provisionning' sh 'sudo btrfs subvolume delete /srv/bench-running-docker' } switch (params.PROFILE) { case "reference": - sh "cd /srv && sudo btrfs subvolume snapshot bench-snapshot bench-running-docker" - sh 'docker run -d --name=cassandra -p 9042:9042 -v /srv/bench-running-docker/cassandra:/var/lib/cassandra cassandra:3.11.3' - sh 'docker run -d --name=elasticsearch -p 9200:9200 -v /srv/bench-running-docker/elasticsearch:/usr/share/elasticsearch/data/elasticsearch --env "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.3.2' - sh 'docker run -d --name=tika apache/tika:1.24' - sh 'docker run -d --name=swift -p 8080:8080 -v /srv/bench-running-docker/swift:/srv/1/node/sdb1 jeantil/openstack-keystone-swift:pike' - sh 'docker run -d --name=rabbitmq -p 15672:15672 -p 5672:5672 rabbitmq:3.8.1-management' - - sh 'docker run -d --hostname HOSTNAME -p 25:25 -p 1080:80 -p 8000:8000 -p 110:110 -p 143:143 -p 465:465 -p 587:587 -p 993:993 --link cassandra:cassandra --link rabbitmq:rabbitmq --link elasticsearch:elasticsearch --link tika:tika --link swift:swift --name james_run -t james_run' + error("reference dataset have been built with swift") break - case ["s3", "swift"]: + case "s3": sh 'docker run -d --name=cassandra -p 9042:9042 cassandra:3.11.3' sh 'docker run -d --name=elasticsearch -p 9200:9200 --env "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.3.2' sh 'docker run -d --name=tika apache/tika:1.24' + sh 'docker run -d --env "REMOTE_MANAGEMENT_DISABLE=1" --env "SCALITY_ACCESS_KEY_ID=accessKey1" --env "SCALITY_SECRET_ACCESS_KEY=secretKey1" --name=s3 zenko/cloudserver:8.2.6' sh 'docker run -d --name=rabbitmq -p 15672:15672 -p 5672:5672 rabbitmq:3.8.1-management' sh 'printenv | grep OS_ > env.file' - sh 'docker run -d --env-file env.file --hostname HOSTNAME -p 25:25 -p 1080:80 -p 8000:8000 -p 110:110 -p 143:143 -p 465:465 -p 587:587 -p 993:993 --link cassandra:cassandra --link rabbitmq:rabbitmq --link elasticsearch:elasticsearch --link tika:tika --name james_run -t james_run' + sh 'docker run -d --env-file env.file --hostname HOSTNAME -p 25:25 -p 1080:80 -p 8000:8000 -p 110:110 -p 143:143 -p 465:465 -p 587:587 -p 993:993 --link cassandra:cassandra --link rabbitmq:rabbitmq --link elasticsearch:elasticsearch --link s3:s3.docker.test --link tika:tika --name james_run -t james_run' break } def jamesCliWithOptions = 'java -jar /root/james-cli.jar -h 127.0.0.1 -p 9999' @@ -88,7 +78,7 @@ pipeline { sh "docker exec james_run ${jamesCliWithOptions} listusers" } } - if (params.PROFILE in ["s3", "swift"]) { + if (params.PROFILE == "s3") { sh "docker exec james_run ${jamesCliWithOptions} removedomain localhost || true" sh "docker exec james_run ${jamesCliWithOptions} removedomain james.linagora.com || true" sh "docker exec james_run ${jamesCliWithOptions} adddomain open-paas.org" @@ -122,7 +112,7 @@ pipeline { node('target') { script { sh 'docker logs james_run || true' - sh 'docker rm -f cassandra rabbitmq elasticsearch tika swift james_run || true' + sh 'docker rm -f cassandra rabbitmq elasticsearch tika s3 james_run || true' sh 'sudo btrfs subvolume delete /srv/bench-running-docker || true' } } diff --git a/README.adoc b/README.adoc index 1d9e287..5479474 100644 --- a/README.adoc +++ b/README.adoc @@ -44,7 +44,7 @@ There is many other ways one can help us: packaging, communication, etc ... * link:#how-to-release-via-maven-release-plugin[How to release via maven release plugin] * link:#how-to-check-the-compilation[How to check the compilation] * link:#how-to-run-james-in-docker[How to run James in Docker] - ** link:#run-james-with-guice-%2Dcassandra-%2Drabbitmq-%2Dswift-%2Delasticsearch[Run James with Guice + Cassandra + RabbitMQ + Swift + ElasticSearch] + ** link:#run-james-with-%2Dguice-%2Dcassandra-%2Drabbitmq-%2Ds3-%2Delasticsearch[Run James with Guice + Cassandra + RabbitMQ + S3 + ElasticSearch] ** link:#run-james-with-guice-%2Dcassandra-%2Delasticsearch[Run James with Guice + Cassandra + ElasticSearch] ** link:#run-james-with-guice-%2Djpa-%2Dlucene[Run James with Guice + JPA + Lucene] ** link:#run-james-with-spring-%2Djpa[Run James with Spring + JPA] @@ -198,13 +198,13 @@ If you are using a a fresh installation of Docker, your DOCKER_HOST should be un This feature is available for three configurations : - * Guice + Cassandra + RabbitMQ + Swift + ElasticSearch + * Guice + Cassandra + RabbitMQ + S3 + ElasticSearch * Guice + Cassandra + ElasticSearch * Guice + JPA + Lucene * Spring + JPA -=== Run James with Guice + Cassandra + RabbitMQ + Swift + ElasticSearch +=== Run James with Guice + Cassandra + RabbitMQ + S3 + ElasticSearch ==== Requirements @@ -226,9 +226,9 @@ You need a running *rabbitmq* in docker. To achieve this run: $ docker run -d --name=rabbitmq rabbitmq:3.8.1-management -You need a running *swift* objectstorage in docker. To achieve this run: +You need a running *s3* compatible objectstorage in docker. To achieve this run: - $ docker run -d --name=swift linagora/openstack-keystone-swift:pike + $ docker run -d -p 8080:8000 --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 zenko/cloudserver:8.2.6 You need a running *ElasticSearch* in docker. To achieve this run: @@ -253,7 +253,7 @@ Then we need to build james container : To run this container : $ docker run --hostname HOSTNAME -p "25:25" -p 80:80 -p "110:110" -p "143:143" -p "465:465" -p "587:587" -p "993:993" -p "127.0.0.1:8000:8000" --link cassandra:cassandra --link rabbitmq:rabbitmq - --link elasticsearch:elasticsearch --link tika:tika --link swift:swift --name james_run -t james_run + --link elasticsearch:elasticsearch --link tika:tika --link s3:s3 --name james_run -t james_run Where : @@ -550,7 +550,7 @@ Where: - ITERATION is the release number used after the last hyphen (e.g. 3.0.1, 3.1.0, 3.2.0...) - BASE is the image jar and executable are copied from. Defaults to linagora/james-project - BASE_LDAP is the image jar and executable are copied from for a deployment with an LDAP user repository. Defaults to linagora/james-ldap-project -- BASE_RABBITMQ is the image jar and executable are copied from for a deployment of linagora/james-project + RabbitMQ + Swift BlobStore. +- BASE_RABBITMQ is the image jar and executable are copied from for a deployment of linagora/james-project + RabbitMQ + S3 BlobStore. Defaults to linagora/james-rabbitmq-project - TAG is the tag of these docker images. Defaults to latest. diff --git a/benchmarks/s3.properties b/benchmarks/s3.properties index 9962979..63aacab 100644 --- a/benchmarks/s3.properties +++ b/benchmarks/s3.properties @@ -2,31 +2,12 @@ # Read https://james.apache.org/server/config-blobstore.html for further details # Choose your BlobStore implementation -# Mandatory, allowed values are: cassandra, objectstorage, hybrid +# Mandatory, allowed values are: cassandra, s3, hybrid # hybrid is using both objectstorage for unfrequently read or big blobs & cassandra for small, often read blobs -implementation=objectstorage +implementation=s3 # ============================================== ObjectStorage ============================================ -# ========================================= ObjectStorage Codec ====================================== -# Codec for blob contents -# Mandatory, allowed values are: DEFAULT, AES256 -# DEFAULT: no encryption -# AES256: AES-256 encryption -objectstorage.payload.codec=DEFAULT - -# ===================================== ObjectStorage AES256 Encryption ==================================== -# Mandatory if you choose AES256 encryption, salt value in string for the encryption -# objectstorage.aes256.hexsalt=salt - -# Mandatory if you choose AES256 encryption, password for the encryption -# objectstorage.aes256.password=password - -# ========================================= ObjectStorage providers ======================================== -# Choosing blob storage service -# Mandatory, Allowed values are: swift, aws-s3 -objectstorage.provider=aws-s3 - # ========================================= ObjectStorage Buckets ========================================== # bucket names prefix # Optional, default no prefix diff --git a/benchmarks/swift.properties b/benchmarks/swift.properties deleted file mode 100644 index ea990ca..0000000 --- a/benchmarks/swift.properties +++ /dev/null @@ -1,96 +0,0 @@ -# ============================================= BlobStore Implementation ================================== -# Read https://james.apache.org/server/config-blobstore.html for further details - -# Choose your BlobStore implementation -# Mandatory, allowed values are: cassandra, objectstorage, hybrid -# hybrid is using both objectstorage for unfrequently read or big blobs & cassandra for small, often read blobs -implementation=objectstorage - -# ============================================== ObjectStorage ============================================ - -# ========================================= ObjectStorage Codec ====================================== -# Codec for blob contents -# Mandatory, allowed values are: DEFAULT, AES256 -# DEFAULT: no encryption -# AES256: AES-256 encryption -objectstorage.payload.codec=DEFAULT - -# ===================================== ObjectStorage AES256 Encryption ==================================== -# Mandatory if you choose AES256 encryption, salt value in string for the encryption -# objectstorage.aes256.hexsalt=salt - -# Mandatory if you choose AES256 encryption, password for the encryption -# objectstorage.aes256.password=password - -# ========================================= ObjectStorage providers ======================================== -# Choosing blob storage service -# Mandatory, Allowed values are: swift, aws-s3 -objectstorage.provider=swift - -# ========================================= ObjectStorage Buckets ========================================== -# bucket names prefix -# Optional, default no prefix -# objectstorage.bucketPrefix=prod- - -# Default bucket name -# Optional, default is bucketPrefix + `default` -objectstorage.namespace=${env:OS_NAMESPACE} - -# ========================================= ObjectStorage on Swift ========================================= -# Specify the geography region which the BlobStore will connect to a in Swift ObjectStorage -# Optional, possible values are those of your Swift service -# objectstorage.swift.region=HKG - -# Specify the authentication mechanism of Swift -# Mandatory, allowed values are: tmpauth, keystone2, keystone3 -objectstorage.swift.authapi=keystone3 - -# Mandatory, authentication endpoint -objectstorage.swift.endpoint=${env:OS_SWIFT_ENDPOINT} - -# Openstack Swift authentication mechanisms require you to pass the user information -# to get the access tokens. User information usually include two parts: -# identity: user identification like username -# credential: like password, here it is the Secret Access Key in Jcloud -# Mandatory -objectstorage.swift.credentials=${env:OS_SWIFT_CREDENTIALS} - -# =================================== ObjectStorage on Swift with Temp Auth ================================ -# Mandatory, authentication identity, the identity contains two part, username and tenantname -# Should be the username & tenant name configured in Swift -# Mandatory -objectstorage.swift.keystone3.user.name=${env:OS_SWIFT_USER_NAME} -# Mandatory -objectstorage.swift.keystone3.user.domain=${env:OS_SWIFT_USER_DOMAIN} - -# ================================ ObjectStorage on Swift with Keystone 3 Auth ============================== -objectstorage.swift.keystone3.scope.domainid=${env:OS_SWIFT_DOMAIN_ID} -objectstorage.swift.keystone3.scope.project.domainid=${env:OS_SWIFT_PROJECT_DOMAIN_ID} -objectstorage.swift.keystone3.scope.project.name=${env:OS_SWIFT_PROJECT_DOMAIN_NAME} - -# ============================================ Blobs Exporting ============================================== -# Read https://james.apache.org/server/config-blob-export.html for further details - -# Choosing blob exporting mechanism, allowed mechanism are: localFile, linshare -# LinShare is a file sharing service, will be explained in the below section -# Optional, default is localFile -blob.export.implementation=localFile - -# ======================================= Local File Blobs Exporting ======================================== -# Optional, directory to store exported blob, directory path follows James file system format -# default is file://var/blobExporting -blob.export.localFile.directory=file://var/blobExporting - -# ======================================= LinShare File Blobs Exporting ======================================== -# LinShare is a sharing service where you can use james, connects to an existing LinShare server and shares files to -# other mail addresses as long as those addresses available in LinShare. For example you can deploy James and LinShare -# sharing the same LDAP repository -# Mandatory if you choose LinShare, url to connect to LinShare service -# blob.export.linshare.url=http://linshare:8080 - -# ======================================= LinShare Configuration BasicAuthentication =================================== -# Authentication is mandatory if you choose LinShare, TechnicalAccount is need to connect to LinShare specific service. -# For Example: It will be formalized to 'Authorization: Basic {Credential of UUID/password}' - -# blob.export.linshare.technical.account.uuid=Technical_Account_UUID -# blob.export.linshare.technical.account.password=password diff --git a/dockerfiles/packaging/guice/cassandra/package/etc/james/templates/blobstore.properties b/dockerfiles/packaging/guice/cassandra/package/etc/james/templates/blobstore.properties index 9ab4beb..0e347ec 100644 --- a/dockerfiles/packaging/guice/cassandra/package/etc/james/templates/blobstore.properties +++ b/dockerfiles/packaging/guice/cassandra/package/etc/james/templates/blobstore.properties @@ -1,11 +1,8 @@ -# Configuration for swift BlobStore -objectstorage.payload.codec=DEFAULT -objectstorage.provider=swift +# Configuration for s3 BlobStore objectstorage.namespace=james -objectstorage.swift.authapi=tmpauth -objectstorage.swift.endpoint=http://swift:8080/auth/v1.0 -objectstorage.swift.credentials=testing -objectstorage.swift.tempauth.username=tester -objectstorage.swift.tempauth.tenantname=test -objectstorage.swift.tempauth.passheadername=X-Storage-Pass -objectstorage.swift.tempauth.userheadername=X-Storage-User \ No newline at end of file +implementation=s3 +objectstorage.namespace=james +objectstorage.s3.endPoint=http://scality:8080/ +objectstorage.s3.region=eu-west-1 +objectstorage.s3.accessKeyId=accessKey1 +objectstorage.s3.secretKey=verySecretKey1 diff --git a/dockerfiles/run/docker-compose.yml b/dockerfiles/run/docker-compose.yml index a048c99..d0fc0fe 100644 --- a/dockerfiles/run/docker-compose.yml +++ b/dockerfiles/run/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '3' services: @@ -8,7 +8,7 @@ services: - cassandra - tika - rabbitmq - - swift + - s3 entrypoint: bash -c "java -Dworking.directory=/root/ -Dlogback.configurationFile=/root/conf/logback.xml -jar james-server.jar" image: linagora/james-rabbitmq-project:latest container_name: james @@ -21,7 +21,7 @@ services: - "465:465" - "587:587" - "993:993" - - "8000:8000" + - "8080:8000" elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 @@ -42,9 +42,7 @@ services: - "5672:5672" - "15672:15672" - swift: - image: linagora/openstack-keystone-swift:pike + s3: + image: scality/s3server:6018536a ports: - - "5000:5000" - - "8080:8080" - - "35357:35357" + - "8080:8000" diff --git a/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/blob.properties b/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/blob.properties index 6e5e8f8..a0cc636 100644 --- a/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/blob.properties +++ b/dockerfiles/run/guice/cassandra-rabbitmq-ldap/destination/conf/blob.properties @@ -2,8 +2,8 @@ # Read https://james.apache.org/server/config-blobstore.html for further details # Choose your BlobStore implementation -# Mandatory, allowed values are: cassandra, objectstorage -implementation=objectstorage +# Mandatory, allowed values are: cassandra, s3 +implementation=s3 # ========================================= ObjectStorage deduplication ======================================== # If you choose to enable deduplication, the mails with the same content will be stored only once. @@ -37,25 +37,6 @@ cache.enable=false # ============================================== ObjectStorage ============================================ -# ========================================= ObjectStorage Codec ====================================== -# Codec for blob contents -# Mandatory, allowed values are: DEFAULT, AES256 -# DEFAULT: no encryption -# AES256: AES-256 encryption -objectstorage.payload.codec=DEFAULT - -# ===================================== ObjectStorage AES256 Encryption ==================================== -# Mandatory if you choose AES256 encryption, salt value in string for the encryption -# objectstorage.aes256.hexsalt=salt - -# Mandatory if you choose AES256 encryption, password for the encryption -# objectstorage.aes256.password=password - -# ========================================= ObjectStorage providers ======================================== -# Choosing blob storage service -# Mandatory, Allowed values are: swift, aws-s3 -objectstorage.provider=swift - # ========================================= ObjectStorage Buckets ========================================== # bucket names prefix # Optional, default no prefix @@ -65,74 +46,18 @@ objectstorage.provider=swift # Optional, default is bucketPrefix + `default` # objectstorage.namespace=james -# ========================================= ObjectStorage on Swift ========================================= -# Specify the geography region which the BlobStore will connect to a in Swift ObjectStorage -# Optional, possible values are those of your Swift service -# objectstorage.swift.region=HKG - -# Specify the authentication mechanism of Swift -# Mandatory, allowed values are: tmpauth, keystone2, keystone3 -objectstorage.swift.authapi=tmpauth - -# Mandatory, authentication endpoint -objectstorage.swift.endpoint=http://swift:8080/auth/v1.0 - -# Openstack Swift authentication mechanisms require you to pass the user information -# to get the access tokens. User information usually include two parts: -# identity: user identification like username -# credential: like password, here it is the Secret Access Key in Jcloud -# Mandatory -objectstorage.swift.credentials=testing - -# =================================== ObjectStorage on Swift with Temp Auth ================================ -# Mandatory, authentication identity, the identity contains two part, username and tenantname -# Should be the username & tenant name configured in Swift -# Mandatory -objectstorage.swift.tempauth.username=tester -# Mandatory -objectstorage.swift.tempauth.tenantname=test - -# Specify http header name for carrying credential information -# Mandatory -objectstorage.swift.tempauth.passheadername=X-Storage-Pass - -# Specify http header name for carrying identity information -# Mandatory -objectstorage.swift.tempauth.userheadername=X-Storage-User - -# ================================ ObjectStorage on Swift with Keystone 2 Auth ============================== -# Similar to tempAuth -# Mandatory if you choose keystone 2 auth -# objectstorage.swift.keystone2.username=tester -# Mandatory if you choose keystone 2 auth -# objectstorage.swift.keystone2.tenantname=test - -# ================================ ObjectStorage on Swift with Keystone 3 Auth ============================== -# Specifying identity information with Keystone 3 Auth -# Mandatory if you choose keystone 3 auth -# objectstorage.swift.keystone3.user.name=demo -# Mandatory if you choose keystone 3 auth -# objectstorage.swift.keystone3.user.domain=Default -# Mandatory if you choose keystone 3 auth -# objectstorage.swift.keystone3.scope.domainid=default - -# Specifying project information in KeyStone 3 Auth (similar to tenant in KeyStone 2) -# Optional -# objectstorage.swift.keystone3.scope.project.name=test -# Optional -# objectstorage.swift.keystone3.scope.project.domainname=Default -# Optional -# objectstorage.swift.keystone3.scope.project.domainid=default - # ========================================= ObjectStorage on S3 ============================================= -# Mandatory if you choose aws-s3 storage service, S3 authentication endpoint -# objectstorage.s3.endPoint=http://james-s3:8000 +# Mandatory if you choose s3 storage service, S3 authentication endpoint +objectstorage.s3.endPoint=http://scality:8080/ + +# Mandatory if you choose s3 storage service, S3 region +objectstorage.s3.region=eu-west-1 # Mandatory if you choose aws-s3 storage service, access key id configured in S3 -# objectstorage.s3.accessKeyId=accessKey +objectstorage.s3.accessKeyId=accessKey1 -# Mandatory if you choose aws-s3 storage service, secret key configured in S3 -# objectstorage.s3.secretKey=secretKey +# Mandatory if you choose s3 storage service, secret key configured in S3 +objectstorage.s3.secretKey=secretKey1 # ============================================ Blobs Exporting ============================================== # Read https://james.apache.org/server/config-blob-export.html for further details diff --git a/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/blob.properties b/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/blob.properties index c579d80..bfe3c59 100644 --- a/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/blob.properties +++ b/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/blob.properties @@ -2,8 +2,8 @@ # Read https://james.apache.org/server/config-blobstore.html for further details # Choose your BlobStore implementation -# Mandatory, allowed values are: cassandra, objectstorage -implementation=objectstorage +# Mandatory, allowed values are: cassandra, s3 +implementation=s3 # ========================================= ObjectStorage deduplication ======================================== # If you choose to enable deduplication, the mails with the same content will be stored only once. @@ -36,25 +36,6 @@ cache.enable=false # ============================================== ObjectStorage ============================================ -# ========================================= ObjectStorage Codec ====================================== -# Codec for blob contents -# Mandatory, allowed values are: DEFAULT, AES256 -# DEFAULT: no encryption -# AES256: AES-256 encryption -objectstorage.payload.codec=DEFAULT - -# ===================================== ObjectStorage AES256 Encryption ==================================== -# Mandatory if you choose AES256 encryption, salt value in string for the encryption -# objectstorage.aes256.hexsalt=salt - -# Mandatory if you choose AES256 encryption, password for the encryption -# objectstorage.aes256.password=password - -# ========================================= ObjectStorage providers ======================================== -# Choosing blob storage service -# Mandatory, Allowed values are: swift, aws-s3 -objectstorage.provider=swift - # ========================================= ObjectStorage Buckets ========================================== # bucket names prefix # Optional, default no prefix @@ -64,74 +45,18 @@ objectstorage.provider=swift # Optional, default is bucketPrefix + `default` # objectstorage.namespace=james -# ========================================= ObjectStorage on Swift ========================================= -# Specify the geography region which the BlobStore will connect to a in Swift ObjectStorage -# Optional, possible values are those of your Swift service -# objectstorage.swift.region=HKG - -# Specify the authentication mechanism of Swift -# Mandatory, allowed values are: tmpauth, keystone2, keystone3 -objectstorage.swift.authapi=tmpauth - -# Mandatory, authentication endpoint -objectstorage.swift.endpoint=http://swift:8080/auth/v1.0 - -# Openstack Swift authentication mechanisms require you to pass the user information -# to get the access tokens. User information usually include two parts: -# identity: user identification like username -# credential: like password, here it is the Secret Access Key in Jcloud -# Mandatory -objectstorage.swift.credentials=testing - -# =================================== ObjectStorage on Swift with Temp Auth ================================ -# Mandatory, authentication identity, the identity contains two part, username and tenantname -# Should be the username & tenant name configured in Swift -# Mandatory -objectstorage.swift.tempauth.username=tester -# Mandatory -objectstorage.swift.tempauth.tenantname=test - -# Specify http header name for carrying credential information -# Mandatory -objectstorage.swift.tempauth.passheadername=X-Storage-Pass - -# Specify http header name for carrying identity information -# Mandatory -objectstorage.swift.tempauth.userheadername=X-Storage-User - -# ================================ ObjectStorage on Swift with Keystone 2 Auth ============================== -# Similar to tempAuth -# Mandatory if you choose keystone 2 auth -# objectstorage.swift.keystone2.username=tester -# Mandatory if you choose keystone 2 auth -# objectstorage.swift.keystone2.tenantname=test - -# ================================ ObjectStorage on Swift with Keystone 3 Auth ============================== -# Specifying identity information with Keystone 3 Auth -# Mandatory if you choose keystone 3 auth -# objectstorage.swift.keystone3.user.name=demo -# Mandatory if you choose keystone 3 auth -# objectstorage.swift.keystone3.user.domain=Default -# Mandatory if you choose keystone 3 auth -# objectstorage.swift.keystone3.scope.domainid=default - -# Specifying project information in KeyStone 3 Auth (similar to tenant in KeyStone 2) -# Optional -# objectstorage.swift.keystone3.scope.project.name=test -# Optional -# objectstorage.swift.keystone3.scope.project.domainname=Default -# Optional -# objectstorage.swift.keystone3.scope.project.domainid=default - # ========================================= ObjectStorage on S3 ============================================= -# Mandatory if you choose aws-s3 storage service, S3 authentication endpoint -# objectstorage.s3.endPoint=http://james-s3:8000 +# Mandatory if you choose s3 storage service, S3 authentication endpoint +objectstorage.s3.endPoint=http://scality:8080/ + +# Mandatory if you choose s3 storage service, S3 region +objectstorage.s3.region=eu-west-1 # Mandatory if you choose aws-s3 storage service, access key id configured in S3 -# objectstorage.s3.accessKeyId=accessKey +objectstorage.s3.accessKeyId=accessKey1 -# Mandatory if you choose aws-s3 storage service, secret key configured in S3 -# objectstorage.s3.secretKey=secretKey +# Mandatory if you choose s3 storage service, secret key configured in S3 +objectstorage.s3.secretKey=secretKey1 # ============================================ Blobs Exporting ============================================== # Read https://james.apache.org/server/config-blob-export.html for further details diff --git a/docs/modules/servers/pages/distributed/architecture.adoc b/docs/modules/servers/pages/distributed/architecture.adoc index 3b2d85d..2736033 100644 --- a/docs/modules/servers/pages/distributed/architecture.adoc +++ b/docs/modules/servers/pages/distributed/architecture.adoc @@ -11,7 +11,7 @@ image::storage.png[Storage responsibilities for the Distributed Server] * *Cassandra* is used for metadata storage * The *blob store* storage interface is responsible of storing potentially large binary data. For instance - email bodies, headers or attachments. Different technologies can be used: *Cassandra*, or *Object Storage* + email bodies, headers or attachments. Different technologies can be used: *Cassandra*, or S3 compatible *Object Storage* (S3 or Swift) * *ElasticSearch* component empowers full text search on emails. * *RabbitMQ* enables James nodes of a same cluster to collaborate together. @@ -261,7 +261,7 @@ Stores potentially large binary data. Mailbox component, Mail Queue component, Deleted Message Vault component relies on it. -Supported backends includes ObjectStorage (link:https://wiki.openstack.org/wiki/Swift[Swift], S3 API). +Supported backends include S3 compatible ObjectStorage (link:https://wiki.openstack.org/wiki/Swift[Swift], S3 API). Encryption can be configured on top of ObjectStorage. diff --git a/docs/modules/servers/pages/distributed/configure/blobstore.adoc b/docs/modules/servers/pages/distributed/configure/blobstore.adoc index 8a56e87..139c961 100644 --- a/docs/modules/servers/pages/distributed/configure/blobstore.adoc +++ b/docs/modules/servers/pages/distributed/configure/blobstore.adoc @@ -10,7 +10,7 @@ James uses the BlobStore for storing blobs which are usually mail contents, atta You can choose the underlying implementation of BlobStore to fit with your James setup. -It could be the implementation on top of Cassandra or file storage service like Openstack Swift, AWS S3. +It could be the implementation on top of Cassandra or file storage service S3 compatible like Openstack Swift and AWS S3. Consult https://github.com/apache/james-project/blob/master/dockerfiles/run/guice/cassandra-rabbitmq/destination/conf/blob.properties[blob.properties] in GIT to get some examples and hints. @@ -65,17 +65,6 @@ Maximum size of stored objects expressed in bytes. === Object storage configuration -==== Underlying Service Configuration - -*objectstorage.provider* : - -* swift: Openstack Swift -* aws-s3: Amazon AWS S3 - -==== Swift Configuration - -(TODO) - ==== AWS S3 Configuration .blobstore.properties S3 related properties @@ -85,6 +74,9 @@ Maximum size of stored objects expressed in bytes. | objectstorage.s3.endPoint | S3 service endpoint +| objectstorage.s3.region +| S3 region + | objectstorage.s3.accessKeyId | https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[S3 access key id] @@ -107,30 +99,6 @@ BucketPrefix is the prefix of bucket names in James BlobStore Unless a special case like storing blobs of deleted messages. |=== -==== Codec Configuration - -.objectstorage.payload.codec -|=== -| objectstorage.payload.codec - -| DEFAULT: no encryption - -| AES-256: AES-256 encryption -|=== - -If an AES-256 payload is used: - -.AES-256 configuration -|=== -| Property name | explanation - -| objectstorage.aes256.hexsalt -| salt string required by AES 256 encryption - -| objectstorage.aes256.password -| password string required by AES 256 encryption -|=== - == Blob Export Blob Exporting is the mechanism to help James to export a blob from an user to another user. diff --git a/docs/modules/servers/pages/distributed/run-docker.adoc b/docs/modules/servers/pages/distributed/run-docker.adoc index 62b7db0..01d73b0 100644 --- a/docs/modules/servers/pages/distributed/run-docker.adoc +++ b/docs/modules/servers/pages/distributed/run-docker.adoc @@ -51,9 +51,9 @@ You need a running *rabbitmq* in docker. To achieve this run: $ docker run -d --name=rabbitmq rabbitmq:3.8.1-management -You need a running *swift* objectstorage in docker. To achieve this run: +You need a running *Zenko Cloudserver* objectstorage in docker. To achieve this run: - $ docker run -d --name=swift linagora/openstack-keystone-swift:pike + $ docker run -d -p 8080:8000 --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 zenko/cloudserver:8.2.6 You need a running *ElasticSearch* in docker. To achieve this run: @@ -78,7 +78,7 @@ Then we need to build james container : To run this container : $ docker run --hostname HOSTNAME -p "25:25" -p 80:80 -p "110:110" -p "143:143" -p "465:465" -p "587:587" -p "993:993" -p "127.0.0.1:8000:8000" --link cassandra:cassandra --link rabbitmq:rabbitmq - --link elasticsearch:elasticsearch --link tika:tika --link swift:swift --name james_run -t james_run + --link elasticsearch:elasticsearch --link tika:tika --link s3:s3 --name james_run -t james_run Where : diff --git a/docs/modules/servers/pages/distributed/run.adoc b/docs/modules/servers/pages/distributed/run.adoc index b72d645..a21facb 100644 --- a/docs/modules/servers/pages/distributed/run.adoc +++ b/docs/modules/servers/pages/distributed/run.adoc @@ -6,8 +6,8 @@ === Requirements * Java 11 SDK -* Docker ∕ ElasticSearch 6.3.2, RabbitMQ Management 3.3.7, Swift -ObjectStorage 2.15.1 and Cassandra 3.11.3 +* Docker ∕ ElasticSearch 6.3.2, RabbitMQ Management 3.3.7, S3 compatible +ObjectStorage and Cassandra 3.11.3 * Maven 3 === Building the artifacts @@ -32,7 +32,7 @@ mvn clean install * Cassandra 3.11.3 * ElasticSearch 6.3.2 * RabbitMQ-Management 3.8.1 -* Swift ObjectStorage 2.15.1 or Scality S3 server or AWS S3 +* Swift ObjectStorage 2.15.1 or Zenko Cloudserver or AWS S3 === James Launch @@ -53,7 +53,7 @@ following command: $ keytool -genkey -alias james -keyalg RSA -keystore conf/keystore ---- -You need to have a Cassandra, ElasticSearch and RabbitMQ instance +You need to have a Cassandra, ElasticSearch, S3 and RabbitMQ instance running. You can either install the servers or launch them via docker: [source,bash] @@ -61,7 +61,7 @@ running. You can either install the servers or launch them via docker: $ docker run -d -p 9042:9042 --name=cassandra cassandra:3.11.3 $ docker run -d -p 9200:9200 --name=elasticsearch --env 'discovery.type=single-node' docker.elastic.co/elasticsearch/elasticsearch:6.3.2 $ docker run -d -p 5672:5672 -p 15672:15672 --name=rabbitmq rabbitmq:3.8.1-management -$ docker run -d -p 5000:5000 -p 8080:8080 -p 35357:35357 --name=swift linagora/openstack-keystone-swift:pike +$ docker run -d -p 8080:8000 --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 zenko/cloudserver:8.2.6 ---- Once everything is set up, you just have to run the jar with: @@ -71,33 +71,24 @@ Once everything is set up, you just have to run the jar with: $ java -Dworking.directory=. -jar target/james-server-cassandra-rabbitmq-guice.jar ---- -==== Using AWS S3 of Scality S3 server +==== Using AWS S3 of Zenko Cloudserver -In order to use AWS S3 or a compatible implementation, -`blobstore.propeties` has to be filled with: +By default, James is configured with [Zenko Cloudserver](https://hub.docker.com/r/zenko/cloudserver) which is compatible with AWS S3, in `blobstore.propeties` as such: -.... -objectstorage.provider=aws-s3 +[source,bash] +---- +implementation=s3 objectstorage.namespace=james -objectstorage.s3.endPoint=http://scality:8080/ +objectstorage.s3.endPoint=http://s3.docker.test:8000/ +objectstorage.s3.region=eu-west-1 objectstorage.s3.accessKeyId=accessKey1 -objectstorage.s3.secretKey=verySecretKey1 -.... - -To use Scality S3 server you have to launch it instead of swift -container: - -.... -$ docker run -d -p 8080:8000 --name=s3 scality/s3server:6018536a -.... - -More information about available options -https://hub.docker.com/r/scality/s3server[here]. +objectstorage.s3.secretKey=secretKey1 +---- == Guice-cassandra-rabbitmq-ldap You can follow the same guide to build and run -guice-cassandra-rabbitmq-swift-ldap artifact, except that: +guice-cassandra-rabbitmq-s3-ldap artifact, except that: * The *jar* and *libs* needs to be retrieve from server/container/guice/cassandra-rabbitmq-ldap-guice/target after diff --git a/src/homepage/howTo/deleted-messages-vault.html b/src/homepage/howTo/deleted-messages-vault.html index 6459044..344c96b 100644 --- a/src/homepage/howTo/deleted-messages-vault.html +++ b/src/homepage/howTo/deleted-messages-vault.html @@ -167,19 +167,12 @@ layout: howTo </p> <pre><code> -implementation=objectstorage - -# Configuration for swift BlobStore -objectstorage.payload.codec=DEFAULT -objectstorage.provider=swift +implementation=s3 objectstorage.namespace=james -objectstorage.swift.authapi=tmpauth -objectstorage.swift.endpoint=http://swift:8080/auth/v1.0 -objectstorage.swift.credentials=testing -objectstorage.swift.tempauth.username=tester -objectstorage.swift.tempauth.tenantname=test -objectstorage.swift.tempauth.passheadername=X-Storage-Pass -objectstorage.swift.tempauth.userheadername=X-Storage-User +objectstorage.s3.endPoint=http://scality:8080/ +objectstorage.s3.region=eu-west-1 +objectstorage.s3.accessKeyId=accessKey1 +objectstorage.s3.secretKey=verySecretKey1 </code></pre> <p> diff --git a/src/site/markdown/server/install/guice-cassandra-rabbitmq-swift.md b/src/site/markdown/server/install/guice-cassandra-rabbitmq-swift.md index d82c159..7377382 100644 --- a/src/site/markdown/server/install/guice-cassandra-rabbitmq-swift.md +++ b/src/site/markdown/server/install/guice-cassandra-rabbitmq-swift.md @@ -1,11 +1,11 @@ -# Guice-Cassandra-Rabbitmq-Swift installation guide +# Guice-Cassandra-Rabbitmq-S3 installation guide ## Building ### Requirements - Java 11 SDK - - Docker ∕ ElasticSearch 6.3.2, RabbitMQ Management 3.3.7, Swift ObjectStorage 2.15.1 and Cassandra 3.11.3 + - Docker ∕ ElasticSearch 6.3.2, RabbitMQ Management 3.8.1, compatible S3 ObjectStorage and Cassandra 3.11.3 - Maven 3 ### Building the artifacts @@ -28,7 +28,7 @@ mvn clean install * Cassandra 3.11.3 * ElasticSearch 6.3.2 * RabbitMQ-Management 3.8.1 - * Swift ObjectStorage 2.15.1 or Scality S3 server or AWS S3 + * Swift ObjectStorage 2.15.1 or Zenko Cloudserver or AWS S3 ### James Launch @@ -44,13 +44,13 @@ You also need to generate a keystore in your conf folder with the following comm $ keytool -genkey -alias james -keyalg RSA -keystore conf/keystore ``` -You need to have a Cassandra, ElasticSearch and RabbitMQ instance running. You can either install the servers or launch them via docker: +You need to have a Cassandra, ElasticSearch, S3 and RabbitMQ instance running. You can either install the servers or launch them via docker: ```bash $ docker run -d -p 9042:9042 --name=cassandra cassandra:3.11.3 $ docker run -d -p 9200:9200 --name=elasticsearch --env 'discovery.type=single-node' docker.elastic.co/elasticsearch/elasticsearch:6.3.2 $ docker run -d -p 5672:5672 -p 15672:15672 --name=rabbitmq rabbitmq:3.8.1-management -$ docker run -d -p 5000:5000 -p 8080:8080 -p 35357:35357 --name=swift linagora/openstack-keystone-swift:pike +$ docker run -d -p 8080:8000 --env 'REMOTE_MANAGEMENT_DISABLE=1' --env 'SCALITY_ACCESS_KEY_ID=accessKey1' --env 'SCALITY_SECRET_ACCESS_KEY=secretKey1' --name=s3 zenko/cloudserver:8.2.6 ``` Once everything is set up, you just have to run the jar with: @@ -59,25 +59,18 @@ Once everything is set up, you just have to run the jar with: $ java -Dworking.directory=. -jar target/james-server-cassandra-rabbitmq-guice.jar ``` -#### Using AWS S3 of Scality S3 server -In order to use AWS S3 or a compatible implementation, `blobstore.propeties` has to be filled with: +#### Using AWS S3 of Zenko Cloudserver +By default, James is configured with [Zenko Cloudserver](https://hub.docker.com/r/zenko/cloudserver) which is compatible with AWS S3, in `blobstore.propeties` as such: ``` -objectstorage.provider=aws-s3 +implementation=s3 objectstorage.namespace=james -objectstorage.s3.endPoint=http://scality:8080/ +objectstorage.s3.endPoint=http://s3.docker.test:8000/ +objectstorage.s3.region=eu-west-1 objectstorage.s3.accessKeyId=accessKey1 -objectstorage.s3.secretKey=verySecretKey1 +objectstorage.s3.secretKey=secretKey1 ``` -To use Scality S3 server you have to launch it instead of swift container: - -``` -$ docker run -d -p 8080:8000 --name=s3 scality/s3server:6018536a -``` - -More information about available options [here](https://hub.docker.com/r/scality/s3server). - ## Guice-cassandra-rabbitmq-ldap You can follow the same guide to build and run guice-cassandra-rabbitmq-swift-ldap artifact, except that: diff --git a/src/site/xdoc/server/config-blobstore.xml b/src/site/xdoc/server/config-blobstore.xml index a6f1a1b..92de092 100644 --- a/src/site/xdoc/server/config-blobstore.xml +++ b/src/site/xdoc/server/config-blobstore.xml @@ -45,7 +45,7 @@ <dl> <dt><strong>implementation</strong></dt> <dd>cassandra: use cassandra based BlobStore</dd> - <dd>objectstorage: use Swift/AWS S3 based BlobStore</dd> + <dd>s3: use AWS S3 based BlobStore</dd> <dt><strong>deduplication/enable</strong></dt> <dd>Mandatory. Supported value: true and false.</dd> @@ -86,22 +86,6 @@ </dl> </subsection> - <subsection name="ObjectStorage BlobStore Codec Configuration"> - <dl> - <dt><strong>objectstorage.payload.codec</strong></dt> - <dd>DEFAULT: no encryption</dd> - <dd>AES-256: AES-256 encryption</dd> - </dl> - <subsection name="ObjectStorage AES256 Encryption"> - <dl> - <dt><strong>objectstorage.aes256.hexsalt</strong></dt> - <dd>salt string required by AES 256 encryption</dd> - - <dt><strong>objectstorage.aes256.password</strong></dt> - <dd>password string required by AES 256 encryption</dd> - </dl> - </subsection> - </subsection> <subsection name="ObjectStorage BlobStore Buckets Configuration"> <dl> <dt><strong>objectstorage.bucketPrefix</strong></dt> @@ -118,99 +102,14 @@ </dl> </subsection> <subsection name="ObjectStorage Underlying Service Configuration"> - <dl> - <dt><strong>objectstorage.provider</strong></dt> - <dd>swift: Openstack Swift</dd> - <dd>aws-s3: Amazon AWS S3</dd> - </dl> - <subsection name="ObjectStorage Swift Configuration"> - <dl> - <dt><strong>objectstorage.swift.region</strong></dt> - <dd> - Specify the geography region which the BlobStore will connect to a in Swift ObjectStorage - </dd> - - <dt><strong>objectstorage.swift.authapi</strong></dt> - <dd> - Specify the authentication mechanism of Swift. - <strong>tmpauth</strong>:<a href="https://docs.openstack.org/swift/latest/overview_auth.html#tempauth";>Swift Temp Auth</a> - <strong>keystone2</strong>:<a href="https://docs.openstack.org/swift/latest/overview_auth.html#keystone-auth";>Swift Keystone Auth</a> - <strong>keystone3</strong>:<a href="https://docs.openstack.org/swift/latest/overview_auth.html#keystone-auth";>Swift Keystone Auth</a> - </dd> - - <dt><strong>objectstorage.swift.endpoint</strong></dt> - <dd> - authentication endpoint - </dd> - - <dt><strong>objectstorage.swift.credentials</strong></dt> - <dd> - authentication credential - </dd> - </dl> - - <subsection name="ObjectStorage Swift temp Authentication Configuration"> - <dl> - <dt><strong>objectstorage.swift.tempauth.username</strong></dt> - <dd>Username of Swift</dd> - - <dt><strong>objectstorage.swift.tempauth.tenantname</strong></dt> - <dd>Tenantname of Swift</dd> - - <dt><strong>objectstorage.swift.tempauth.passheadername</strong></dt> - <dd>Specify http header name for carrying credential information</dd> - - <dt><strong>objectstorage.swift.tempauth.userheadername</strong></dt> - <dd>Specify http header name for carrying identity information</dd> - </dl> - </subsection> - - <subsection name="ObjectStorage Swift keystone2 Authentication Configuration"> - <dl> - <dt><strong>objectstorage.swift.keystone2.username</strong></dt> - <dd>Similar to tempAuth</dd> - - <dt><strong>objectstorage.swift.keystone2.tenantname</strong></dt> - <dd>Similar to tempAuth</dd> - </dl> - </subsection> - - <subsection name="ObjectStorage Swift keystone3 Authentication Configuration"> - <dl> - <dt><strong>objectstorage.swift.keystone3.user.name</strong></dt> - <dd>Specifying username in identity information</dd> - - <dt><strong>objectstorage.swift.keystone3.user.domain</strong></dt> - <dd>Specifying user domain in identity information</dd> - - <dt><strong>objectstorage.swift.keystone3.scope.domainid</strong></dt> - <dd>Specifying scope domain in identity information</dd> - - <dt><strong>objectstorage.swift.keystone3.scope.project.name</strong></dt> - <dd> - Specifying project name in project information. - Equivalent to <strong>jclouds.keystone.scope</strong> jclouds keystore configuration - </dd> - - <dt><strong>objectstorage.swift.keystone3.scope.project.domainname</strong></dt> - <dd> - Specifying project domain name in project information. - Equivalent to <strong>jclouds.keystone.project-domain-name</strong> jclouds keystore configuration - </dd> - - <dt><strong>objectstorage.swift.keystone3.scope.project.domainid</strong></dt> - <dd> - Specifying project domain id in project information. - Equivalent to <strong>jclouds.keystone.project-domain-id</strong> jclouds keystore configuration - </dd> - </dl> - </subsection> - </subsection> <subsection name="ObjectStorage AWS S3 Configuration"> <dl> <dt><strong>objectstorage.s3.endPoint</strong></dt> <dd>S3 service endpoint</dd> + <dt><strong>objectstorage.s3.region</strong></dt> + <dd>S3 region</dd> + <dt><strong>objectstorage.s3.accessKeyId</strong></dt> <dd><a href="https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys";>S3 access key id</a></dd> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
