Repository: zeppelin Updated Branches: refs/heads/master e9caebcfa -> 5bb38c89a
[ZEPPELIN-1465] Add an option to allow S3 server-side encryption ### What is this PR for? Provide a configuration option that will cause the S3 Notebook repo to request server-side encryption of saved notebooks. ### What type of PR is it? Improvement ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1465 ### How should this be tested? Enable the configuration option, save a notebook in zeppelin, and confirm in the AWS S3 Console that the related file was saved with AES-256 encryption on the server-side. (Properties tab, Detail section) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No. * Does this needs documentation? I added mentions of the new option in existing documentation. Thank you! Author: Jeff Plourde <jplou...@cyft.io> Closes #1969 from jeff-cyft/s3_sse and squashes the following commits: 26f5264 [Jeff Plourde] code style - remove tab 3c657ac [Jeff Plourde] Configuration option to request S3 SSE when notebooks are saved. Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5bb38c89 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5bb38c89 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5bb38c89 Branch: refs/heads/master Commit: 5bb38c89ae67f95858547f73d0e833ef91b3d6ee Parents: e9caebc Author: Jeff Plourde <jplou...@cyft.io> Authored: Thu Feb 2 11:09:57 2017 -0500 Committer: Lee moon soo <m...@apache.org> Committed: Sun Feb 5 15:55:08 2017 +0900 ---------------------------------------------------------------------- conf/zeppelin-env.cmd.template | 1 + conf/zeppelin-env.sh.template | 1 + conf/zeppelin-site.xml.template | 8 ++++++++ docs/install/configuration.md | 6 ++++++ docs/storage/storage.md | 18 ++++++++++++++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 5 +++++ .../zeppelin/notebook/repo/S3NotebookRepo.java | 15 ++++++++++++++- 7 files changed, 53 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/conf/zeppelin-env.cmd.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index 5fc3acf..1bbde86 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -34,6 +34,7 @@ REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook REM set ZEPPELIN_NOTEBOOK_S3_ENDPOINT REM Endpoint of the bucket REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID REM AWS KMS key ID REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION REM AWS KMS key region +REM set ZEPPELIN_NOTEBOOK_S3_SSE REM Server-side encryption enabled for notebooks REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default. REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/conf/zeppelin-env.sh.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 7e777b6..46fd481 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -35,6 +35,7 @@ # export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json # export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID # export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region +# export ZEPPELIN_NOTEBOOK_S3_SSE # Server-side encryption enabled for notebooks # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index bd8d7dd..abaff30 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -129,6 +129,14 @@ </property> --> +<!-- Server-side encryption enabled for notebooks --> +<!-- +<property> + <name>zeppelin.notebook.s3.sse</name> + <value>true</value> + <description>Server-side encryption enabled for notebooks</description> +</property> +--> <!-- If using Azure for storage use the following settings --> <!-- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/docs/install/configuration.md ---------------------------------------------------------------------- diff --git a/docs/install/configuration.md b/docs/install/configuration.md index befb520..a8ebf54 100644 --- a/docs/install/configuration.md +++ b/docs/install/configuration.md @@ -195,6 +195,12 @@ If both are defined, then the **environment variables** will take priority. <td>Class name of a custom S3 encryption materials provider implementation to use for encrypting data in S3 (optional)</td> </tr> <tr> + <td>ZEPPELIN_NOTEBOOK_S3_SSE</td> + <td>zeppelin.notebook.s3.sse</td> + <td>false</td> + <td>Save notebooks to S3 with server-side encryption enabled</td> + </tr> + <tr> <td>ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</td> <td>zeppelin.notebook.azure.connectionString</td> <td></td> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/docs/storage/storage.md ---------------------------------------------------------------------- diff --git a/docs/storage/storage.md b/docs/storage/storage.md index 0ab01da..73388da 100644 --- a/docs/storage/storage.md +++ b/docs/storage/storage.md @@ -165,6 +165,24 @@ Or using the following setting in **zeppelin-site.xml**: <description>Custom encryption materials provider used to encrypt notebook data in S3</description> ``` +#### Enable server-side encryption + +To request server-side encryption of notebooks, set the following environment variable in the file **zeppelin-env.sh**: + +``` +export ZEPPELIN_NOTEBOOK_S3_SSE = true +``` + +Or using the following setting in **zeppelin-site.xml**: + +``` +<property> + <name>zeppelin.notebook.s3.sse</name> + <value>true</value> + <description>Server-side encryption enabled for notebooks</description> +</property> +``` + </br> ## Notebook Storage in Azure <a name="Azure"></a> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 2c8d91c..0708719 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -380,6 +380,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP); } + public boolean isS3ServerSideEncryption() { + return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE); + } + public String getInterpreterListPath() { return getRelativeDir(String.format("%s/interpreter-list", getConfDir())); } @@ -587,6 +591,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null), ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null), ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null), + ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false), ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null), ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"), ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"), http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5bb38c89/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 26781b8..bd7fe1a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -55,6 +55,7 @@ import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; @@ -86,12 +87,14 @@ public class S3NotebookRepo implements NotebookRepo { private final AmazonS3 s3client; private final String bucketName; private final String user; + private final boolean useServerSideEncryption; private final ZeppelinConfiguration conf; public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException { this.conf = conf; bucketName = conf.getBucketName(); user = conf.getUser(); + useServerSideEncryption = conf.isS3ServerSideEncryption(); // always use the default provider chain AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); @@ -234,7 +237,17 @@ public class S3NotebookRepo implements NotebookRepo { Writer writer = new OutputStreamWriter(new FileOutputStream(file)); writer.write(json); writer.close(); - s3client.putObject(new PutObjectRequest(bucketName, key, file)); + + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file); + + if (useServerSideEncryption) { + // Request server-side encryption. + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + putRequest.setMetadata(objectMetadata); + } + + s3client.putObject(putRequest); } catch (AmazonClientException ace) { throw new IOException("Unable to store note in S3: " + ace, ace);