Repository: incubator-gobblin Updated Branches: refs/heads/master 492b93d54 -> af8462581
[GOBBLIN-224] Added a util function that can decrypt keyring based GPG file Closes #2076 from zxliucmu/gobblin-gpgdecryption Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/af846258 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/af846258 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/af846258 Branch: refs/heads/master Commit: af8462581977acb3e18468527fe8e4a6b54ed753 Parents: 492b93d Author: Zixuan Liu <[email protected]> Authored: Wed Aug 23 13:07:57 2017 -0700 Committer: Issac Buenrostro <[email protected]> Committed: Wed Aug 23 13:07:57 2017 -0700 ---------------------------------------------------------------------- .../apache/gobblin/crypto/GPGFileDecryptor.java | 133 ++++++++++++++++--- .../gobblin/crypto/GPGFileDecryptorTest.java | 61 +++++++++ .../crypto/gpg/KeyBasedEncryptionFile.txt.gpg | Bin 0 -> 383 bytes .../gpg/PasswordBasedEncryptionFile.txt.gpg | 2 + .../src/test/resources/crypto/gpg/private.key | 59 ++++++++ 5 files changed, 237 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af846258/gobblin-modules/gobblin-crypto/src/main/java/org/apache/gobblin/crypto/GPGFileDecryptor.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-crypto/src/main/java/org/apache/gobblin/crypto/GPGFileDecryptor.java b/gobblin-modules/gobblin-crypto/src/main/java/org/apache/gobblin/crypto/GPGFileDecryptor.java index ace76bd..ec28273 100644 --- a/gobblin-modules/gobblin-crypto/src/main/java/org/apache/gobblin/crypto/GPGFileDecryptor.java +++ b/gobblin-modules/gobblin-crypto/src/main/java/org/apache/gobblin/crypto/GPGFileDecryptor.java @@ -20,42 +20,48 @@ import java.io.IOException; import java.io.InputStream; import java.security.Security; +import java.util.Iterator; +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openpgp.PGPCompressedData; import org.bouncycastle.openpgp.PGPEncryptedDataList; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPLiteralData; +import org.bouncycastle.openpgp.PGPOnePassSignatureList; import org.bouncycastle.openpgp.PGPPBEEncryptedData; +import org.bouncycastle.openpgp.PGPPrivateKey; +import org.bouncycastle.openpgp.PGPPublicKeyEncryptedData; +import org.bouncycastle.openpgp.PGPSecretKey; +import org.bouncycastle.openpgp.PGPSecretKeyRingCollection; import org.bouncycastle.openpgp.PGPUtil; import org.bouncycastle.openpgp.jcajce.JcaPGPObjectFactory; +import org.bouncycastle.openpgp.operator.bc.BcKeyFingerprintCalculator; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; import org.bouncycastle.openpgp.operator.jcajce.JcePBEDataDecryptorFactoryBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; +import org.bouncycastle.openpgp.operator.jcajce.JcePublicKeyDataDecryptorFactoryBuilder; /** - * A utility class that decrypts password based encryption files. + * A utility class that decrypts both password based and key based encryption files. * * Code reference - org.bouncycastle.openpgp.examples.PBEFileProcessor + * - org.bouncycastle.openpgp.examples.KeyBasedFileProcessor */ +@UtilityClass public class GPGFileDecryptor { - public static InputStream decryptFile(InputStream inputStream, String passPhrase) throws IOException { - - if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { - Security.addProvider(new BouncyCastleProvider()); - } - inputStream = PGPUtil.getDecoderStream(inputStream); - - JcaPGPObjectFactory pgpF = new JcaPGPObjectFactory(inputStream); - PGPEncryptedDataList enc; - Object pgpfObject = pgpF.nextObject(); - - if (pgpfObject instanceof PGPEncryptedDataList) { - enc = (PGPEncryptedDataList) pgpfObject; - } else { - enc = (PGPEncryptedDataList) pgpF.nextObject(); - } + /** + * Taking in a file inputstream and a passPhrase, generate a decrypted file inputstream. + * @param inputStream file inputstream + * @param passPhrase passPhrase + * @return + * @throws IOException + */ + public InputStream decryptFile(InputStream inputStream, String passPhrase) throws IOException { + PGPEncryptedDataList enc = getPGPEncryptedDataList(inputStream); PGPPBEEncryptedData pbe = (PGPPBEEncryptedData) enc.get(0); InputStream clear; @@ -65,7 +71,7 @@ public class GPGFileDecryptor { .setProvider(BouncyCastleProvider.PROVIDER_NAME).build(passPhrase.toCharArray())); JcaPGPObjectFactory pgpFact = new JcaPGPObjectFactory(clear); - pgpfObject = pgpFact.nextObject(); + Object pgpfObject = pgpFact.nextObject(); if (pgpfObject instanceof PGPCompressedData) { PGPCompressedData cData = (PGPCompressedData) pgpfObject; pgpFact = new JcaPGPObjectFactory(cData.getDataStream()); @@ -78,4 +84,95 @@ public class GPGFileDecryptor { throw new IOException(e); } } + + /** + * Taking in a file inputstream, keyring inputstream and a passPhrase, generate a decrypted file inputstream. + * @param inputStream file inputstream + * @param keyIn keyring inputstream + * @param passPhrase passPhrase + * @return + * @throws IOException + */ + @SneakyThrows (PGPException.class) + public InputStream decryptFile(InputStream inputStream, InputStream keyIn, String passPhrase) + throws IOException { + + PGPEncryptedDataList enc = getPGPEncryptedDataList(inputStream); + Iterator it = enc.getEncryptedDataObjects(); + PGPPrivateKey sKey = null; + PGPPublicKeyEncryptedData pbe =null; + PGPSecretKeyRingCollection pgpSec = new PGPSecretKeyRingCollection( + PGPUtil.getDecoderStream(keyIn), new BcKeyFingerprintCalculator()); + + while(sKey == null && it.hasNext()) { + pbe = (PGPPublicKeyEncryptedData)it.next(); + sKey = findSecretKey(pgpSec, pbe.getKeyID(), passPhrase); + } + + if (sKey == null) { + throw new IllegalArgumentException("secret key for message not found."); + } + + try (InputStream clear = pbe.getDataStream( + new JcePublicKeyDataDecryptorFactoryBuilder().setProvider(BouncyCastleProvider.PROVIDER_NAME).build(sKey))) { + + JcaPGPObjectFactory pgpFact = new JcaPGPObjectFactory(clear); + Object pgpfObject = pgpFact.nextObject(); + + if (pgpfObject instanceof PGPCompressedData) { + PGPCompressedData cData = (PGPCompressedData) pgpfObject; + pgpFact = new JcaPGPObjectFactory(cData.getDataStream()); + pgpfObject = pgpFact.nextObject(); + PGPLiteralData ld = (PGPLiteralData) pgpfObject; + return ld.getInputStream(); + } else if (pgpfObject instanceof PGPOnePassSignatureList) { + throw new PGPException("encrypted message contains a signed message - not literal data."); + } else { + throw new PGPException("message is not a simple encrypted file - type unknown."); + } + } + } + + /** + * Private util function that finds the private key from keyring collection based on keyId and passPhrase + * @param pgpSec keyring collection + * @param keyID keyID for this encryption file + * @param passPhrase passPhrase for this encryption file + * @throws PGPException + */ + private PGPPrivateKey findSecretKey(PGPSecretKeyRingCollection pgpSec, long keyID, String passPhrase) + throws PGPException { + + PGPSecretKey pgpSecKey = pgpSec.getSecretKey(keyID); + if (pgpSecKey == null) { + return null; + } + return pgpSecKey.extractPrivateKey( + new JcePBESecretKeyDecryptorBuilder() + .setProvider(BouncyCastleProvider.PROVIDER_NAME).build(passPhrase.toCharArray())); + } + + /** + * Generate a PGPEncryptedDataList from an inputstream + * @param inputStream file inputstream that needs to be decrypted + * @throws IOException + */ + private PGPEncryptedDataList getPGPEncryptedDataList(InputStream inputStream) throws IOException { + + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + inputStream = PGPUtil.getDecoderStream(inputStream); + + JcaPGPObjectFactory pgpF = new JcaPGPObjectFactory(inputStream); + PGPEncryptedDataList enc; + Object pgpfObject = pgpF.nextObject(); + + if (pgpfObject instanceof PGPEncryptedDataList) { + enc = (PGPEncryptedDataList) pgpfObject; + } else { + enc = (PGPEncryptedDataList) pgpF.nextObject(); + } + return enc; + } } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af846258/gobblin-modules/gobblin-crypto/src/test/java/org/apache/gobblin/crypto/GPGFileDecryptorTest.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-crypto/src/test/java/org/apache/gobblin/crypto/GPGFileDecryptorTest.java b/gobblin-modules/gobblin-crypto/src/test/java/org/apache/gobblin/crypto/GPGFileDecryptorTest.java new file mode 100644 index 0000000..437caf6 --- /dev/null +++ b/gobblin-modules/gobblin-crypto/src/test/java/org/apache/gobblin/crypto/GPGFileDecryptorTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.crypto; + +import com.google.common.base.Charsets; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + + +/** + * Test class for {@link GPGFileDecryptor} + * Test key and test passphrase are generated offline + */ +public class GPGFileDecryptorTest { + + private static final String fileDir = "src/test/resources/crypto/gpg/"; + private static final String privateKey = "private.key"; + private static final String passwdBasedFile = "PasswordBasedEncryptionFile.txt.gpg"; + private static final String keyBasedFile = "KeyBasedEncryptionFile.txt.gpg"; + private static final String passPhrase = "test"; + + private static final String expectedPasswdFileContent = "This is a password based encryption file.\n"; + private static final String expectedKeyFileContent = "This is a key based encryption file.\n"; + + @Test + public void keyBasedDecryptionTest() throws IOException { + try(InputStream is = GPGFileDecryptor.decryptFile( + FileUtils.openInputStream( + new File(fileDir, keyBasedFile)), FileUtils.openInputStream(new File(fileDir, privateKey)), passPhrase)) { + Assert.assertEquals(IOUtils.toString(is, Charsets.UTF_8), expectedKeyFileContent); + } + } + + @Test + public void passwordBasedDecryptionTest() throws IOException { + try(InputStream is = GPGFileDecryptor.decryptFile( + FileUtils.openInputStream(new File(fileDir, passwdBasedFile)), passPhrase)) { + Assert.assertEquals(IOUtils.toString(is, Charsets.UTF_8), expectedPasswdFileContent); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af846258/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/KeyBasedEncryptionFile.txt.gpg ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/KeyBasedEncryptionFile.txt.gpg b/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/KeyBasedEncryptionFile.txt.gpg new file mode 100644 index 0000000..21aefd8 Binary files /dev/null and b/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/KeyBasedEncryptionFile.txt.gpg differ http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af846258/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/PasswordBasedEncryptionFile.txt.gpg ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/PasswordBasedEncryptionFile.txt.gpg b/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/PasswordBasedEncryptionFile.txt.gpg new file mode 100644 index 0000000..69c203f --- /dev/null +++ b/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/PasswordBasedEncryptionFile.txt.gpg @@ -0,0 +1,2 @@ +� ��$���"���N�%�Ô�[/3u J�0�7�A�|o�PNù:p��C�S��_�(�(���Wc�zFo��K_�Uz���=4 +c�@�I�3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af846258/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/private.key ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/private.key b/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/private.key new file mode 100644 index 0000000..6aa7da2 --- /dev/null +++ b/gobblin-modules/gobblin-crypto/src/test/resources/crypto/gpg/private.key @@ -0,0 +1,59 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v2.0.14 (GNU/Linux) + +lQO+BFmcyxwBCACwp8BUiYHqyaTnzyi3YJ/u6Fb+LxBkAnb1y3WoIogLXQIG7EL1 +LWv250xTQYa3iLnD5dBYZjndHqqDKjjyniik7eXTrVqcFikM1E4tndRDzJhY1oPZ +HAT+vc5yFGcL2unDCGt95IDJLDP2nVWz0IFtrQr/cu0BpmH07GG3zbbmzAOleTUJ +IvRBX0svalyLxrOEwg/4wBA7qVlRVAN+ALfTWnOFWf06uzjPTNOYm50uMgnHCHjE ++4czHA1APdwVhYeJEve72zLf6YKRs39ViHqBDHDAMLPY618r0+t0oxUikgwlrIoP +9XTRXUr6/du2ie/ZtEknKO/PfEDpD7BoaFK/ABEBAAH+AgMCHQojM9GPUbTS40dm +7OvwOENmyVcKPddLoS2tH/jLHs0DWyrwx2jR/yQP53byupgJOKOYnsXBlAtPAlvz +w5ekMGq+owJbSqcsaODr7DLC/cOJhwPrrjDRdy0kcoh+7zXNjlj0WhKChld9Ou2b +Yye7MP1/gjdUbErwTxFdrYOM6wxGoSr8NR1CayOEzfbfAB1ls0+0mwLBM4aHbTVE +pE0aNUQ6hrsU6dgfNcms+MEgZ1gSwYNVM/WK2Iqa3RDdPdBMrESMsDUUQ4hO+G50 +otmztI1bHiMlj9jY87u0WJhr5jhITmUTXC56gFRhCOFxGygrfvWYb2GbpHPovY1/ +ry2ahL35IyqL+0R8/yilb3TiQMkWFxlYlPFmgLeLI1EGaHOlf0NXu++cF2dB3IbG +YSRtRJkxn0cP9HapTpJo8m84LNbMEZZXbalCC1YWQfVPXYjxy2H3S3K6yV2D29oj +NtlmYVMNlcTzdGzvz1wlrTgZPW/6CxGwyXKxn+gE58jzjnnj4jHUq/3QlTfksqXO +AAWBM9RRW8cedKu0G7DdXDdpdq+hU3WT/8souJinXLqdSAPVMMtjsHSsAC6WYXkK +KycUYh/JoPoHitSz3DkreSM4Lx3sUGR3FtANC3ta1vdl+BvLgB/5OEP9TsXgA6qB +hmXfF2dxU+L76vcrcylM5L575cXHhQTaDRfCliimZjosAnpaPNe0rapv2CI1QGdn +ChfwcKiaZnaox/9qHtGSLYoNSaIb49im3mI/0FAcn2WrGCKWTnraMNa8gkjsotKa +NAXPaSufGJ6fAPRDfTXJt3ibePafFAGBH0zHdlH+0BD2wDfZc/k1aa47sx3DDAFA +KBi1+fe05k0NsVaD7xaaxw3FdaHLFYQ7OEXUaZ2kM/cwORbthme62S+lG/Y4Fzy7 +SrQ8dGVzdCBrZXkgKFRoaXMgaXMgb25seSB1c2VkIGZvciB1bml0IHRlc3QpIDx0 +ZXN0QGxpbmtlZC5jb20+iQE3BBMBAgAiBQJZnMscAhsDBgsJCAcDAgYVCAIJCgsE +FgIDAQIeAQIXgAAKCRDXinc+vrjyqHRgB/Yl0exXWsF9/tWx4Id4/NtoHSOlK5qP +MVvoiqLVJaEHOWdWqnPzlETgFZ4LYpmSYyPk5r/goNEsezonSqEQKFPbb2Ipq+qh +tb+/s619N9cYgOKIK22Rjod83vF1LGmtT3+3uLLtpBGXkJnGrKCfMVvpZHby3/TN +g7NEtybrAyZG2S6wEiQA0hZn3SoK1sT9TKV2dkjc/m6louFUcSbSB8OnaNr9UrUQ +1KUtxCPhrspRcNSBjNNXZo5XCHkallZglXxE4nDOX+hmPBISPJT/NrD4/e0LFsEX +RgFHJvqto57p3JeYoKhJFjAsxYt8/s4HsgNgpKeT12OzGg/48GRZ9yCdA74EWZzL +HAEIAOcNJVZ4F8+d/nqXxf+imNZQduHkZJ94yJoRqfQVM1/KDHtpXk2wl4Zarity +xe6j5j7D1UU5z1N57tRAOGsfFn+facqsawI+klZsaa1BgMvHqOGqIRAVu2zDqkIN +ZOlWUvCy4LMgVfbkelbc9U92s0USQOwappUrNK2kxVjFjDTKIO4vo+za7qGLA5Ig +/9ymRdwiHwkygNNS5MZy47TI2nFCHCOFP5vl59FBWJoXrmifqsVw6Yqov97FSNv7 +l0SGBqgx81Cijqjji++Fx+c49NN/ffD4wvRzcZk2pGzJE0YWrCEZM1F/a6XcYqwg +4Q/WJfoDl/rIuJLaFgJxfcYqG2EAEQEAAf4CAwIdCiMz0Y9RtNIp6FqVcc7XuoA2 +IsefGFwV0bKWu1Pq1yU/WTYq9fez0CkR/+CYs4ujkMuEJA4SbQ16BLtbZLxI/pzm +tWN2JYHwDUd6RB6nS5ETuUAdmEkmMRT1ytmYdCqyvlZTLusuTzsQaRhkH7ezHlvu +K5xyntauiiHp4max+dY0tp7xraoJt50/bz2ewaCnpzwnxnjERQGVgoAXalF2wNh0 ++L+Vn2Mq7bt5Ge61JxIVrgjgbAfwgCAjsUW50DT4pxBWCwO+lAYOqsyRRyklv8Kb +NInxrrmE8i/5JjlN5YfjrFAeiEvLobf8Z38P1yeovR3t8BHwRtWok9ZBvKfhDzWe ++pYnr6R6egTBTVlLT9BqEeZ12ax+WM1Ao5VHZfrdJEWdd4QufOkj2Wo1EnXbpTNz +OmjB4yqmCbGUulmMszuMU9kaxPH3O0n9QPipPFREI6M/KjiOQcGQsK+2uAva5P9j +zdHUP5Oyf7oL91Jc+MwsLN4IW4Id/rifmF0rQ5f0s/wohLXMW0xUJEfRpEvUERc4 +E0ytnjTxDEwKuoO4B3f7vkPvph54ZpM67RUM5v0/K6SmjRMIDYT7nSqESzR8X2B3 +++x3AfGEdLrh7umrsULT9DYa2LM3S9MhQzamC1uqJWEvMMHj2Jkez2d+pCRUl5p3 +VhUKXjXT3dia27jllMhQy33jk6HPVj2/S7zPXncswxiSGfe3epwV++LxuxRBxLlO +pX1UXsym96lbA4PM3PaevhBtzfpUuoYY7NvWtEHvRfuhBcBo0dlGZRp2/vQ3gVyq +G8yFLwG3+RcK8Se4ITwYZmARaeGpBvNCwkg4geCRpjpsRVBJ22f5jD2NJiLCMsrf +p1r2DFiTyc+Chjryt2Qr6KmAw9AP5ThutrjQYyll2rXy9PecG8Pkr2ZmiQEfBBgB +AgAJBQJZnMscAhsMAAoJENeKdz6+uPKoNSoH/2eNjH+heStK2uhkLuxb0W+m/O1m +xNaSXt/USnfaLjQV7LSl6MvefmgFtL869jJaFBGaTsMsxTwnPvBYLk090170ekTl +bozjuFa4qftE7kW3MUo1s2quF9995BvnBpBl3Yb5EuC6RBOTaTDSJYtcp08A9QTh +anz7PIVYS5UU5ZXF940fm5148A8wa0xVoeWrb6cSad1w1qD3kM92vc+z2PbtbalO +IiPG8hIHNdC2f1ZVMDRfoGZNkWTHX7laMHBis0T/tZ3ZObmtDVQvjAak2qoIGrwC +Z1oS973uZZYdeIF5ylNAowhxlog0kF2kYeqRDf/rEkBgX2nXbGPRjREGWC4= +=Cu43 +-----END PGP PRIVATE KEY BLOCK-----
