Repository: incubator-gobblin Updated Branches: refs/heads/master f861dca32 -> a81a3288d
[GOBBLIN-662] Enhance SSH-based access to Git to enable/disable host key checking. Closes #2533 from sv2000/hostKeyCheck Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/a81a3288 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/a81a3288 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/a81a3288 Branch: refs/heads/master Commit: a81a3288daf18b488eb1a142dce9fe7aee8bd2cf Parents: f861dca Author: suvasude <[email protected]> Authored: Mon Jan 14 09:22:40 2019 -0800 Committer: Hung Tran <[email protected]> Committed: Mon Jan 14 09:22:40 2019 -0800 ---------------------------------------------------------------------- .../configuration/ConfigurationKeys.java | 6 ++- .../modules/core/GitMonitoringService.java | 48 ++++++++++++++++---- 2 files changed, 45 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/a81a3288/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java index 34b227e..c0feda1 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java @@ -926,7 +926,11 @@ public class ConfigurationKeys { public static final String GIT_MONITOR_PASSWORD = "password"; //Configuration keys for authentication using SSH with Public Key public static final String GIT_MONITOR_SSH_WITH_PUBLIC_KEY_ENABLED = "isSshWithPublicKeyEnabled"; - public static final String GIT_MONITOR_PRIVATE_KEY_PATH = "privateKeyPath"; + public static final String GIT_MONITOR_SSH_PRIVATE_KEY_PATH = "privateKeyPath"; + public static final String GIT_MONITOR_SSH_PRIVATE_KEY_BASE64_ENCODED = "privateKeyBase64"; public static final String GIT_MONITOR_SSH_PASSPHRASE = "passphrase"; + public static final String GIT_MONITOR_SSH_STRICT_HOST_KEY_CHECKING_ENABLED = "isStrictHostKeyCheckingEnabled"; + public static final String GIT_MONITOR_SSH_KNOWN_HOSTS = "knownHosts"; + public static final String GIT_MONITOR_SSH_KNOWN_HOSTS_FILE = "knownHostsFile"; public static final String GIT_MONITOR_JSCH_LOGGER_ENABLED = "isJschLoggerEnabled"; } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/a81a3288/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GitMonitoringService.java ---------------------------------------------------------------------- diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GitMonitoringService.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GitMonitoringService.java index 11694f2..d577bdd 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GitMonitoringService.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GitMonitoringService.java @@ -17,10 +17,12 @@ package org.apache.gobblin.service.modules.core; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,11 +31,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.eclipse.jgit.api.CloneCommand; -import org.eclipse.jgit.api.FetchCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportConfigCallback; @@ -88,8 +89,12 @@ public abstract class GitMonitoringService extends AbstractIdleService { private final ScheduledExecutorService scheduledExecutor; private String privateKeyPath; + private byte[] privateKey; private String passphrase; private boolean isJschLoggerEnabled; + private boolean strictHostKeyCheckingEnabled; + private String knownHosts; + private String knownHostsFile; final GitMonitoringService.GitRepository gitRepo; final String repositoryDir; @@ -114,16 +119,30 @@ public abstract class GitMonitoringService extends AbstractIdleService { Either<CredentialsProvider, SshSessionFactory> providerSessionFactoryEither; boolean isSshWithPublicKeyEnabled = ConfigUtils.getBoolean(config, ConfigurationKeys.GIT_MONITOR_SSH_WITH_PUBLIC_KEY_ENABLED, false); if (isSshWithPublicKeyEnabled) { - this.privateKeyPath = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_PRIVATE_KEY_PATH, null); - if (Strings.isNullOrEmpty(this.privateKeyPath)) { - throw new RuntimeException("Path to private key must be provided"); + this.privateKeyPath = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_SSH_PRIVATE_KEY_PATH, null); + String privateKeyBase64Encoded = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_SSH_PRIVATE_KEY_BASE64_ENCODED, null); + + if ((Strings.isNullOrEmpty(this.privateKeyPath)) && ((Strings.isNullOrEmpty(privateKeyBase64Encoded)))) { + throw new RuntimeException("Path to private key or private key string must be provided"); + } + + if (!Strings.isNullOrEmpty(privateKeyBase64Encoded)) { + this.privateKey = Base64.decodeBase64(privateKeyBase64Encoded); } + String passPhraseEnc = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_SSH_PASSPHRASE, null); - if (passPhraseEnc != null) { + if (!Strings.isNullOrEmpty(passPhraseEnc)) { this.passphrase = passwordManager.readPassword(passPhraseEnc); } providerSessionFactoryEither = Either.right(getSshSessionFactory()); this.isJschLoggerEnabled = ConfigUtils.getBoolean(config, ConfigurationKeys.GIT_MONITOR_JSCH_LOGGER_ENABLED, false); + this.strictHostKeyCheckingEnabled = ConfigUtils.getBoolean(config, ConfigurationKeys.GIT_MONITOR_SSH_STRICT_HOST_KEY_CHECKING_ENABLED, + true); + this.knownHosts = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_SSH_KNOWN_HOSTS, null); + this.knownHostsFile = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_SSH_KNOWN_HOSTS_FILE, null); + if (strictHostKeyCheckingEnabled && Strings.isNullOrEmpty(knownHostsFile) && Strings.isNullOrEmpty(knownHosts)) { + throw new RuntimeException("Either StrictHostKeyChecking should be disabled or a knownHostFile or knownHosts string must be provided"); + } } else { //Use CredentialsProvider String username = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_USERNAME, null); String passwordEnc = ConfigUtils.getString(config, ConfigurationKeys.GIT_MONITOR_PASSWORD, null); @@ -417,7 +436,9 @@ public abstract class GitMonitoringService extends AbstractIdleService { JschConfigSessionFactory sessionFactory = new JschConfigSessionFactory() { @Override protected void configure(OpenSshConfig.Host hc, Session session) { - //Do nothing. + if (!GitMonitoringService.this.strictHostKeyCheckingEnabled) { + session.setConfig("StrictHostKeyChecking", "no"); + } } @Override @@ -426,7 +447,18 @@ public abstract class GitMonitoringService extends AbstractIdleService { JSch.setLogger(new JschLogger()); } JSch defaultJSch = super.createDefaultJSch(fs); - defaultJSch.addIdentity(GitMonitoringService.this.privateKeyPath, GitMonitoringService.this.passphrase); + defaultJSch.getIdentityRepository().removeAll(); + if (GitMonitoringService.this.privateKeyPath != null) { + defaultJSch.addIdentity(GitMonitoringService.this.privateKeyPath, GitMonitoringService.this.passphrase); + } else { + defaultJSch.addIdentity("gaas-git", GitMonitoringService.this.privateKey, null, + GitMonitoringService.this.passphrase.getBytes(Charset.forName("UTF-8"))); + } + if (!Strings.isNullOrEmpty(GitMonitoringService.this.knownHosts)) { + defaultJSch.setKnownHosts(new ByteArrayInputStream(GitMonitoringService.this.knownHosts.getBytes(Charset.forName("UTF-8")))); + } else if (!Strings.isNullOrEmpty(GitMonitoringService.this.knownHostsFile)) { + defaultJSch.setKnownHosts(GitMonitoringService.this.knownHostsFile); + } return defaultJSch; } };
