Repository: incubator-gobblin Updated Branches: refs/heads/master 0ae37a849 -> 5eee52d93
[GOBBLIN-567] Create config store that downloads and reads from a local jar Closes #2430 from jack-moseley/zip-config-store Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/5eee52d9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/5eee52d9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/5eee52d9 Branch: refs/heads/master Commit: 5eee52d93ffe663c1b6c7de3a0e3f2d9898708e1 Parents: 0ae37a8 Author: Jack Moseley <[email protected]> Authored: Tue Sep 11 09:10:47 2018 -0700 Committer: Hung Tran <[email protected]> Committed: Tue Sep 11 09:10:47 2018 -0700 ---------------------------------------------------------------------- .../hdfs/SimpleHDFSConfigStoreFactory.java | 8 +- .../store/hdfs/SimpleHDFSStoreMetadata.java | 4 +- .../hdfs/SimpleHadoopFilesystemConfigStore.java | 30 ++- .../config/store/zip/IvyConfigStoreFactory.java | 116 +++++++++++ .../config/store/zip/ZipFileConfigStore.java | 199 +++++++++++++++++++ ....gobblin.config.store.api.ConfigStoreFactory | 1 + .../store/zip/ZipFileConfigStoreTest.java | 101 ++++++++++ .../src/test/resources/zipStoreTest.zip | Bin 0 -> 1821 bytes gobblin-utility/build.gradle | 1 + .../org/apache/gobblin/util/DownloadUtils.java | 96 +++++++++ gradle/scripts/dependencyDefinitions.gradle | 1 + 11 files changed, 537 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java index b20d1e2..cfa36cf 100644 --- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java +++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSConfigStoreFactory.java @@ -18,20 +18,14 @@ package org.apache.gobblin.config.store.hdfs; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; -import com.google.common.base.Strings; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; -import org.apache.gobblin.config.store.api.ConfigStoreCreationException; import org.apache.gobblin.config.store.api.ConfigStoreFactory; import org.apache.gobblin.util.ConfigUtils; @@ -44,7 +38,7 @@ import org.apache.gobblin.util.ConfigUtils; */ public class SimpleHDFSConfigStoreFactory extends SimpleHadoopFilesystemConfigStoreFactory { - protected static final String HDFS_SCHEME_NAME = "hdfs"; + public static final String HDFS_SCHEME_NAME = "hdfs"; /** Instantiates a new instance using standard typesafe config defaults: * {@link ConfigFactory#load()} */ http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java index 98d79d2..d196449 100644 --- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java +++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHDFSStoreMetadata.java @@ -55,7 +55,7 @@ public class SimpleHDFSStoreMetadata { * @param fs where metadata is stored * @param configStoreDir path to {@link SimpleHadoopFilesystemConfigStore#CONFIG_STORE_NAME} */ - SimpleHDFSStoreMetadata(final FileSystem fs, final Path configStoreDir) { + public SimpleHDFSStoreMetadata(final FileSystem fs, final Path configStoreDir) { this.storeMetadataFilePath = new Path(configStoreDir, CONFIG_STORE_METADATA_FILENAME); this.fs = fs; } @@ -119,7 +119,7 @@ public class SimpleHDFSStoreMetadata { * Get the current version from {@link #CONFIG_STORE_METADATA_FILENAME} file at {@link #storeMetadataFilePath} * */ - String getCurrentVersion() throws IOException { + public String getCurrentVersion() throws IOException { return readMetadata().getString(CONFIG_STORE_METADATA_CURRENT_VERSION_KEY); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java index da541e2..4008065 100644 --- a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java +++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/hdfs/SimpleHadoopFilesystemConfigStore.java @@ -122,10 +122,10 @@ import org.apache.gobblin.util.io.StreamUtils; @ConfigStoreWithStableVersioning public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployable<FsDeploymentConfig> { - protected static final String CONFIG_STORE_NAME = "_CONFIG_STORE"; + public static final String CONFIG_STORE_NAME = "_CONFIG_STORE"; - private static final String MAIN_CONF_FILE_NAME = "main.conf"; - private static final String INCLUDES_CONF_FILE_NAME = "includes.conf"; + public static final String MAIN_CONF_FILE_NAME = "main.conf"; + public static final String INCLUDES_CONF_FILE_NAME = "includes.conf"; private static final String INCLUDES_KEY_NAME = "includes"; private final FileSystem fs; @@ -265,14 +265,7 @@ public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployabl FileStatus includesFileStatus = this.fs.getFileStatus(includesFile); if (!includesFileStatus.isDirectory()) { try (InputStream includesConfInStream = this.fs.open(includesFileStatus.getPath())) { - /* - * The includes returned are used to build a fallback chain. - * With the natural order, if a key found in the first include it is not be overriden by the next include. - * By reversing the list, the Typesafe fallbacks are constructed bottom up. - */ - configKeyPaths.addAll(Lists.newArrayList( - Iterables.transform(Lists.reverse(resolveIncludesList(IOUtils.readLines(includesConfInStream, Charsets.UTF_8), runtimeConfig)), - new IncludesToConfigKey()))); + configKeyPaths.addAll(getResolvedConfigKeyPaths(includesConfInStream, runtimeConfig)); } } } catch (IOException e) { @@ -283,6 +276,21 @@ public class SimpleHadoopFilesystemConfigStore implements ConfigStore, Deployabl } /** + * Get resolved config key paths given an includes file as an {@link InputStream} + * + * The includes returned are used to build a fallback chain. + * With the natural order, if a key found in the first include it is not be overriden by the next include. + * By reversing the list, the Typesafe fallbacks are constructed bottom up. + * + * @param includesConfInStream includes.conf file as an {@link InputStream} + * @return a {@link List} of resolved ConfigKeyPaths + */ + public static List<ConfigKeyPath> getResolvedConfigKeyPaths(InputStream includesConfInStream, Optional<Config> runtimeConfig) throws IOException { + return Lists.newArrayList(Iterables.transform(Lists.reverse(resolveIncludesList( + IOUtils.readLines(includesConfInStream, Charsets.UTF_8), runtimeConfig)), new IncludesToConfigKey())); + } + + /** * A helper to resolve System properties and Environment variables in includes paths * The method loads the list of unresolved <code>includes</code> into an in-memory {@link Config} object and reolves * with a fallback on {@link ConfigFactory#defaultOverrides()} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java new file mode 100644 index 0000000..dba7d3a --- /dev/null +++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/IvyConfigStoreFactory.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.config.store.zip; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Paths; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; + +import com.sun.nio.zipfs.ZipFileSystem; + +import org.apache.gobblin.config.store.api.ConfigStoreCreationException; +import org.apache.gobblin.config.store.api.ConfigStoreFactory; +import org.apache.gobblin.config.store.hdfs.SimpleHDFSConfigStoreFactory; +import org.apache.gobblin.config.store.hdfs.SimpleHDFSStoreMetadata; +import org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore; +import org.apache.gobblin.util.DownloadUtils; + + +/** + * {@link ConfigStoreFactory} that downloads a jar file containing the config store paths through ivy and creates a + * {@link ZipFileConfigStore} with it. May be useful to avoid making many HDFS calls for large config stores. + * + * An ivy settings file must be present on the classpath named {@link DownloadUtils#IVY_SETTINGS_FILE_NAME} + */ +public class IvyConfigStoreFactory implements ConfigStoreFactory<ZipFileConfigStore> { + + private static final String IVY_SCHEME_PREFIX = "ivy-"; + private static final String ORG_KEY = "org"; + private static final String MODULE_KEY = "module"; + private static final String STORE_PREFIX_KEY = "storePrefix"; + + @Override + public String getScheme() { + return getSchemePrefix() + SimpleHDFSConfigStoreFactory.HDFS_SCHEME_NAME; + } + + public String getSchemePrefix() { + return IVY_SCHEME_PREFIX; + } + + /** + * Example configKey URI (configuration is passed as part of the query) + * + * ivy-hdfs://<hdfsURI>/path/to/config/store?org=<jarOrg>&module=<jarModule>&storePrefix=_CONFIG_STORE + * + * ivy-hdfs: scheme for this factory + * hdfsURI/path/to/config/store: location of HDFS config store (used for getting current version) + * org/module: org and module of jar containing config store + * storePrefix: prefix to paths in config store + */ + @Override + public ZipFileConfigStore createConfigStore(URI configKey) throws ConfigStoreCreationException { + if (!configKey.getScheme().equals(getScheme())) { + throw new ConfigStoreCreationException(configKey, "Config key URI must have scheme " + getScheme()); + } + + Properties factoryProps = new Properties(); + for (NameValuePair param : URLEncodedUtils.parse(configKey, "UTF-8")) { + factoryProps.setProperty(param.getName(), param.getValue()); + } + + String jarOrg = factoryProps.getProperty(ORG_KEY); + String jarModule = factoryProps.getProperty(MODULE_KEY); + + if (jarOrg == null || jarModule == null) { + throw new ConfigStoreCreationException(configKey, "Config key URI must contain org and module to download from"); + } + + try { + SimpleHDFSStoreMetadata metadata = new SimpleHDFSStoreMetadata( + org.apache.hadoop.fs.FileSystem.get(new Configuration()), new Path(configKey.getPath(), + SimpleHadoopFilesystemConfigStore.CONFIG_STORE_NAME)); + String currentVersion = metadata.getCurrentVersion(); + + URI[] uris = DownloadUtils.downloadJar(jarOrg, jarModule, currentVersion, false); + + if (uris.length != 1) { + throw new ConfigStoreCreationException(configKey, "Expected one jar file from URI"); + } + + FileSystem zipFs = FileSystems.newFileSystem(Paths.get(uris[0].getPath()), null); + + if (!(zipFs instanceof ZipFileSystem)) { + throw new ConfigStoreCreationException(configKey, "Downloaded file must be a zip or jar file"); + } + + return new ZipFileConfigStore((ZipFileSystem) zipFs, configKey, currentVersion, factoryProps.getProperty(STORE_PREFIX_KEY, "")); + } catch (IOException e) { + throw new ConfigStoreCreationException(configKey, e); + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java new file mode 100644 index 0000000..5475b49 --- /dev/null +++ b/gobblin-config-management/gobblin-config-core/src/main/java/org/apache/gobblin/config/store/zip/ZipFileConfigStore.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.config.store.zip; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; + +import org.apache.commons.lang.StringUtils; + +import com.google.common.base.Charsets; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.sun.nio.zipfs.ZipFileSystem; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import org.apache.gobblin.config.common.impl.SingleLinkedListConfigKeyPath; +import org.apache.gobblin.config.store.api.ConfigKeyPath; +import org.apache.gobblin.config.store.api.ConfigStore; +import org.apache.gobblin.config.store.api.VersionDoesNotExistException; +import org.apache.gobblin.config.store.hdfs.SimpleHadoopFilesystemConfigStore; + +import lombok.extern.slf4j.Slf4j; + + +/** + * {@link ConfigStore} that uses a zipped file containing all the config store paths. + * + * Similar to {@link SimpleHadoopFilesystemConfigStore} but using java APIs instead of Hadoop APIs for the Filesystem to + * allow reading the file without unzipping. + * + * It is assumed that the version passed in the constructor will be the only version used. + */ +@Slf4j +public class ZipFileConfigStore implements ConfigStore { + + private final FileSystem fs; + private final URI logicalStoreRoot; + private String version; + private String storePrefix; + + /** + * Construct a ZipFileConfigStore + * + * @param fs A {@link ZipFileSystem} created using the zip file or jar containing the config store + * @param logicalStoreRoot URI of this config store's root + * @param version Config store version to use (only version allowed for lookups is the version passed here) + * @param storePrefix Prefix to use if all paths in config store are under a parent directory + */ + public ZipFileConfigStore(ZipFileSystem fs, URI logicalStoreRoot, String version, String storePrefix) { + Preconditions.checkNotNull(fs); + Preconditions.checkNotNull(logicalStoreRoot); + Preconditions.checkNotNull(version); + + this.fs = fs; + this.logicalStoreRoot = logicalStoreRoot; + this.version = version; + this.storePrefix = storePrefix; + } + + @Override + public String getCurrentVersion() { + return this.version; + } + + @Override + public URI getStoreURI() { + return this.logicalStoreRoot; + } + + /** + * Retrieves all the children of the given {@link ConfigKeyPath} using {@link Files#walk} to list files + */ + @Override + public Collection<ConfigKeyPath> getChildren(ConfigKeyPath configKey, String version) + throws VersionDoesNotExistException { + Preconditions.checkNotNull(configKey, "configKey cannot be null!"); + Preconditions.checkArgument(version.equals(getCurrentVersion())); + + List<ConfigKeyPath> children = new ArrayList<>(); + Path datasetDir = getDatasetDirForKey(configKey); + + try { + + if (!Files.exists(this.fs.getPath(datasetDir.toString()))) { + return children; + } + + Stream<Path> files = Files.walk(datasetDir, 1); + + for (Iterator<Path> it = files.iterator(); it.hasNext();) { + Path path = it.next(); + + if (Files.isDirectory(path) && !path.equals(datasetDir)) { + children.add(configKey.createChild(StringUtils.removeEnd(path.getName(path.getNameCount() - 1).toString(), + SingleLinkedListConfigKeyPath.PATH_DELIMETER))); + } + + } + return children; + } catch (IOException e) { + throw new RuntimeException(String.format("Error while getting children for configKey: \"%s\"", configKey), e); + } + } + + /** + * Retrieves all the {@link ConfigKeyPath}s that are imported by the given {@link ConfigKeyPath}. Similar to + * {@link SimpleHadoopFilesystemConfigStore#getOwnImports} + */ + @Override + public List<ConfigKeyPath> getOwnImports(ConfigKeyPath configKey, String version) { + return getOwnImports(configKey, version, Optional.<Config>absent()); + } + + public List<ConfigKeyPath> getOwnImports(ConfigKeyPath configKey, String version, Optional<Config> runtimeConfig) + throws VersionDoesNotExistException { + Preconditions.checkNotNull(configKey, "configKey cannot be null!"); + Preconditions.checkArgument(version.equals(getCurrentVersion())); + + List<ConfigKeyPath> configKeyPaths = new ArrayList<>(); + Path datasetDir = getDatasetDirForKey(configKey); + Path includesFile = this.fs.getPath(datasetDir.toString(), SimpleHadoopFilesystemConfigStore.INCLUDES_CONF_FILE_NAME); + + try { + if (!Files.exists(includesFile)) { + return configKeyPaths; + } + + if (!Files.isDirectory(includesFile)) { + try (InputStream includesConfInStream = Files.newInputStream(includesFile)) { + configKeyPaths = SimpleHadoopFilesystemConfigStore.getResolvedConfigKeyPaths(includesConfInStream, runtimeConfig); + } + } + } catch (IOException e) { + throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey), e); + } + + return configKeyPaths; + } + + /** + * Retrieves the {@link Config} for the given {@link ConfigKeyPath}. Similar to + * {@link SimpleHadoopFilesystemConfigStore#getOwnConfig} + */ + @Override + public Config getOwnConfig(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException { + Preconditions.checkNotNull(configKey, "configKey cannot be null!"); + Preconditions.checkArgument(version.equals(getCurrentVersion())); + + Path datasetDir = getDatasetDirForKey(configKey); + Path mainConfFile = this.fs.getPath(datasetDir.toString(), SimpleHadoopFilesystemConfigStore.MAIN_CONF_FILE_NAME); + + try { + if (!Files.exists(mainConfFile)) { + return ConfigFactory.empty(); + } + + if (!Files.isDirectory(mainConfFile)) { + try (InputStream mainConfInputStream = Files.newInputStream(mainConfFile)) { + return ConfigFactory.parseReader(new InputStreamReader(mainConfInputStream, Charsets.UTF_8)); + } + } + return ConfigFactory.empty(); + } catch (IOException e) { + throw new RuntimeException(String.format("Error while getting config for configKey: \"%s\"", configKey), e); + } + } + + /** + * Get path object using zipped file system and relative path + */ + private Path getDatasetDirForKey(ConfigKeyPath configKey) throws VersionDoesNotExistException { + return this.fs.getPath(this.storePrefix, configKey.getAbsolutePathString()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory b/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory index cf3359a..69865c1 100644 --- a/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory +++ b/gobblin-config-management/gobblin-config-core/src/main/resources/META-INF/services/org.apache.gobblin.config.store.api.ConfigStoreFactory @@ -18,3 +18,4 @@ org.apache.gobblin.config.store.hdfs.SimpleHDFSConfigStoreFactory org.apache.gobblin.config.store.hdfs.SimpleLocalHDFSConfigStoreFactory org.apache.gobblin.config.store.hdfs.DefaultCapableLocalConfigStoreFactory +org.apache.gobblin.config.store.zip.IvyConfigStoreFactory http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java b/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java new file mode 100644 index 0000000..70ff132 --- /dev/null +++ b/gobblin-config-management/gobblin-config-core/src/test/java/org/apache/gobblin/config/store/zip/ZipFileConfigStoreTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.config.store.zip; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; + +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import com.sun.nio.zipfs.ZipFileSystem; +import com.typesafe.config.Config; + +import org.apache.gobblin.config.common.impl.SingleLinkedListConfigKeyPath; +import org.apache.gobblin.config.store.api.ConfigKeyPath; +import org.apache.gobblin.config.store.api.ConfigStoreCreationException; + + +/** + * Unit tests for {@link ZipFileConfigStore} + */ +@Test +public class ZipFileConfigStoreTest { + + private ZipFileConfigStore store; + private String version = "testVersion"; + private ConfigKeyPath rootPath = SingleLinkedListConfigKeyPath.ROOT; + private ConfigKeyPath testPath = rootPath.createChild("test"); + private ConfigKeyPath child1Path = testPath.createChild("child1"); + private ConfigKeyPath child2Path = testPath.createChild("child2"); + + @BeforeClass + public void setUp() throws URISyntaxException, ConfigStoreCreationException, IOException { + Path path = Paths.get(this.getClass().getClassLoader().getResource("zipStoreTest.zip").getPath()); + FileSystem fs = FileSystems.newFileSystem(path, null); + + this.store = new ZipFileConfigStore((ZipFileSystem) fs, path.toUri(), this.version, "_CONFIG_STORE"); + } + + @Test + public void testGetOwnConfig() { + Config config1 = this.store.getOwnConfig(this.rootPath, this.version); + Assert.assertEquals(config1.getString("gobblin.property.test1"), "prop1"); + Assert.assertEquals(config1.getString("gobblin.property.test2"), "prop2"); + + Config config2 = this.store.getOwnConfig(this.testPath, this.version); + Assert.assertEquals(config2.getString("gobblin.test.property"), "string1"); + + Config config3 = this.store.getOwnConfig(this.child1Path, this.version); + Assert.assertEquals(config3.getString("gobblin.test.property"), "string2"); + + Config config4 = this.store.getOwnConfig(this.child2Path, this.version); + Assert.assertEquals(config4.getString("gobblin.test.property"), "string3"); + + } + + @Test + public void testGetOwnImports() { + Collection<ConfigKeyPath> imports1 = this.store.getOwnImports(this.child1Path, this.version); + Assert.assertEquals(imports1.size(), 1); + Assert.assertTrue(imports1.contains(this.child1Path)); + + Collection<ConfigKeyPath> imports2 = this.store.getOwnImports(this.child2Path, this.version); + Assert.assertEquals(imports2.size(), 0); + } + + @Test + public void testGetChildren() { + Collection<ConfigKeyPath> children1 = this.store.getChildren(this.rootPath, this.version); + Assert.assertEquals(children1.size(), 1); + Assert.assertTrue(children1.contains(this.testPath)); + + Collection<ConfigKeyPath> children2 = this.store.getChildren(this.testPath, this.version); + Assert.assertEquals(children2.size(), 2); + Assert.assertTrue(children2.contains(this.child1Path)); + Assert.assertTrue(children2.contains(this.child2Path)); + + Collection<ConfigKeyPath> children3 = this.store.getChildren(this.child1Path, this.version); + Assert.assertEquals(children3.size(), 0); + } +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip ---------------------------------------------------------------------- diff --git a/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip b/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip new file mode 100644 index 0000000..ffe9603 Binary files /dev/null and b/gobblin-config-management/gobblin-config-core/src/test/resources/zipStoreTest.zip differ http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-utility/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-utility/build.gradle b/gobblin-utility/build.gradle index a8f87bb..ad49820 100644 --- a/gobblin-utility/build.gradle +++ b/gobblin-utility/build.gradle @@ -46,6 +46,7 @@ dependencies { compile externalDependency.gson compile externalDependency.opencsv compile externalDependency.hadoopHdfs + compile externalDependency.groovy runtime externalDependency.hadoopCommon runtime externalDependency.hadoopClientCore http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java new file mode 100644 index 0000000..ab5468f --- /dev/null +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/DownloadUtils.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.util; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URL; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Map; + +import com.google.common.collect.Maps; +import com.google.common.io.Resources; + +import groovy.grape.Grape; +import groovy.lang.GroovyClassLoader; + + +/** + * Utility class for downloads using grape + */ +public class DownloadUtils { + public static final String IVY_SETTINGS_FILE_NAME = "ivysettings.xml"; + + /** + * Download jar through {@link Grape} given an org, module and version + * It is assumed that an ivy settings file exists on the classpath + */ + public static URI[] downloadJar(String org, String module, String version, boolean transitive) throws IOException { + Map<String, Object> artifactMap = Maps.newHashMap(); + artifactMap.put("org", org); + artifactMap.put("module", module); + artifactMap.put("version", version); + artifactMap.put("transitive", transitive); + return downloadJar(artifactMap); + } + + public static URI[] downloadJar(Map<String, Object> artifactMap) throws IOException { + System.setProperty("grape.config", getIvySettingsFile().getAbsolutePath()); + + Map<String, Object> args = Maps.newHashMap(); + args.put("classLoader", AccessController.doPrivileged(new PrivilegedAction<GroovyClassLoader>() { + @Override + public GroovyClassLoader run() { + return new GroovyClassLoader(); + } + })); + return Grape.resolve(args, artifactMap); + } + + /** + * Get ivy settings file from classpath + */ + public static File getIvySettingsFile() throws IOException { + URL settingsUrl = Thread.currentThread().getContextClassLoader().getResource(IVY_SETTINGS_FILE_NAME); + if (settingsUrl == null) { + throw new IOException("Failed to find " + IVY_SETTINGS_FILE_NAME + " from class path"); + } + + // Check if settingsUrl is file on classpath + File ivySettingsFile = new File(settingsUrl.getFile()); + if (ivySettingsFile.exists()) { + // can access settingsUrl as a file + return ivySettingsFile; + } + + // Create temporary Ivy settings file. + ivySettingsFile = File.createTempFile("ivy.settings", ".xml"); + ivySettingsFile.deleteOnExit(); + + try (OutputStream os = new BufferedOutputStream(new FileOutputStream(ivySettingsFile))) { + Resources.copy(settingsUrl, os); + } + + return ivySettingsFile; + } +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/5eee52d9/gradle/scripts/dependencyDefinitions.gradle ---------------------------------------------------------------------- diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle index c1f53db..f82b7e2 100644 --- a/gradle/scripts/dependencyDefinitions.gradle +++ b/gradle/scripts/dependencyDefinitions.gradle @@ -48,6 +48,7 @@ ext.externalDependency = [ "datanucleusRdbms": "org.datanucleus:datanucleus-rdbms:3.2.9", "eventhub": "com.microsoft.azure:azure-eventhubs:0.9.0", "guava": "com.google.guava:guava:15.0", + "groovy": "org.codehaus.groovy:groovy:2.4.8", "gson": "com.google.code.gson:gson:2.6.2", "findBugsAnnotations": "com.google.code.findbugs:jsr305:" + findBugsVersion, "hadoopCommon": "org.apache.hadoop:hadoop-common:" + hadoopVersion,
