Updated Branches: refs/heads/trunk b22d56cb9 -> 2ff690f11
GIRAPH-746: Track and log versions of dependencies. Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/2ff690f1 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/2ff690f1 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/2ff690f1 Branch: refs/heads/trunk Commit: 2ff690f110627782c4142eed744d1b7239d4bec4 Parents: b22d56c Author: Nitay Joffe <[email protected]> Authored: Thu Aug 22 16:48:14 2013 -0400 Committer: Nitay Joffe <[email protected]> Committed: Tue Aug 27 11:53:10 2013 -0400 ---------------------------------------------------------------------- giraph-core/pom.xml | 6 ++ .../giraph/benchmark/GiraphBenchmark.java | 10 +- .../apache/giraph/utils/GiraphDepVersions.java | 100 +++++++++++++++++++ .../org/apache/giraph/utils/LogVersions.java | 52 ++++++++++ .../org/apache/giraph/versions.properties | 46 +++++++++ 5 files changed, 212 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/2ff690f1/giraph-core/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-core/pom.xml b/giraph-core/pom.xml index b2dcbd1..f5ed867 100644 --- a/giraph-core/pom.xml +++ b/giraph-core/pom.xml @@ -40,6 +40,12 @@ under the License. </properties> <build> + <resources> + <resource> + <directory>src/main/resources</directory> + <filtering>true</filtering> + </resource> + </resources> <finalName>giraph-${project.version}-${forHadoop}</finalName> <plugins> <plugin> http://git-wip-us.apache.org/repos/asf/giraph/blob/2ff690f1/giraph-core/src/main/java/org/apache/giraph/benchmark/GiraphBenchmark.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/benchmark/GiraphBenchmark.java b/giraph-core/src/main/java/org/apache/giraph/benchmark/GiraphBenchmark.java index 4631065..46642da 100644 --- a/giraph-core/src/main/java/org/apache/giraph/benchmark/GiraphBenchmark.java +++ b/giraph-core/src/main/java/org/apache/giraph/benchmark/GiraphBenchmark.java @@ -25,6 +25,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; import org.apache.giraph.conf.GiraphConfiguration; import org.apache.giraph.job.GiraphJob; +import org.apache.giraph.utils.LogVersions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Tool; import org.apache.log4j.Logger; @@ -80,8 +81,13 @@ public abstract class GiraphBenchmark implements Tool { GiraphJob job = new GiraphJob(getConf(), getClass().getName()); int workers = Integer.parseInt(BenchmarkOption.WORKERS.getOptionValue(cmd)); - job.getConfiguration().setWorkerConfiguration(workers, workers, 100.0f); - prepareConfiguration(job.getConfiguration(), cmd); + + GiraphConfiguration giraphConf = job.getConfiguration(); + giraphConf.addWorkerObserverClass(LogVersions.class); + giraphConf.addMasterObserverClass(LogVersions.class); + + giraphConf.setWorkerConfiguration(workers, workers, 100.0f); + prepareConfiguration(giraphConf, cmd); boolean isVerbose = false; if (BenchmarkOption.VERBOSE.optionTurnedOn(cmd)) { http://git-wip-us.apache.org/repos/asf/giraph/blob/2ff690f1/giraph-core/src/main/java/org/apache/giraph/utils/GiraphDepVersions.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/GiraphDepVersions.java b/giraph-core/src/main/java/org/apache/giraph/utils/GiraphDepVersions.java new file mode 100644 index 0000000..ebc250b --- /dev/null +++ b/giraph-core/src/main/java/org/apache/giraph/utils/GiraphDepVersions.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.utils; + +import org.apache.log4j.Logger; + +import com.google.common.collect.Maps; +import com.google.common.io.Resources; + +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import java.util.Properties; + +/** + * Versions of Giraph dependencies. This pulls version information from a known + * properties file. That file is created by maven when building the jar and + * populated with versions from the pom. We put the properties file in a known + * resources folder so that it is easy to pick up. + * + * See http://bit.ly/19LQyrK for more information. + */ +public class GiraphDepVersions { + /** Logger */ + private static final Logger LOG = Logger.getLogger(GiraphDepVersions.class); + + /** Path to resource */ + private static final String RESOURCE_NAME = + "org/apache/giraph/versions.properties"; + + /** Singleton */ + private static final GiraphDepVersions INSTANCE = new GiraphDepVersions(); + + /** The properties read */ + private final Properties properties; + + /** Constructor */ + private GiraphDepVersions() { + URL url = Resources.getResource(RESOURCE_NAME); + properties = new Properties(); + try { + properties.load(url.openStream()); + } catch (IOException e) { + LOG.error("Could not read giraph versions from file " + RESOURCE_NAME); + } + } + + /** + * Get singleton instance + * + * @return singleton + */ + public static GiraphDepVersions get() { + return INSTANCE; + } + + public Properties getProperties() { + return properties; + } + + /** + * Get version of the named dependency, or null if not found + * + * @param name dependency name + * @return version, or null + */ + public String versionOf(String name) { + return properties.getProperty(name); + } + + /** Log the dependency versions we're using */ + public void logVersionsUsed() { + Map<String, String> sortedVersions = Maps.newTreeMap(); + for (Map.Entry<Object, Object> entry : properties.entrySet()) { + sortedVersions.put(entry.getKey().toString(), + entry.getValue().toString()); + } + StringBuilder sb = new StringBuilder(sortedVersions.size() * 20); + for (Map.Entry<String, String> entry : sortedVersions.entrySet()) { + sb.append(" ").append(entry.getKey()).append(": "). + append(entry.getValue()).append("\n"); + } + LOG.info("Versions of Giraph dependencies =>\n" + sb); + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/2ff690f1/giraph-core/src/main/java/org/apache/giraph/utils/LogVersions.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/LogVersions.java b/giraph-core/src/main/java/org/apache/giraph/utils/LogVersions.java new file mode 100644 index 0000000..8305df7 --- /dev/null +++ b/giraph-core/src/main/java/org/apache/giraph/utils/LogVersions.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.utils; + +import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; +import org.apache.giraph.master.MasterObserver; +import org.apache.giraph.worker.WorkerObserver; + +/** + * Logs versions of Giraph dependencies on job start. + */ +public class LogVersions implements WorkerObserver, MasterObserver { + @Override + public void applicationFailed(Exception e) { } + + @Override + public void setConf(ImmutableClassesGiraphConfiguration configuration) { } + + @Override + public ImmutableClassesGiraphConfiguration getConf() { + return null; + } + + @Override + public void preApplication() { + GiraphDepVersions.get().logVersionsUsed(); + } + + @Override + public void postApplication() { } + + @Override + public void preSuperstep(long superstep) { } + + @Override + public void postSuperstep(long superstep) { } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/2ff690f1/giraph-core/src/main/resources/org/apache/giraph/versions.properties ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/resources/org/apache/giraph/versions.properties b/giraph-core/src/main/resources/org/apache/giraph/versions.properties new file mode 100644 index 0000000..c56e510 --- /dev/null +++ b/giraph-core/src/main/resources/org/apache/giraph/versions.properties @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +accumulo.version=${dep.accumulo.version} +airline.version=${dep.airline.version} +base64.version=${dep.base64.version} +cli-parser.version=${dep.cli-parser.version} +codehaus-jackson.version=${dep.codehaus-jackson.version} +commons-cli.version=${dep.commons-cli.version} +commons-collections.version=${dep.commons-collections.version} +commons-configuration.version=${dep.commons-configuration.version} +commons-httpclient.version=${dep.commons-httpclient.version} +commons-io.version=${dep.commons-io.version} +commons-logging.version=${dep.commons-logging.version} +commons-net.version=${dep.commons-net.version} +facebook-hadoop.version=${dep.facebook-hadoop.version} +fasterxml-jackson.version=${dep.fasterxml-jackson.version} +fastutil.version=${dep.fastutil.version} +hbase.version=${dep.hbase.version} +guava.version=${dep.guava.version} +hcatalog.version=${dep.hcatalog.version} +hive.version=${dep.hive.version} +hiveio.version=${dep.hiveio.version} +json.version=${dep.json.version} +junit.version=${dep.junit.version} +jython.version=${dep.jython.version} +mockito.version=${dep.mockito.version} +netty.version=${dep.netty.version} +slf4j.version=${dep.slf4j.version} +typetools.version=${dep.typetools.version} +yammer-metrics.version=${dep.yammer-metrics.version} +yourkit-api.version=${dep.yourkit-api.version} +zookeeper.version=${dep.zookeeper.version}
