Hi
To learn a bit about the internal structure of osmosis I developed a new
task in the "replication" section. It's basicly the same as the little
replag script [1] I use on my servers.
I also added a munin plugin that plots the replication lag into a graph.
On some systems the plugin needs some extra time (upper munin-node
timeout) because of javas slow startup.
I don't know if this task should be comitted to svn or if it's too
simple (its task can also be fulfilled by that simple shell-script). I
also don't know where to place the muinin plugin in the source tree and
how to make it installed into the right place when using debuild to make
a debian package.
Please find the patch attached.
Peter
[1] <http://svn.toolserver.org/svnroot/mazder/diff-import/replag>
Index:
replication/src/org/openstreetmap/osmosis/replication/ReplicationPluginLoader.java
===================================================================
---
replication/src/org/openstreetmap/osmosis/replication/ReplicationPluginLoader.java
(Revision 22433)
+++
replication/src/org/openstreetmap/osmosis/replication/ReplicationPluginLoader.java
(Arbeitskopie)
@@ -12,8 +12,8 @@
import
org.openstreetmap.osmosis.replication.v0_6.ReplicationDownloaderInitializerFactory;
import org.openstreetmap.osmosis.replication.v0_6.ReplicationFileMergerFactory;
import
org.openstreetmap.osmosis.replication.v0_6.ReplicationFileMergerInitializerFactory;
+import org.openstreetmap.osmosis.replication.v0_6.ReplicationLagReaderFactory;
-
/**
* The plugin loader for the replication tasks.
*
@@ -43,6 +43,9 @@
factoryMap.put("merge-replication-files-init", new
ReplicationFileMergerInitializerFactory());
factoryMap.put("mrfi", new
ReplicationFileMergerInitializerFactory());
+ factoryMap.put("read-replication-lag", new
ReplicationLagReaderFactory());
+ factoryMap.put("rrl", new ReplicationLagReaderFactory());
+
factoryMap.put("read-change-interval-0.6", new
IntervalDownloaderFactory());
factoryMap.put("read-change-interval-init-0.6", new
IntervalDownloaderInitializerFactory());
factoryMap.put("read-replication-interval-0.6", new
ReplicationDownloaderFactory());
Index: replication/src/org/openstreetmap/osmosis/replication/munin/sample-config
===================================================================
--- replication/src/org/openstreetmap/osmosis/replication/munin/sample-config
(Revision 0)
+++ replication/src/org/openstreetmap/osmosis/replication/munin/sample-config
(Revision 0)
@@ -0,0 +1,3 @@
+[osm*]
+env.osmosis /opt/osmosis/bin/osmosis
+env.workingDirectory /home/osm/data/diffs/
Index:
replication/src/org/openstreetmap/osmosis/replication/munin/osm-replication-lag
===================================================================
---
replication/src/org/openstreetmap/osmosis/replication/munin/osm-replication-lag
(Revision 0)
+++
replication/src/org/openstreetmap/osmosis/replication/munin/osm-replication-lag
(Revision 0)
@@ -0,0 +1,36 @@
+#!/bin/sh
+# -*- sh -*-
+
+. $MUNIN_LIBDIR/plugins/plugin.sh
+
+if [ ! $workingDirectory ]; then
+ echo "no workingDirectory configured" >&2
+ exit 1
+fi
+
+[ $osmosis ] || osmosis="/opt/osmosis/bin/osmosis"
+
+
+
+if [ "$1" = "autoconf" ]; then
+ echo yes
+ exit 0
+fi
+
+if [ "$1" = "config" ]; then
+
+ echo 'graph_title OSM PostGIS Database Replag'
+ echo 'graph_args --base 1000'
+ echo 'graph_vlabel seconds behind main database'
+ echo 'graph_category osm'
+
+ echo 'lag.label replication lag'
+ echo 'lag.draw LINE'
+
+ exit 0
+fi
+
+lag=$($osmosis --read-replication-lag workingDirectory="$workingDirectory"
2>/dev/null)
+echo "lag=$lag"
+
+exit 0
Eigenschaftsänderungen:
replication/src/org/openstreetmap/osmosis/replication/munin/osm-replication-lag
___________________________________________________________________
Hinzugefügt: svn:executable
+ *
Index:
replication/src/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReaderFactory.java
===================================================================
---
replication/src/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReaderFactory.java
(Revision 0)
+++
replication/src/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReaderFactory.java
(Revision 0)
@@ -0,0 +1,55 @@
+// This software is released into the Public Domain. See copying.txt for
details.
+package org.openstreetmap.osmosis.replication.v0_6;
+
+import java.io.File;
+
+import org.openstreetmap.osmosis.core.pipeline.common.RunnableTaskManager;
+import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
+import org.openstreetmap.osmosis.core.pipeline.common.TaskManager;
+import org.openstreetmap.osmosis.core.pipeline.common.TaskManagerFactory;
+
+
+/**
+ * The task manager factory for a replication file download initializer.
+ */
+public class ReplicationLagReaderFactory extends TaskManagerFactory {
+ private static final String ARG_HUMAN_READABLE = "humanReadable";
+ private static final boolean DEFAULT_HUMAN_READABLE = false;
+ private static final String ARG_WORKING_DIRECTORY = "workingDirectory";
+ private static final String DEFAULT_WORKING_DIRECTORY = "./";
+
+
+ /**
+ * {...@inheritdoc}
+ */
+ @Override
+ protected TaskManager createTaskManagerImpl(TaskConfiguration
taskConfig) {
+ String workingDirectoryString;
+ boolean humanReadableFlag;
+ File workingDirectory;
+
+ // Get the task arguments.
+ workingDirectoryString = getStringArgument(
+ taskConfig,
+ ARG_WORKING_DIRECTORY,
+ getDefaultStringArgument(taskConfig,
DEFAULT_WORKING_DIRECTORY)
+ );
+ humanReadableFlag = getBooleanArgument(
+ taskConfig,
+ ARG_HUMAN_READABLE,
+ DEFAULT_HUMAN_READABLE
+ );
+
+ // Convert argument strings to strongly typed objects.
+ workingDirectory = new File(workingDirectoryString);
+
+ return new RunnableTaskManager(
+ taskConfig.getId(),
+ new ReplicationLagReader(
+ workingDirectory,
+ humanReadableFlag
+ ),
+ taskConfig.getPipeArgs()
+ );
+ }
+}
Index:
replication/src/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java
===================================================================
---
replication/src/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java
(Revision 0)
+++
replication/src/org/openstreetmap/osmosis/replication/v0_6/ReplicationLagReader.java
(Revision 0)
@@ -0,0 +1,162 @@
+// This software is released into the Public Domain. See copying.txt for
details.
+package org.openstreetmap.osmosis.replication.v0_6;
+
+import java.io.File;
+import java.util.Date;
+import java.util.Properties;
+import java.util.logging.Logger;
+import java.text.MessageFormat;
+
+import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
+import org.openstreetmap.osmosis.core.task.common.RunnableTask;
+import org.openstreetmap.osmosis.core.util.FileBasedLock;
+import org.openstreetmap.osmosis.core.util.PropertiesPersister;
+import org.openstreetmap.osmosis.replication.common.ReplicationState;
+import org.openstreetmap.osmosis.replication.common.ServerStateReader;
+import
org.openstreetmap.osmosis.replication.v0_6.impl.ReplicationDownloaderConfiguration;
+
+/**
+ * Downloads a set of replication files from a HTTP server, and merges them
into a
+ * single output stream. It tracks the intervals covered by the current files
+ * and stores the current timestamp between invocations forming the basis of a
+ * replication mechanism.
+ *
+ * @author Brett Henderson
+ */
+public class ReplicationLagReader implements RunnableTask {
+ private static final Logger LOG =
Logger.getLogger(ReplicationLagReader.class.getName());
+ private static final String LOCK_FILE_NAME = "download.lock";
+ private static final String CONFIG_FILE = "configuration.txt";
+ private static final String LOCAL_STATE_FILE = "state.txt";
+
+ private boolean humanReadable;
+ private File workingDirectory;
+ private ServerStateReader serverStateReader;
+
+ /**
+ * Creates a new instance.
+ *
+ * @param workingDirectory
+ * The directory containing configuration and tracking files.
+ */
+ public ReplicationLagReader(File workingDirectory, boolean
humanReadable) {
+ this.workingDirectory = workingDirectory;
+ this.humanReadable = humanReadable;
+
+ serverStateReader = new ServerStateReader();
+ }
+
+
+ /**
+ * Initializes a working directory.
+ */
+ private void getLag() {
+ ReplicationDownloaderConfiguration configuration;
+ ReplicationState serverState;
+ ReplicationState localState;
+ PropertiesPersister localStatePersistor;
+ Properties localStateProperties;
+
+ // Instantiate utility objects.
+ configuration = new ReplicationDownloaderConfiguration(new
File(workingDirectory, CONFIG_FILE));
+
+ // Obtain the server state.
+ LOG.fine("Reading current server state.");
+ serverState =
serverStateReader.getServerState(configuration.getBaseUrl());
+
+ // Build the local state persister which is used for both
loading and storing local state.
+ localStatePersistor = new PropertiesPersister(new
File(workingDirectory, LOCAL_STATE_FILE));
+
+ // If local state isn't available we need to copy server state
to be the initial local state
+ // then exit.
+ if (!localStatePersistor.exists()) {
+ throw new OsmosisRuntimeException("Can't read local
state.");
+ }
+
+ // fetch the locale state from the file
+ localStateProperties = localStatePersistor.load();
+ localState = new ReplicationState(localStateProperties);
+
+ // extract the time of the local and the remote state files
+ long local = localState.getTimestamp().getTime();
+ long server = serverState.getTimestamp().getTime();
+
+ // we assume the server beeing before us while calculating the
difference
+ long lag = (server - local) / 1000;
+
+ // check if a human readable version is requested
+ if(this.humanReadable)
+ {
+ // more than a day
+ if(lag > 86400)
+ {
+ Object[] args = {
+ new Long(lag / 86400),
+ new Long((lag % 86400) / 3600)
+ };
+ System.out.println(
+ new MessageFormat("{0} day(s) and {1}
hour(s)").format(args)
+ );
+ }
+
+ // morte than an hour
+ else if(lag > 3600)
+ {
+ Object[] args = {
+ new Long(lag / 3600),
+ new Long((lag % 3600) / 60)
+ };
+ System.out.println(
+ new MessageFormat("{0} hour(s) and {1}
minute(s)").format(args)
+ );
+ }
+
+ // more than a minute
+ else if(lag > 60)
+ {
+ Object[] args = {
+ new Long(lag / 60),
+ new Long(lag % 60)
+ };
+ System.out.println(
+ new MessageFormat("{0} minute(s) and
{1} second(s)").format(args)
+ );
+ }
+
+ // just some seconds
+ else
+ {
+ System.out.println(
+ new MessageFormat("{0}
second(s)").format(lag)
+ );
+ }
+ }
+ else
+ {
+ // print out the raw number of seconds
+ System.out.println(lag);
+ }
+ }
+
+
+ /**
+ * {...@inheritdoc}
+ */
+ @Override
+ public void run() {
+ FileBasedLock fileLock;
+
+ fileLock = new FileBasedLock(new File(workingDirectory,
LOCK_FILE_NAME));
+
+ try {
+ fileLock.lock();
+
+ getLag();
+
+ fileLock.unlock();
+
+ } finally {
+ fileLock.release();
+ }
+ }
+}
_______________________________________________
osmosis-dev mailing list
[email protected]
http://lists.openstreetmap.org/listinfo/osmosis-dev
