TS-2259: Introduce failover hosts for logging system This patch introduce failover hosts for collation host by using '|' delimiter to extend <CollationHosts> tags in logs_xml.config.
Let me give an example: <CollationHosts = "host1:5000|host2:5000|host3:6000, 209.131.52.129:6000"/> In the example above, host2/host3 are failover hosts for host1. When host1 disconnected, log entries will be sent to host2, and then if host2 failed again, log entries will be sent to host3 until host1 or host2 comes back. Signed-off-by: Yunkai Zhang <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/54883f2f Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/54883f2f Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/54883f2f Branch: refs/heads/5.0.x Commit: 54883f2ff62a5b09d238b54ffb4279ea9f45e796 Parents: 134b8a6 Author: Yunkai Zhang <[email protected]> Authored: Mon Sep 30 11:46:38 2013 +0800 Committer: Yunkai Zhang <[email protected]> Committed: Tue Oct 1 12:52:17 2013 +0800 ---------------------------------------------------------------------- proxy/config/logs_xml.config.default | 8 ++++++++ proxy/logging/LogBufferSink.h | 4 +++- proxy/logging/LogConfig.cc | 23 +++++++++++++++++------ proxy/logging/LogFile.cc | 10 ++++++---- proxy/logging/LogFile.h | 2 +- proxy/logging/LogHost.cc | 31 ++++++++++++++++++++++--------- proxy/logging/LogHost.h | 5 +++-- 7 files changed, 60 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/config/logs_xml.config.default ---------------------------------------------------------------------- diff --git a/proxy/config/logs_xml.config.default b/proxy/config/logs_xml.config.default index 62f614f..7633808 100644 --- a/proxy/config/logs_xml.config.default +++ b/proxy/config/logs_xml.config.default @@ -186,6 +186,14 @@ Example2: do not log requests for domain unwanted.com either a name or an ip. For example: <CollationHosts = "host1.company.com:5000, 209.131.52.129:6000"/> + Also, we can introduce failover hosts for each collation host by + using '|' delimiter, For example: + <CollationHosts = "host1:5000|host2:5000|host3:6000, 209.131.52.129:6000"/> + + In the example above, host2/host3 are failover hosts for host1. When host1 + disconnected, log entries will be sent to host2, and then if host2 failed + again, log entries will be sent to host3 until host1 or host2 comes back. + <Header = "header"/> This tag specifies a string to be written at the beginning of the log file, just before the first record. http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/logging/LogBufferSink.h ---------------------------------------------------------------------- diff --git a/proxy/logging/LogBufferSink.h b/proxy/logging/LogBufferSink.h index b7d4ca6..53789df 100644 --- a/proxy/logging/LogBufferSink.h +++ b/proxy/logging/LogBufferSink.h @@ -42,7 +42,9 @@ public: // Of course, this function may not free memory directly, it // can delegate another function to do it. // - virtual void preproc_and_try_delete(LogBuffer * buffer) = 0; + // return 0 if success, -1 on error. + // + virtual int preproc_and_try_delete(LogBuffer * buffer) = 0; virtual ~ LogBufferSink() { }; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/logging/LogConfig.cc ---------------------------------------------------------------------- diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index d9810ce..40e0069 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -2308,13 +2308,24 @@ LogConfig::read_xml_log_config(int from_memory) char *host; SimpleTokenizer tok(collationHosts_str, ','); while (host = tok.getNext(), host != 0) { - LogHost *lh = NEW(new LogHost(obj->get_full_filename(), obj->get_signature())); - if (lh->set_name_or_ipstr(host)) { - Warning("Could not set \"%s\" as collation host", host); - delete lh; - } else { - obj->add_loghost(lh, false); + LogHost *prev = NULL; + char *failover_str; + SimpleTokenizer failover_tok(host, '|'); // split failover hosts + + while (failover_str = failover_tok.getNext(), failover_str != 0) { + LogHost *lh = NEW(new LogHost(obj->get_full_filename(), obj->get_signature())); + + if (lh->set_name_or_ipstr(failover_str)) { + Warning("Could not set \"%s\" as collation host", host); + delete lh; + } else if (!prev){ + obj->add_loghost(lh, false); + prev = lh; + } else { + prev->failover_link.next = lh; + prev = lh; + } } } } http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/logging/LogFile.cc ---------------------------------------------------------------------- diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc index 71ac5f6..55fe900 100644 --- a/proxy/logging/LogFile.cc +++ b/proxy/logging/LogFile.cc @@ -470,14 +470,15 @@ LogFile::roll(long interval_start, long interval_end) preprocess the given buffer data before write to target file and try to delete it when its reference become zero. -------------------------------------------------------------------------*/ -void +int LogFile::preproc_and_try_delete(LogBuffer * lb) { + int ret = -1; LogBufferHeader *buffer_header; if (lb == NULL) { Note("Cannot write LogBuffer to LogFile %s; LogBuffer is NULL", m_name); - return; + return -1; } ink_atomic_increment(&lb->m_references, 1); @@ -528,10 +529,11 @@ LogFile::preproc_and_try_delete(LogBuffer * lb) // // LogBuffer will be deleted in flush thread // - return; + return 0; } else if (m_file_format == ASCII_LOG || m_file_format == ASCII_PIPE) { write_ascii_logbuffer3(buffer_header); + ret = 0; } else { Note("Cannot write LogBuffer to LogFile %s; invalid file format: %d", @@ -540,7 +542,7 @@ LogFile::preproc_and_try_delete(LogBuffer * lb) done: LogBuffer::destroy(lb); - return; + return ret; } /*------------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/logging/LogFile.h ---------------------------------------------------------------------- diff --git a/proxy/logging/LogFile.h b/proxy/logging/LogFile.h index 3a3dc45..144a2b5 100644 --- a/proxy/logging/LogFile.h +++ b/proxy/logging/LogFile.h @@ -141,7 +141,7 @@ public: LOG_FILE_FILESYSTEM_CHECKS_FAILED }; - void preproc_and_try_delete(LogBuffer * lb); + int preproc_and_try_delete(LogBuffer * lb); int roll(long interval_start, long interval_end); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/logging/LogHost.cc ---------------------------------------------------------------------- diff --git a/proxy/logging/LogHost.cc b/proxy/logging/LogHost.cc index 84e08f4..05e9f28 100644 --- a/proxy/logging/LogHost.cc +++ b/proxy/logging/LogHost.cc @@ -263,14 +263,15 @@ LogHost::create_orphan_LogFile_object() // preprocess the given buffer data before sent to target host // and try to delete it when its reference become zero. // -void +int LogHost::preproc_and_try_delete (LogBuffer *lb) { + int ret = -1; int bytes; if (lb == NULL) { Note("Cannot write LogBuffer to LogHost %s; LogBuffer is NULL", name()); - return; + return -1; } LogBufferHeader *buffer_header = lb->header(); if (buffer_header == NULL) { @@ -291,7 +292,7 @@ LogHost::preproc_and_try_delete (LogBuffer *lb) if (!connect ()) { Note("Cannot write LogBuffer to LogHost %s; not connected", name()); orphan_write_and_try_delete(lb); - return; + return -1; } } @@ -308,8 +309,9 @@ LogHost::preproc_and_try_delete (LogBuffer *lb) // TODO: We currently don't try to make the log buffers handle little vs big endian. TS-1156. // lb->convert_to_host_order (); orphan_write_and_try_delete(lb); - return; + return -1; } + ret = 0; #else // !defined(IOCORE_LOG_COLLATION) // create a new collation client if necessary @@ -326,14 +328,15 @@ LogHost::preproc_and_try_delete (LogBuffer *lb) Debug("log-buftrak", "[%d]LogHost::preproc_and_try_delete - orphan write complete", lb->header()->id); #endif // defined(LOG_BUFFER_TRACKING) + return -1; } - return; + return 0; #endif // !defined(IOCORE_LOG_COLLATION) done: LogBuffer::destroy(lb); - return; + return ret; } // @@ -355,7 +358,7 @@ LogHost::orphan_write_and_try_delete(LogBuffer * lb) Debug("log-host", "Sending LogBuffer to orphan file %s", m_orphan_file->get_name()); m_orphan_file->preproc_and_try_delete(lb); } else { - Note("logging space exhausted, failed to write orphan file, drop(%" PRIu32 ") bytes", + Debug("log-host", "logging space exhausted, failed to write orphan file, drop(%" PRIu32 ") bytes", lb->header()->byte_count); LogBuffer::destroy(lb); } @@ -454,9 +457,10 @@ LogHostList::clear() } } -void +int LogHostList::preproc_and_try_delete(LogBuffer * lb) { + int ret; unsigned nr_host, nr; ink_release_assert(lb->m_references == 0); @@ -465,12 +469,21 @@ LogHostList::preproc_and_try_delete(LogBuffer * lb) ink_atomic_increment(&lb->m_references, nr_host); for (LogHost * host = first(); host && nr; host = next(host)) { - host->preproc_and_try_delete(lb); + LogHost *lh = host; + + do { + ink_atomic_increment(&lb->m_references, 1); + ret = lh->preproc_and_try_delete(lb); + } while (ret < 0 && (lh = lh->failover_link.next)); + + LogBuffer::destroy(lb); nr--; } if (nr_host == 0) delete lb; + + return 0; } void http://git-wip-us.apache.org/repos/asf/trafficserver/blob/54883f2f/proxy/logging/LogHost.h ---------------------------------------------------------------------- diff --git a/proxy/logging/LogHost.h b/proxy/logging/LogHost.h index e8bec1e..55588d9 100644 --- a/proxy/logging/LogHost.h +++ b/proxy/logging/LogHost.h @@ -56,7 +56,7 @@ public: // preprocess the given buffer data before sent to target host // and try to delete it when its reference become zero. // - void preproc_and_try_delete(LogBuffer * lb); + int preproc_and_try_delete(LogBuffer * lb); char const* name() const { return m_name ? m_name : "UNKNOWN"; } IpAddr const& ip_addr() const { return m_ip; } @@ -96,6 +96,7 @@ private: public: LINK(LogHost, link); + SLINK(LogHost, failover_link); private: // -- member functions not allowed -- @@ -115,7 +116,7 @@ public: void add(LogHost * host, bool copy = true); unsigned count(); void clear(); - void preproc_and_try_delete(LogBuffer * lb); + int preproc_and_try_delete(LogBuffer * lb); LogHost *first() { return m_host_list.head; } LogHost *next(LogHost * here) { return (here->link).next; }
