Author: jsdelfino
Date: Mon Apr 9 06:22:16 2012
New Revision: 1311137
URL: http://svn.apache.org/viewvc?rev=1311137&view=rev
Log:
Cleanup Scribe log output and add ability to log to a firehose Unix fifo pipe.
Added:
tuscany/sca-cpp/trunk/components/log/scribed-central-firehose-conf
- copied, changed from r1311136,
tuscany/sca-cpp/trunk/components/log/scribed-central-conf
tuscany/sca-cpp/trunk/components/log/scribed-central-mkfirehose
- copied, changed from r1311136,
tuscany/sca-cpp/trunk/components/log/scribe-tail-start
tuscany/sca-cpp/trunk/patches/scribe.patch
Modified:
tuscany/sca-cpp/trunk/components/cache/memcached-start
tuscany/sca-cpp/trunk/components/log/Makefile.am
tuscany/sca-cpp/trunk/components/log/scribe-cat.cpp
tuscany/sca-cpp/trunk/components/log/scribe-tail-start
tuscany/sca-cpp/trunk/components/log/scribed-central-conf
tuscany/sca-cpp/trunk/components/sqldb/pgsql-conf
tuscany/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
tuscany/sca-cpp/trunk/hosting/server/ssl-start
tuscany/sca-cpp/trunk/hosting/server/start
tuscany/sca-cpp/trunk/ubuntu/ubuntu-install
tuscany/sca-cpp/trunk/ubuntu/ubuntu-install-all
Modified: tuscany/sca-cpp/trunk/components/cache/memcached-start
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/cache/memcached-start?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/cache/memcached-start (original)
+++ tuscany/sca-cpp/trunk/components/cache/memcached-start Mon Apr 9 06:22:16
2012
@@ -34,7 +34,7 @@ memcached_prefix=`cat $here/memcached.pr
if [ -f "$root/memcached/log.conf" ]; then
log=`cat $root/memcached/log.conf`
- v="-vv"
+ v="-v"
else
mkdir -p $root/logs
log="cat >>$root/logs/memcached"
Modified: tuscany/sca-cpp/trunk/components/log/Makefile.am
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/log/Makefile.am?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/log/Makefile.am (original)
+++ tuscany/sca-cpp/trunk/components/log/Makefile.am Mon Apr 9 06:22:16 2012
@@ -22,7 +22,7 @@ INCLUDES = -I${THRIFT_INCLUDE} -I${FB303
incl_HEADERS = *.hpp
incldir = $(prefix)/include/components/log
-dist_comp_SCRIPTS = scribed-central-conf scribed-client-conf
scribed-central-start scribed-central-stop scribed-client-start
scribed-client-stop scribe-tail-start scribe-tail-stop
+dist_comp_SCRIPTS = scribed-central-conf scribed-central-firehose-conf
scribed-central-mkfirehose scribed-client-conf scribed-central-start
scribed-central-stop scribed-client-start scribed-client-stop scribe-tail-start
scribe-tail-stop
compdir=$(prefix)/components/log
comp_DATA = scribe.prefix thrift.prefix
Modified: tuscany/sca-cpp/trunk/components/log/scribe-cat.cpp
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/log/scribe-cat.cpp?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/log/scribe-cat.cpp (original)
+++ tuscany/sca-cpp/trunk/components/log/scribe-cat.cpp Mon Apr 9 06:22:16 2012
@@ -36,33 +36,35 @@
namespace tuscany {
namespace scribecat {
-int cat(const string& category, const string& type) {
+int cat(const string& host, const string& category, const string& type) {
// Connect to Scribe
scribe::Scribe& sc = *(new (gc_new<scribe::Scribe>())
scribe::Scribe("localhost", 1464));
// Read lines from stdin and log them
- char buf[8192];
+ char buf[8193];
for (;;) {
- const char* s = fgets(buf, 8192, stdin);
+ gc_scoped_pool();
+
+ // Write line prefix
+ ostringstream os;
+ os << "[" << host << "] ";
+ if (length(type) != 0)
+ os << "[" << logTime() << "] [" << type << "] ";
+ const string prefix = str(os);
+ const int pl = length(prefix);
+ strcpy(buf, c_str(prefix));
+
+ // Read log line
+ const char* s = fgets(buf + pl, 8192 - pl, stdin);
if (s == NULL)
return 0;
const size_t l = strlen(s);
if (l < 2)
return 0;
- buf[l - 1] = '\0';
+ buf[pl + l - 1] = '\0';
- // Log each line as is
- if (length(type) == 0) {
- const failable<bool> val = scribe::log(buf, category, sc);
- if (!hasContent(val))
- return 1;
- continue;
- }
-
- // Log each line prefixed with time and a type tag
- ostringstream os;
- os << "[" << logTime() << "] [" << type << "] " << buf;
- const failable<bool> val = scribe::log(c_str(str(os)), category, sc);
+ // Log the line
+ const failable<bool> val = scribe::log(buf, category, sc);
if (!hasContent(val))
return 1;
}
@@ -72,6 +74,6 @@ int cat(const string& category, const st
}
int main(const int argc, const char** argv) {
- return tuscany::scribecat::cat(argc < 2? "default" : argv[1], argc < 3? ""
: argv[2]);
+ return tuscany::scribecat::cat(argc < 2? "localhost" : argv[1], argc < 3?
"default" : argv[2], argc < 4? "" : argv[3]);
}
Modified: tuscany/sca-cpp/trunk/components/log/scribe-tail-start
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/log/scribe-tail-start?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/log/scribe-tail-start (original)
+++ tuscany/sca-cpp/trunk/components/log/scribe-tail-start Mon Apr 9 06:22:16
2012
@@ -35,10 +35,11 @@ else
file=$1
fi
fi
+host=`hostname`
mkdir -p `dirname $file`
touch $file
file=`echo "import os; print os.path.realpath('$file')" | python`
-tail -f -n 0 $file | $here/scribe-cat $category $type &
+tail -f -n 0 $file | $here/scribe-cat $host $category $type &
Modified: tuscany/sca-cpp/trunk/components/log/scribed-central-conf
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/log/scribed-central-conf?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/log/scribed-central-conf (original)
+++ tuscany/sca-cpp/trunk/components/log/scribed-central-conf Mon Apr 9
06:22:16 2012
@@ -42,7 +42,6 @@ check_interval=3
<store>
category=default
type=buffer
-
target_write_size=20480
max_write_interval=1
buffer_send_rate=2
@@ -50,7 +49,10 @@ retry_interval=30
retry_interval_range=10
<primary>
+category=default
type=file
+target_write_size=20480
+max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central
base_filename=central
@@ -62,7 +64,10 @@ rotate_minute=10
</primary>
<secondary>
+category=default
type=file
+target_write_size=20480
+max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central-secondary
base_filename=central
Copied: tuscany/sca-cpp/trunk/components/log/scribed-central-firehose-conf
(from r1311136, tuscany/sca-cpp/trunk/components/log/scribed-central-conf)
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/log/scribed-central-firehose-conf?p2=tuscany/sca-cpp/trunk/components/log/scribed-central-firehose-conf&p1=tuscany/sca-cpp/trunk/components/log/scribed-central-conf&r1=1311136&r2=1311137&rev=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/log/scribed-central-conf (original)
+++ tuscany/sca-cpp/trunk/components/log/scribed-central-firehose-conf Mon Apr
9 06:22:16 2012
@@ -30,6 +30,8 @@ fi
mkdir -p $root/scribe/conf
mkdir -p $root/scribe/logs/central
mkdir -p $root/scribe/logs/central-secondary
+mkdir -p $root/scribe/logs/firehose
+mkdir -p $root/scribe/logs/firehose-secondary
cat >$root/scribe/conf/scribe-central.conf <<EOF
# Generated by: scribed-central-conf $*
@@ -41,8 +43,13 @@ check_interval=3
# Log store configuration
<store>
category=default
-type=buffer
+type=multi
+target_write_size=20480
+max_write_interval=1
+<store0>
+category=default
+type=buffer
target_write_size=20480
max_write_interval=1
buffer_send_rate=2
@@ -50,7 +57,10 @@ retry_interval=30
retry_interval_range=10
<primary>
+category=default
type=file
+target_write_size=20480
+max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central
base_filename=central
@@ -62,12 +72,54 @@ rotate_minute=10
</primary>
<secondary>
+category=default
type=file
+target_write_size=20480
+max_write_interval=1
fs_type=std
file_path=$root/scribe/logs/central-secondary
base_filename=central
max_size=3000000
</secondary>
+</store0>
+
+<store1>
+category=default
+type=buffer
+target_write_size=20480
+max_write_interval=1
+buffer_send_rate=2
+retry_interval=30
+retry_interval_range=10
+
+<primary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/firehose
+base_filename=central
+max_size=1000000
+add_newlines=1
+write_stats=no
+create_symlink=no
+</primary>
+
+<secondary>
+category=default
+type=file
+target_write_size=20480
+max_write_interval=1
+fs_type=std
+file_path=$root/scribe/logs/firehose-secondary
+base_filename=central
+max_size=3000000
+</secondary>
+
+</store1>
+
</store>
+
EOF
Copied: tuscany/sca-cpp/trunk/components/log/scribed-central-mkfirehose (from
r1311136, tuscany/sca-cpp/trunk/components/log/scribe-tail-start)
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/log/scribed-central-mkfirehose?p2=tuscany/sca-cpp/trunk/components/log/scribed-central-mkfirehose&p1=tuscany/sca-cpp/trunk/components/log/scribe-tail-start&r1=1311136&r2=1311137&rev=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/log/scribe-tail-start (original)
+++ tuscany/sca-cpp/trunk/components/log/scribed-central-mkfirehose Mon Apr 9
06:22:16 2012
@@ -17,28 +17,18 @@
# specific language governing permissions and limitations
# under the License.
-# Tail a file and pipe into scribe-cat
+# Create a firehose fifo pipe for a log category
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname
$here`
+mkdir -p $1
+root=`echo "import os; print os.path.realpath('$1')" | python`
-category=""
-type=""
-file=""
-if [ "$3" != "" ]; then
- category=$1
- type=$2
- file=$3
-else
- if [ "$2" != "" ]; then
- category=$1
- file=$2
- else
- file=$1
- fi
+category=$2
+if [ "$category" = "" ]; then
+ category="default"
fi
-mkdir -p `dirname $file`
-touch $file
-file=`echo "import os; print os.path.realpath('$file')" | python`
-
-tail -f -n 0 $file | $here/scribe-cat $category $type &
+mkdir -p $root/scribe/logs/firehose/$category
+if [ ! -e "$root/scribe/logs/firehose/$category/$category""_00000" ]; then
+ mkfifo "$root/scribe/logs/firehose/$category/$category""_00000"
+fi
Modified: tuscany/sca-cpp/trunk/components/sqldb/pgsql-conf
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/sqldb/pgsql-conf?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/sqldb/pgsql-conf (original)
+++ tuscany/sca-cpp/trunk/components/sqldb/pgsql-conf Mon Apr 9 06:22:16 2012
@@ -61,15 +61,15 @@ cat >>$root/sqldb/data/postgresql.conf <
# Generated by: pgsql-conf $*
# Setup logging
-log_min_messages = INFO
-log_min_error_statement = INFO
-log_min_duration_statement = 0
+log_min_messages = NOTICE
+log_min_error_statement = NOTICE
+log_min_duration_statement = -1
log_checkpoints = on
-log_connections = on
-log_disconnections = on
-log_duration = on
+log_connections = off
+log_disconnections = off
+log_duration = off
log_lock_waits = on
-log_statement = all
+log_statement = none
# Listen
listen_addresses = '$ip'
Modified: tuscany/sca-cpp/trunk/components/sqldb/pgsql-standby-conf
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/components/sqldb/pgsql-standby-conf?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/components/sqldb/pgsql-standby-conf (original)
+++ tuscany/sca-cpp/trunk/components/sqldb/pgsql-standby-conf Mon Apr 9
06:22:16 2012
@@ -63,7 +63,7 @@ mkdir -p $root/sqldb/archive
# Initialize from a backup of the master
if [ ! -f $root/sqldb/data/postgresql.conf ]; then
- (curl -L http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb -xz)
2>&1 | sh $root/sqldb/logger
+ (curl -L -# http://$mhost:$mhttpport/pgsql-backup | tar -C $root/sqldb
-xz) 2>&1 | grep -v "100.0%" | sh $root/sqldb/logger
rm -rf $root/sqldb/data/postmaster.pid $root/sqldb/data/pg_xlog
mkdir -p $root/sqldb/data/pg_xlog/archive_status
chmod 700 $root/sqldb/data/pg_xlog/archive_status
@@ -73,7 +73,18 @@ fi
cp $root/sqldb/data/postgresql-init.conf $root/sqldb/data/postgresql.conf
cat >>$root/sqldb/data/postgresql.conf <<EOF
-# Generated by: standby-conf $*
+# Generated by: pgsql-standby-conf $*
+
+# Setup logging
+log_min_messages = NOTICE
+log_min_error_statement = NOTICE
+log_min_duration_statement = -1
+log_checkpoints = on
+log_connections = off
+log_disconnections = off
+log_duration = off
+log_lock_waits = on
+log_statement = none
# Listen
listen_addresses = '$ip'
@@ -95,7 +106,7 @@ EOF
# Generate recovery configuration
cat >$root/sqldb/data/recovery.conf << EOF
-# Generated by: pgsql-slave-conf $*
+# Generated by: pgsql-standby-conf $*
# Start in standby mode
standby_mode = 'on'
@@ -104,7 +115,7 @@ primary_conninfo = 'host=$mhost port=$mp
# Failover
trigger_file = '$root/sqldb/failover'
-restore_command = 'curl http://$mhost:$mhttpport/pgsql-archive/%f -o "%p"'
+restore_command = 'curl -L -# http://$mhost:$mhttpport/pgsql-archive/%f -o
"%p" 2>&1 | grep -v "100.0%"'
EOF
@@ -119,7 +130,7 @@ chmod 700 $root/sqldb/scripts/backup
# Configure HTTPD to serve backup and archive files
if [ -f "$root/conf/httpd.conf" ]; then
cat >>$root/conf/httpd.conf <<EOF
-# Generated by: pgsql-conf $*
+# Generated by: pgsql-standby-conf $*
# Serve PostgreSQL backup and WAL archive files
ScriptAlias /pgsql-backup "$root/sqldb/scripts/backup"
Modified: tuscany/sca-cpp/trunk/hosting/server/ssl-start
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/hosting/server/ssl-start?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/hosting/server/ssl-start (original)
+++ tuscany/sca-cpp/trunk/hosting/server/ssl-start Mon Apr 9 06:22:16 2012
@@ -22,6 +22,7 @@
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname
$here`
jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" |
python`
+host=`hostname`
# Create SSL certificates
../../modules/http/ssl-ca-conf tmp www.example.com
@@ -81,20 +82,20 @@ fi
if [ -x ../../components/log/scribe-cat ]; then
cat >tmp/conf/log.conf <<EOF
# Generated by: ssl-start $*
-ErrorLog "|$here/../../components/log/scribe-cat server"
-CustomLog "|$here/../../components/log/scribe-cat server" combined
+ErrorLog "|$here/../../components/log/scribe-cat $host server"
+CustomLog "|$here/../../components/log/scribe-cat $host server" combined
EOF
cat >tmp/conf/log-ssl.conf <<EOF
# Generated by: ssl-start $*
-CustomLog "|$here/../../components/log/scribe-cat server" sslcombined
+CustomLog "|$here/../../components/log/scribe-cat $host server" sslcombined
EOF
# cat >tmp/conf/mod-security-log.conf <<EOF
# Generated by: ssl-start $*
-#SecAuditLog "|$here/../../components/log/scribe-cat secaudit"
+#SecAuditLog "|$here/../../components/log/scribe-cat $host secaudit"
#
#EOF
Modified: tuscany/sca-cpp/trunk/hosting/server/start
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/hosting/server/start?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/hosting/server/start (original)
+++ tuscany/sca-cpp/trunk/hosting/server/start Mon Apr 9 06:22:16 2012
@@ -22,6 +22,7 @@
here=`echo "import os; print os.path.realpath('$0')" | python`; here=`dirname
$here`
jsprefix=`echo "import os; print os.path.realpath('$here/../../modules/js')" |
python`
+host=`hostname`
# Configure and start logging
if [ -x ../../components/log/scribe-cat ]; then
@@ -48,8 +49,8 @@ if [ -x ../../components/log/scribe-cat
cat >tmp/conf/log.conf <<EOF
# Generated by: start $*
LogLevel notice
-ErrorLog "|$here/../../components/log/scribe-cat server"
-CustomLog "|$here/../../components/log/scribe-cat server" combined
+ErrorLog "|$here/../../components/log/scribe-cat $host server"
+CustomLog "|$here/../../components/log/scribe-cat $host server" combined
EOF
Added: tuscany/sca-cpp/trunk/patches/scribe.patch
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/patches/scribe.patch?rev=1311137&view=auto
==============================================================================
--- tuscany/sca-cpp/trunk/patches/scribe.patch (added)
+++ tuscany/sca-cpp/trunk/patches/scribe.patch Mon Apr 9 06:22:16 2012
@@ -0,0 +1,50 @@
+--- src/common.h
++++ src/common.h
+@@ -42,6 +42,8 @@
+ #include <sys/types.h>
+ #include <sys/stat.h>
+ #include <unistd.h>
++#include <signal.h>
++#include <fcntl.h>
+ #include <boost/shared_ptr.hpp>
+ #include <boost/filesystem/operations.hpp>
+ #include <boost/filesystem/convenience.hpp>
+
+--- src/file.cpp
++++ src/file.cpp
+@@ -74,9 +74,21 @@ bool StdFile::openRead() {
+ }
+
+ bool StdFile::openWrite() {
++ // if file is a fifo, temporarily open it for read
++ int fd = -1;
++ struct stat st;
++ int s = stat(filename.c_str(), &st);
++ if (s != -1 && S_ISFIFO(st.st_mode))
++ fd = ::open(filename.c_str(), O_RDONLY | O_NONBLOCK);
++
+ // open file for write in append mode
+ ios_base::openmode mode = fstream::out | fstream::app;
+- return open(mode);
++ bool r = open(mode);
++
++ // close fifo
++ if (fd != -1)
++ ::close(fd);
++ return r;
+ }
+
+ bool StdFile::openTruncate() {
+
+--- src/scribe_server.cpp
++++ src/scribe_server.cpp
+@@ -55,6 +55,8 @@ int main(int argc, char **argv) {
+ if (-1 == setrlimit(RLIMIT_NOFILE, &r_fd)) {
+ LOG_OPER("setrlimit error (setting max fd size)");
+ }
++
++ signal(SIGPIPE, SIG_IGN);
+
+ int next_option;
+ const char* const short_options = "hp:c:";
+
Modified: tuscany/sca-cpp/trunk/ubuntu/ubuntu-install
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/ubuntu/ubuntu-install?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/ubuntu/ubuntu-install (original)
+++ tuscany/sca-cpp/trunk/ubuntu/ubuntu-install Mon Apr 9 06:22:16 2012
@@ -302,6 +302,8 @@ fi
curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
tar xzf scribe-2.2.tar.gz
cd scribe
+curl -OL
http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
+patch -p0 <scribe.patch
./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin
--with-thriftpath=$build/thrift-0.2.0-bin
--with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
make
make install
Modified: tuscany/sca-cpp/trunk/ubuntu/ubuntu-install-all
URL:
http://svn.apache.org/viewvc/tuscany/sca-cpp/trunk/ubuntu/ubuntu-install-all?rev=1311137&r1=1311136&r2=1311137&view=diff
==============================================================================
--- tuscany/sca-cpp/trunk/ubuntu/ubuntu-install-all (original)
+++ tuscany/sca-cpp/trunk/ubuntu/ubuntu-install-all Mon Apr 9 06:22:16 2012
@@ -356,6 +356,8 @@ fi
curl -OL http://github.com/downloads/facebook/scribe/scribe-2.2.tar.gz
tar xzf scribe-2.2.tar.gz
cd scribe
+curl -OL
http://svn.apache.org/repos/asf/tuscany/sca-cpp/trunk/patches/scribe.patch
+patch -p0 <scribe.patch
./bootstrap.sh --prefix=$build/scribe-2.2-bin PY_PREFIX=$build/scribe-2.2-bin
--with-thriftpath=$build/thrift-0.2.0-bin
--with-fb303path=$build/thrift-0.2.0-bin/contrib/fb303 --disable-static
make
make install
