On Wed, Aug 22, 2012 at 3:05 PM, Shaka Lewis <shaka.le...@gmail.com> wrote:
> And what OSSEC processes are running at this point?
> Did you run analysisd in gdb? Did it crash? Is there a backtrace?
>
> I'll throw in some more questions, because I need some more to not be
> answered. Is this a server or a standalone installation? Has it ever
> worked? Did you change anything?
>
> The ossec processes running at this point are execd, logcollector, and
> monitord.
>
>
> AnalysisD crashed and here is the output:
>
> Program received signal SIGSEGV, Segmentation fault.
> [Switching to process 26814]
> 0x0000000000000000 in ?? ()
> Missing separate debuginfos, use: debuginfo-install
> glibc-2.12-1.47.el6_2.12.x86_64
> (This version of glibc is already installed on the system)
>
>
> This is a server install and stopped working after migrating to new hardware.
>
>
> 2012/08/21 17:19:43 ossec-syscheckd(1210): ERROR: Queue
> '/var/ossec/queue/ossec/queue' not accessible: 'Connection refused'.
> 2012/08/21 17:19:43 ossec-rootcheck(1210): ERROR: Queue
> '/var/ossec/queue/ossec/queue' not accessible: 'Connection refused'.
> 2012/08/21 17:19:51 ossec-syscheckd(1210): ERROR: Queue
> '/var/ossec/queue/ossec/queue' not accessible: 'Connection refused'.
> 2012/08/21 17:19:51 ossec-rootcheck(1210): ERROR: Queue
> '/var/ossec/queue/ossec/queue' not accessible: 'Connection refused'.
> 2012/08/21 17:20:04 ossec-syscheckd(1210): ERROR: Queue
> '/var/ossec/queue/ossec/queue' not accessible: 'Connection refused'.
>
>
>
>
>
>
>
> Here is the sequence of events before the problems started occuring.
>
> 1. The old ossec server was replaced with new hardware. I had the
> UNIX admin copy the ossec directories and ossec groups and users to
> the new hardware. This caused permissions issues that I had to
> resolve.
>
Yeah, no idea what all could be broken with this. Just backup the
configs and the rids and reinstall. Stop making this so difficult.
> The only changes I made to the local rules file was copying a ftp
> connect signature and limiting the number of times it alerts. Current
> state as of today is that ossec processes run for about an hours or so
> then the error messages start.
>
> First error messages before first socket error:
>
> 2012/08/21 18:16:41 ossec-rootcheck: DEBUG: Completed with all checks.
> 2012/08/21 18:16:46 ossec-rootcheck: INFO: Ending rootcheck scan.
> 2012/08/21 18:16:46 ossec-rootcheck: DEBUG: Leaving run_rk_check
> 2012/08/21 19:22:09 ossec-logcollector: socketerr (not available).
> 2012/08/21 19:22:09 ossec-logcollector(1224): ERROR: Error sending
> message to queue.
> 2012/08/21 19:22:12 ossec-logcollector(1210): ERROR: Queue
> '/var/ossec/queue/ossec/queue' not accessible: 'Connection refused'.
> 2012/08/21 19:22:12 ossec-logcollector(1211): ERROR: Unable to access
> queue: '/var/ossec/queue/ossec/queue'. Giving up..
> 2012/08/21 19:31:30 ossec-monitord: socketerr (not available).
>
>
> How many times did I ask which processes were running? ( I also
> answered this question) Here are the processes running:
>
> root 30371 1 0 Aug21 ? 00:00:00 /var/ossec/bin/ossec-execd -d
> ossec 30394 1 0 Aug21 ? 00:00:00 /var/ossec/bin/ossec-monitord
> -d
>
You answered this after I asked a second time.
>
>
> stack trace:
>
>
>
>
> read(5, "Aug 21 10:10:24 alalpsec006 sudo"..., 4096) = 107
> stat("/var/ossec/queue/ossec/.wait", 0x7fff415fc1d0) = -1 ENOENT (No
> such file or directory)
> sendto(4, "1:/var/log/messages:Aug 21 10:10"..., 127, 0, NULL, 0) = 127
> read(5, "", 4096) = 0
> read(6, "Aug 21 10:10:24 alalpsec006 sudo"..., 4096) = 114
> stat("/var/ossec/queue/ossec/.wait", 0x7fff415fc1d0) = -1 ENOENT (No
> such file or directory)
> sendto(4, "1:/var/log/secure:Aug 21 10:10:2"..., 132, 0, NULL, 0) = 132
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "", 4096) = 0
> read(6, "", 4096) = 0
> read(7, "", 4096) = 0
> read(8, "", 4096) = 0
> select(0, NULL, NULL, NULL, {2, 0}) = 0 (Timeout)
> read(5, "Aug 21 10:10:50 alalpsec006 kern"..., 4096) = 146
> stat("/var/ossec/queue/ossec/.wait", 0x7fff415fc1d0) = -1 ENOENT (No
> such file or directory)
> sendto(4, "1:/var/log/messages:Aug 21 10:10"..., 166, 0, NULL, 0) = -1
> ECONNREFUSED (Connection refused)
> stat("/etc/localtime", {st_mode=S_IFREG|0644, st_size=3519, ...}) = 0
> open("/var/ossec/logs/ossec.log", O_WRONLY|O_CREAT|O_APPEND, 0666) = 9
> fstat(9, {st_mode=S_IFREG|0755, st_size=23385862, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0x7fb962828000
> fstat(9, {st_mode=S_IFREG|0755, st_size=23385862, ...}) = 0
> lseek(9, 23385862, SEEK_SET) = 23385862
> write(9, "2012/08/21 10:10:51 ossec-logcol"..., 67) = 67
> close(9) = 0
> munmap(0x7fb962828000, 4096) = 0
> close(4) = 0
> stat("/etc/localtime", {st_mode=S_IFREG|0644, st_size=3519, ...}) = 0
> open("/var/ossec/logs/ossec.log", O_WRONLY|O_CREAT|O_APPEND, 0666) = 4
> fstat(4, {st_mode=S_IFREG|0755, st_size=23385929, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0x7fb962828000
> fstat(4, {st_mode=S_IFREG|0755, st_size=23385929, ...}) = 0
> lseek(4, 23385929, SEEK_SET) = 23385929
> write(4, "2012/08/21 10:10:51 ossec-logcol"..., 85) = 85
> close(4) = 0
> munmap(0x7fb962828000, 4096) = 0
> stat("/var/ossec/queue/ossec/queue", {st_mode=S_IFSOCK|0660,
> st_size=0, ...}) = 0
> socket(PF_FILE, SOCK_DGRAM, 0) = 4
> connect(4, {sa_family=AF_FILE, path="/var/ossec/queue/ossec/queue"},
> 30) = -1 ECONNREFUSED (Connection refused)
> rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
> rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
> rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> nanosleep({1, 0}, 0x7fff415fda50) = 0
> socket(PF_FILE, SOCK_DGRAM, 0) = 9
> connect(9, {sa_family=AF_FILE, path="/var/ossec/queue/ossec/queue"},
> 30) = -1 ECONNREFUSED (Connection refused)
> rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
> rt_sigaction(SIGCHLD, NULL, {SIG_DFL, [], 0}, 8) = 0
> rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> nanosleep({2, 0}, 0x7fff415fda50) = 0
> socket(PF_FILE, SOCK_DGRAM, 0) = 10
> connect(10, {sa_family=AF_FILE, path="/var/ossec/queue/ossec/queue"},
> 30) = -1 ECONNREFUSED (Connection refused)
> stat("/etc/localtime", {st_mode=S_IFREG|0644, st_size=3519, ...}) = 0
> open("/var/ossec/logs/ossec.log", O_WRONLY|O_CREAT|O_APPEND, 0666) = 11
> fstat(11, {st_mode=S_IFREG|0755, st_size=23386263, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0x7fb962828000
> fstat(11, {st_mode=S_IFREG|0755, st_size=23386263, ...}) = 0
> lseek(11, 23386263, SEEK_SET) = 23386263
> write(11, "2012/08/21 10:10:54 ossec-logcol"..., 128) = 128
> close(11) = 0
> munmap(0x7fb962828000, 4096) = 0
> stat("/etc/localtime", {st_mode=S_IFREG|0644, st_size=3519, ...}) = 0
> open("/var/ossec/logs/ossec.log", O_WRONLY|O_CREAT|O_APPEND, 0666) = 11
> fstat(11, {st_mode=S_IFREG|0755, st_size=23386391, ...}) = 0
> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
> 0) = 0x7fb962828000
> fstat(11, {st_mode=S_IFREG|0755, st_size=23386391, ...}) = 0
> lseek(11, 23386391, SEEK_SET) = 23386391
> write(11, "2012/08/21 10:10:54 ossec-logcol"..., 121) = 121
> close(11) = 0
> munmap(0x7fb962828000, 4096) = 0
> exit_group(1) = ?
> Process 19124 detached
>
>
That's the oddest gdb output I've ever seen.
