Thanks Ed and Geert ! One question regarding the Xquery solution ... would the call to xdmp:filesystem-file($logfile) require Expanded Tree Cache size to be as big as the largest ErrorLog.txt I anticipate ?
Regards, Danny On Wed, May 29, 2013 at 3:16 AM, Geert Josten <[email protected]> wrote: > 30-ish? > > > > let $raw := > > <mllog>{ > > let $logfile := "./Data/Logs/ErrorLog.txt" > > let $log := xdmp:filesystem-file($logfile) > > let $lines := tokenize($log, "\n") > > for $line in $lines > > return > > if (contains($line, 'Notice: Starting')) then > > <start time="{xs:time(substring-before(substring-after($line, ' > '), ' '))}">{$line}</start> > > else if (contains($line, 'Notice: Restarting')) then > > <stop time="{xs:time(substring-before(substring-after($line, ' '), > ' '))}">{$line}</stop> > > else if (contains($line, 'Warning: Hung')) then > > <hung time="{xs:time(substring-before(substring-after($line, ' '), > ' '))}" delay="{xs:dayTimeDuration(concat('PT', > substring-before(substring-after($line, 'Warning: Hung '), ' sec'), > 'S'))}">{$line}</hung> > > else () > > }</mllog> > > let $active-at-start := empty($raw/(start|stop)) or > exists($raw/(start|stop)[1]/self::stop) > > let $downtime := > > sum( > > for $stop in $raw/stop > > let $next-start := xs:time(($stop/following-sibling::start[1]/@time, > "23:59.9999")[1]) > > return > > $next-start - xs:time($stop/@time) > > ) + > > sum( > > for $hung in $raw/hung > > return > > xs:dayTimeDuration($hung/@delay) > > ) > > return > > <mllog active-at-start="{string($active-at-start)}" > uptime="{xs:dayTimeDuration('P1D') - $downtime}" downtime="{$downtime}" > host="{xdmp:host-name(xdmp:host())}" date="{current-date()}">{ > > $raw/* > > }</mllog> > > > > *Van:* Geert Josten [mailto:[email protected]] > *Verzonden:* woensdag 29 mei 2013 8:28 > *Aan:* MarkLogic Developer Discussion; [email protected] > *CC:* DL-ICS-MARKLOGIC > *Onderwerp:* RE: [MarkLogic Dev General] Computing ML uptime percentage > (Danny Sinang) > > > > Hmm, can’t help but wonder how many lines of ML-xquery it would need to do > this from within MarkLogic.. > > > > J > > > > *Van:* [email protected] [mailto: > [email protected]] *Namens *Ed Felt > *Verzonden:* woensdag 29 mei 2013 0:40 > *Aan:* [email protected]; [email protected] > *CC:* DL-ICS-MARKLOGIC > *Onderwerp:* [MarkLogic Dev General] Computing ML uptime percentage > (Danny Sinang) > > > > Danny: > > I created a python script that reads ErrorLog(_*).txt for this purpose and > creates xml files with the "down" time included it in. You are free to use > it. I included it below in this email. I just use an @daily cron to run > it, as I need daily reports, so I can later pull the xml files in to > MarkLogic for analysis. It works on Red-Hat Linux or an OS that supports > BASH and Python. This script uses grep, which should use very low system > resources. It has not been tested for performance or usability on multiple > MarkLogic versions. This is also a per host solution, so you would have to > decide what to do with the data from all nodes in a cluster to calculate > down time. > > > > mldown.py: > ############################################## > #!/usr/bin/python > > """ > mldown.py calculates and stores MarkLogic server down time (in seconds) > for the single node it runs on from <error_log_dir>/ErrorLog*.txt and saves > logs to <xml_out_dir>/mldownlog<YYYMMDDHHMMSS.NNN><hostname>.xml for each > outage > Sample xml: > <?xml version="1.0" ?> > <mldown downtime_sec="250.0" host="edfeltlenovow530" > run="2013-05-23T23:53:26"> > <stop>2013-05-23 16:01:44.907 Notice: Stopping by SIGTERM from pid > 2489</stop> > <start>2013-05-23 16:05:54.916 Notice: Starting MarkLogic Server > 7.0-20130513 x86_64 in /opt/MarkLogic with data in > /opt/MarkLogic/Data</start> > </mldown> > """ > > #debug: > def p(x): > print(x) > > #standard python libs > from subprocess import Popen, PIPE > from xml.dom.minidom import Document > from socket import gethostname > import datetime > from time import strptime,mktime > import sys > > if len(sys.argv) != 3: > print("required parameters (with no trailing forward slash '/': " + > sys.argv[0] + " (error log directory) (xml output directory)") > #/opt/MarkLogic/Data/Logs /opt/backups/monitor/mldown > sys.exit() > > error_log_dir=sys.argv[1] #usually "/opt/MarkLogic/Data/Logs" > xml_out_dir=sys.argv[2] #usually "/opt/backups/monitor/mldown" > > Popen("mkdir -p " + xml_out_dir, shell=True, stdout=PIPE).communicate() > #possible lines: > #--- > #line before startup > #startup line > last_line = "ERROR" > for line in Popen("egrep -B 1 -h 'Notice: Starting MarkLogic Server' " + > error_log_dir + "/ErrorLo*.txt", shell=True, > stdout=PIPE).communicate()[0].split("\n"): > #only time to add stuff to the xml document: > if 'Notice: Starting MarkLogic Server' in line: > if last_line != "ERROR" and last_line != "--": #(throw out bad > data we can't make sense of) > date_string_index = last_line.find('2') > date_string_index_end = date_string_index+19 > last_line_date_raw = > last_line[date_string_index:date_string_index_end] > last_line_date = last_line_date_raw.replace(' ','T') > last_line_date_time = strptime(last_line_date_raw, "%Y-%m-%d > %H:%M:%S") > date_string_index = line.find('2') > date_string_index_end = date_string_index+19 > line_date_raw = line[date_string_index:date_string_index_end] > line_date = line_date_raw.replace(' ','T') > line_date_time = strptime(line_date_raw, "%Y-%m-%d %H:%M:%S") > > diff = mktime(line_date_time) - mktime(last_line_date_time) > down = str(diff) > # create a document object > doc = Document() > # create elements for xml file > mldown = doc.createElement("mldown") > startline = doc.createElement("start") > startline.appendChild(doc.createTextNode(line)) > stopline = doc.createElement("stop") > stopline.appendChild(doc.createTextNode(last_line)) > # create attributes for mlmon element > host = gethostname() > mldown.setAttribute("host", host) > now = datetime.datetime.now() > run = now.strftime("%Y-%m-%dT%H:%M:%S") > start_filename = line_date_raw[0:23].replace(":","").replace(" > ","").replace("-","") > log = xml_out_dir + "/mldownlog" + start_filename + host + > ".xml" > #p(log) > mldown.setAttribute("run", run) > mldown.setAttribute("down_sec", down) > mldown.setAttribute("down_date_time", last_line_date) > > mldown.appendChild(stopline) > mldown.appendChild(startline) > doc.appendChild(mldown) > # print doc to screen > p(doc.toprettyxml()) > # open xmlfile > f = open(log, "w") > # write what is in doc and put it into xmlfile > f.write(doc.toprettyxml()) > f.close() > #there is no last line for our "future" Starting line > last_line = "ERROR" > else: > last_line = line > ############################################## > > C. Ed Felt ICS x20697 [email protected] (MarkLogic Group: x27176, > [email protected]) > > > > NOTICE: This email message is for the sole use of the intended > recipient(s) and may contain confidential and privileged information. Any > unauthorized review, use, disclosure or distribution is prohibited. If you > are not the intended recipient, please contact the sender by reply email > and destroy all copies of the original message. > > > > _______________________________________________ > General mailing list > [email protected] > http://developer.marklogic.com/mailman/listinfo/general > >
_______________________________________________ General mailing list [email protected] http://developer.marklogic.com/mailman/listinfo/general
