Hello, I'm working on a script that allows to generate picviz parsers interactively. The script should offer three levels of setup : - order of axis, colors (parser level) - parameters given to the parser, in order to colorize a value (command line call of the parser) - picviz parameters (picviz level)
An important functionality should be to allow the rewriting of a parser, by
loading it and modifying its setup. This is done by writing in the header of
the parser information concerning the previous setup.
The v0.1 of the script will allow you to generate a picviz parser for
dansguardian logs interactively. You can choose the order of the log fields you
need and the axis order.
If you've already generated a parser but you want to modify it, you can enter
the file name and the order of axis will be loaded.
For the moment you can only play with Time, User, IP used and URL requested.
As the code is pretty independant from the type of log you will parse, it
should be easy to modify the script in order to be able to create parser for
other logs.
Next steps are :
- setup of picviz options
- more log fields to play with
Exemple :
$ ./genPicvizParser.py
#### Directory listing ####
##################
access20Short.log.csv
genPicvizParser.py
##################
Enter the name of the parser to use or to create :parser.pl
File created
Available fields :
1- Time when the request was done
2- User making the request
3- IP address used by the user
4- URL requested by the user
5- HTTP method used : PUT or GET
6- Size of the answer
7- Mime Type : HTML, CSS, ...
Parser structure configuration (for instance: 1/2/3/4 ) [1/2/3/4]:
1/2/3/4
Resume :
1 Time
2 Login
3 IP
4 URL
Enter the graph name [DansGuardian picviz analysis]:
$ ls
access.log.csv genPicvizParser.py parser.pl
$ chmod +x parser.pl && ./parser.pl access.log.csv
header {
title = "DansGuardian picviz analysis";
}
axes {
timeline t [label="Time",relative="true"];
string n [label="Login",relative="true"];
ipv4 i [label="Adr_IP",relative="true"];
string u [label="URL",relative="true"];
}
data {
...
}
The output can then be used to generate ||-coords graphs.
Fell free to comment.
Best regards,
--
Julien Miotte
#!/usr/bin/python
# genPicvizParser.pl : Picviz Parser Generator
# Made by Julien Miotte (License : GPLv3)
VER="0.1"
import os
# Show files in the current directory
print "#### Directory listing ####"
print "###########################"
for i in os.listdir("."):
print i
print "###########################\n"
# Recovery of old parser or creation of a new one
filename=raw_input("Enter the name of the parser to use or to create :")
recupere=False
try:
file=open(filename,'r')
# Read the second line and strip '#' and '\n'
choice=file.readlines()[1].split('# ')[1].split('\n')[0]
recupere=True
file.close()
print "File recovered"
backup=raw_input("Create backup file ? (yes/no) : ")
if backup[0]=='y':
cmd="cp "+filename+" "+filename+".bck"
os.system(cmd)
except:
print "File created"
file=open(filename,"w")
fields={}
fields['1']=("Time","Time when the request was done","print \" timeline t [label=\\\"Time\\\",relative=\\\"true\\\"];\\n\";","t=\\\"$time\\\"")
fields['2']=("Login","User making the request","print \" string n [label=\\\"Login\\\",relative=\\\"true\\\"];\\n\"; ","n=\\\"$last $first\\\"")
fields['3']=("IP","IP address used by the user","print \" ipv4 i [label=\\\"Adr_IP\\\",relative=\\\"true\\\"];\\n\";","i=\\\"$d.$c.$b.$a\\\"")
fields['4']=("URL","URL requested by the user","print \" string u [label=\\\"URL\\\",relative=\\\"true\\\"];\\n\";","u=\\\"$f\\\"")
fields['5']=("HTTP method","HTTP method used : PUT or GET","print \" string m [label=\\\"HTTPMethod\\\",relative=\\\"true\\\"];\\n\";")
fields['6']=("Size","Size of the answer","print \" int s [label=\\\"Size\\\",relative=\\\"true\\\"];\\n\";","")
fields['7']=("Mime Type","Mime Type : HTML, CSS, ...","print \" string m [label=\\\"Mime_Type\\\",relative=\\\"true\\\"];\\n\";","")
# Parser structure
if not recupere:
print "\nAvailable fields : "
items=fields.keys()
items.sort()
for i in items:
print str(i)+"- "+fields[i][1]
choice=raw_input("Parser structure configuration (for instance: 1/2/3/4 ) [1/2/3/4]: \n" )
if choice == '':
choice='1/2/3/4'
print "\nResume :"
compt=0
try:
for i in choice.split("/"):
compt += 1
print compt, fields[i][0]
except:
print "Wrong choice"
# Name the graph
title=raw_input("\nEnter the graph name [DansGuardian picviz analysis]: ")
if title=='':
title="DansGuardian picviz analysis"
# Write the parser
file.write("#!/usr/bin/perl\n")
file.write("# "+choice+"\n")
file.write("# Parser generated using genPicvizParser v."+VER+"\n")
file.write("print \"header {\\n\";\n")
file.write("print \" title = \\\""+title+"\\\";\\n\";\n")
file.write("print \"}\\n\";\n\n")
file.write("print \"axes {\\n\";\n")
# Declare each field
for i in choice.split("/"):
file.write(fields[i][2]+"\n")
file.write("print \"}\\n\";\n")
file.write("print \"data {\\n\";\n")
file.write("while ($line = <>) {\n")
# Match each log line for important data
file.write(" $line =~ s/\\\"//g;\n")
file.write(" $line =~ s/,/ /g;\n")
file.write(" if ($line =~ m/^\S* (\d+:\d+):\d+ (\w*.\w*) (\S*) http:\/\/([\w\.-]*)(\S*) /) {\n\n")
file.write(" $time=$1;\n")
file.write(" $name=$2;\n")
file.write(" $ipaddr=$3;\n")
file.write(" $url=$4;\n")
file.write(" if ($name =~ m/(\w*)\.(\w*)/) {\n")
file.write(" $first = $1;\n")
file.write(" $last = $2;\n")
file.write(" }\n")
file.write(" else {\n")
file.write(" $last=$name;\n")
file.write(" $first=\"\";\n")
file.write(" }\n")
file.write(" $ipaddr =~ m/(\d*)\.(\d*)\.(\d*)\.(\d*)/;\n")
file.write(" $a=$1;\n")
file.write(" $b=$2;\n")
file.write(" $c=$3;\n")
file.write(" $d=$4;\n")
file.write(" if ($url =~ m/^\w*\.(\w*)$/) {\n")
file.write(" $e=\"www\";\n")
file.write(" $f=$url;\n")
file.write(" }\n")
file.write(" else {\n")
file.write(" if ($url =~ /\d$/) {\n")
file.write(" $e=\"\";\n")
file.write(" $f=$url;\n")
file.write(" }\n")
file.write(" else {\n")
file.write(" if ($url =~ /^\w*$/) {\n")
file.write(" $f=$url;\n")
file.write(" $e=\"\";")
file.write(" }\n")
file.write(" else {\n")
file.write(" $url =~ m/^([^\/\.]*)\.(\S*)/;\n")
file.write(" $e=$1;\n")
file.write(" $f=$2;\n")
file.write(" }\n")
file.write(" }\n")
file.write(" }\n")
# Assign each string match to a field
line=" print \" "
for i in choice.split("/"):
line+=fields[i][3]+", "
# Replace the last comma by ';'
line=line[:-2]+";\\n\";\n"
file.write(line)
file.write(" }\n")
file.write("}\n")
file.write("print \"}\\n\";\n")
file.close()
parser.pl
Description: Perl program
_______________________________________________ Picviz mailing list Picviz@wallinfire.net http://www.wallinfire.net/cgi-bin/mailman/listinfo/picviz
