Fix pylint and pep8 issues for the following Python files: ./tools/devel/dot/trace2dot ./tools/devel/review/patch-tokenize.py ./src/imm/tools/immxml-merge ./src/imm/tools/immxml-validate ./src/imm/tools/baseimm.py --- src/imm/tools/baseimm.py | 376 ++++++++------- src/imm/tools/immxml-merge | 890 +++++++++++++++++++---------------- src/imm/tools/immxml-validate | 792 +++++++++++++++++-------------- tools/devel/dot/trace2dot | 70 +-- tools/devel/review/patch-tokenize.py | 16 +- 5 files changed, 1176 insertions(+), 968 deletions(-)
diff --git a/src/imm/tools/baseimm.py b/src/imm/tools/baseimm.py index 09a9b85..fe1c3ce 100644 --- a/src/imm/tools/baseimm.py +++ b/src/imm/tools/baseimm.py @@ -1,221 +1,258 @@ -''' - -*- OpenSAF -*- - - (C) Copyright 2009 The OpenSAF Foundation - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed - under the GNU Lesser General Public License Version 2.1, February 1999. - The complete license can be accessed from the following location: - http://opensource.org/licenses/lgpl-license.php - See the Copying file included with the OpenSAF distribution for full - licensing terms. - - Author(s): Ericsson AB -''' - -import sys,glob,os +# -*- OpenSAF -*- +# +# (C) Copyright 2009 The OpenSAF Foundation +# (C) Copyright Ericsson AB 2015, 2016, 2017. All rights reserved. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed +# under the GNU Lesser General Public License Version 2.1, February 1999. +# The complete license can be accessed from the following location: +# http://opensource.org/licenses/lgpl-license.php +# See the Copying file included with the OpenSAF distribution for full +# licensing terms. +# +# Author(s): Ericsson AB +# +""" Common utils to manipulate immxml files """ +from __future__ import print_function +import os +import sys +import glob +import xml.dom.minidom from subprocess import call -class BaseOptions: + +class BaseOptions(object): + """ Base options for immxml tools """ traceOn = False @staticmethod - def printOptionSettings(): - return 'Options:\n traceOn:%s\n' % (BaseOptions.traceOn) - + def print_option_settings(): + return 'Options:\n traceOn:%s\n' % BaseOptions.traceOn -class BaseImmDocument: +class BaseImmDocument(object): + """ Base class with common methods to manipulate IMM XML documents """ imm_content_element_name = "imm:IMM-contents" imm_content_no_namespace_spec = "<imm:IMM-contents>" - imm_content_with_namespace_spec = "<imm:IMM-contents xmlns:imm=\"http://www.saforum.org/IMMSchema\"; xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; xsi:noNamespaceSchemaLocation=\"SAI-AIS-IMM-XSD-A.02.13.xsd\">" - - # helper function to identify a problem where python/minidom - # fails to parse some imm.xml files where the imm: namespace is missing - # (for example the output of the immdump tool) - def verifyInputXmlDocumentFileIsParsable(self, filename): - if os.stat(filename).st_size == 0: - abort_script("The inputfile %s is empty!", filename) - - f = open(filename) - immContentsTagFound = False - for line in f: - s = line - str = s.replace(self.imm_content_no_namespace_spec, self.imm_content_with_namespace_spec) - if len(str) != len(s): - # there was a substitution....file will not be possible to parse.... - abort_script("The inputfile lacks namespace specification in <imm:IMM-contents> tag") - elif s.find(self.imm_content_element_name) != -1: - # Assume file is ok w.r.t namespace + imm_content_with_namespace_spec = \ + "<imm:IMM-contents xmlns:imm=\"http://www.saforum.org/IMMSchema\"; " \ + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"; " \ + "xsi:noNamespaceSchemaLocation=\"SAI-AIS-IMM-XSD-A.02.13.xsd\">" + + def verify_input_xml_document_file_is_parsable(self, file_name): + """ Helper function to identify a problem where python/minidom fails to + parse some imm.xml files where the imm namespace is missing (for ex, + the output of the immdump tool) + """ + if os.stat(file_name).st_size == 0: + abort_script("The input file %s is empty!", file_name) + + doc = open(file_name) + for _line in doc: + line = _line + string = line.replace(self.imm_content_no_namespace_spec, + self.imm_content_with_namespace_spec) + if len(string) != len(line): + # There was a substitution....file will not be possible + # to parse.... + abort_script("The input file lacks namespace specification in " + "<imm:IMM-contents> tag") + elif line.find(self.imm_content_element_name) != -1: + # Assume file is ok w.r.t namespace return - - - def validateXmlFileWithSchema(self, filename, schemaFilename): + @staticmethod + def validate_xml_file_with_schema(file_name, schema_file_name): + """ Validate xml file against its schema """ trace("") - - trace("validate xml file:%s", filename) - # validate the result file with xmllint - command = "/usr/bin/xmllint --noout"+" --schema "+schemaFilename+" "+filename + trace("Validate xml file: %s", file_name) + + # Validate the result file with xmllint + command = "/usr/bin/xmllint --noout"+" --schema " + schema_file_name \ + + " " + file_name print_info_stderr("XML Schema validation (using xmllint):") - trace("validate command:%s", command) - retCode = call(command, shell=True) - if retCode != 0: - return retCode + trace("Validate command: %s", command) + ret_code = call(command, shell=True) + if ret_code != 0: + return ret_code - - trace("Successfully validated xml document file: %s", filename) + trace("Successfully validated xml document file: %s", file_name) return 0 - def formatXmlFileWithXmlLint(self, infilename, outfilename): - trace("formatXmlFileWithXmlLint() prettify xml file:%s", infilename) - # "prettify" the result file with xmllint (due to inadequate python/minidom prettify functionality) - #command = "/bin/sh -c 'XMLLINT_INDENT=\" \" ; export XMLLINT_INDENT; /usr/bin/xmllint --format "+infilename+" --output "+outfilename +"'" - command = "/bin/sh -c 'XMLLINT_INDENT=\""+"\t"+"\" ; export XMLLINT_INDENT; /usr/bin/xmllint --format "+infilename+" --output "+outfilename +"'" - trace("formatting command:%s", command) - retCode = call(command, shell=True) - #if retCode != 0: - #return retCode - #validate_failed("failed to validate input file %s:", filename) - return retCode - - def printToStdoutAndRemoveTmpFile(self, tempfile): + @staticmethod + def format_xml_file_with_xmllint(in_file_name, out_file_name): + """ Format xml file with xmllint """ + trace("formatXmlFileWithXmlLint() prettify xml file:%s", in_file_name) + + # "prettify" the result file with xmllint + # (due to inadequate python/minidom prettify functionality) + command = "/bin/sh -c 'XMLLINT_INDENT=\"" + "\t" + \ + "\"; export XMLLINT_INDENT; /usr/bin/xmllint --format " + \ + in_file_name + " --output " + out_file_name + "'" + trace("Formatting command: %s", command) + ret_code = call(command, shell=True) + # if ret_code != 0: + # return ret_code + # validate_failed("Failed to validate input file: %s", file_name) + return ret_code + + @staticmethod + def print_to_stdout_and_remove_tmp_file(temp_file): + """ Print the temp file content to stdout and remove the temp file """ # Print the tmp file to the standard output - trace("Print the tmp file to the standard output: %s", tempfile) - f = file(tempfile) + trace("Print the tmp file to the standard output: %s", temp_file) + tmp_file = open(temp_file) while True: - line = f.readline() - if len(line) == 0: + line = tmp_file.readline() + if not line: break - print line, # notice comma - f.close() - # remove temp file - trace("delete the stdout tmp file: %s", tempfile) - os.remove(tempfile) - - # helper function to handle a problem where python/minidom - # fails to parse some imm.xml files where the imm: namespace is missing - # (for example the output of the immdump tool) - # Currently NOT used..... - def openXmlDocumentFileAndCheckNamespace(self, filename): - f = open(filename) + print (line, end=" ") # Print on the same line + tmp_file.close() + # Remove temp file + trace("Delete the stdout tmp file: %s", temp_file) + os.remove(temp_file) + + def open_xml_document_file_and_check_namespace(self, file_name): + """ Helper function to handle a problem where python/minidom fails to + parse some imm.xml files where the imm namespace is missing (for ex, + the output of the immdump tool). + *** This method is currently NOT used **** + """ + doc = open(file_name) str_list = [] - immContentsTagFound = False - immContentsTagReplaced = False - for line in f: - s = line - if (immContentsTagFound == False): - str = s.replace(self.imm_content_no_namespace_spec, self.imm_content_with_namespace_spec) - if len(str) != len(s): - s = str - immContentsTagFound = True - immContentsTagReplaced = True - elif s.find(self.imm_content_element_name) != -1: - immContentsTagFound = True - - str_list.append(s) - - xmlstr = ' '.join(str_list) - - - if Options.schemaFilename != None: - if immContentsTagReplaced == True: - print_info_stderr("Cannot validate inputfile '%s' with schema file because of missing namespace specification in element <imm:IMM-contents>. \nProceeding with processing of modified input data!", filename) - else: - if self.validateXmlFile(filename) != 0: - abort_script("Failed to validate the inputfile %s:", filename) - - - return xml.dom.minidom.parseString(xmlstr) - - # helper function to remove some whitespace which we do not want to have in result of toPrettyXml - # Currently NOT used..... - def openXmlDocumentAndStrip(self, filename): - f = open(filename) + imm_contents_tag_found = False + # imm_contents_tag_replaced = False + for _line in doc: + line = _line + if not imm_contents_tag_found: + string = line.replace(self.imm_content_no_namespace_spec, + self.imm_content_with_namespace_spec) + if len(string) != len(line): + line = string + imm_contents_tag_found = True + # imm_contents_tag_replaced = True + elif line.find(self.imm_content_element_name) != -1: + imm_contents_tag_found = True + + str_list.append(line) + + xml_str = ' '.join(str_list) + + # if Options.schemaFilename is not None: + # if imm_contents_tag_replaced: + # print_info_stderr("Cannot validate input file '%s' with " + # "schema file because of missing namespace " + # "specification in element " + # "<imm:IMM-contents>. \nProceeding with " + # "processing of modified input data!", + # file_name) + # else: + # if self.validate_xml_file(file_name) != 0: + # abort_script("Failed to validate the input file: %s", + # file_name) + return xml.dom.minidom.parseString(xml_str) + + @staticmethod + def open_xml_document_and_strip(file_name): + """ Helper function to remove some whitespace which we do not want to + have in result of to_pretty_xml + *** This method is currently NOT used *** + """ + doc = open(file_name) str_list = [] - for line in f: - #s = line.rstrip().lstrip() - s = line.strip('\t\n') - if len(s) < 5: - trace("short line *%s*", s) - str_list.append(s) - - xmlstr = ' '.join(str_list) - return xml.dom.minidom.parseString(xmlstr) -# end of Class BaseImmDocument + for line in doc: + # string = line.rstrip().lstrip() + string = line.strip('\t\n') + if len(string) < 5: + trace("short line *%s*", string) + str_list.append(string) + + xml_str = ' '.join(str_list) + return xml.dom.minidom.parseString(xml_str) + +# End of BaseImmDocument class def trace(*args): - if BaseOptions.traceOn == True: + """ Print traces to stderr if trace option is enabled """ + if BaseOptions.traceOn: printf_args = [] for i in range(1, len(args)): printf_args.append(args[i]) - - formatStr = "TRACE:\t" + args[0] - print >> sys.stderr, formatStr % tuple(printf_args) + format_str = "TRACE:\t" + args[0] + print (format_str % tuple(printf_args), file=sys.stderr) -def retrieveFilenames(args): - fileList = [] - trace("before glob args:%s", args) - fileList = glob.glob(args[0]) # wildcard expansion for WIN support, however not tested.... - trace("after glob filelist:%s length:%d", fileList, len(fileList)) +def retrieve_file_names(args): + """ Retrieve file names """ + trace("Before glob args:%s", args) + # Wildcard expansion for WIN support, however not tested.... + file_list = glob.glob(args[0]) + trace("After glob file list: %s length: %d", file_list, len(file_list)) - if (len(fileList) < 2 and len(args) >= 1): - fileList = args - - trace("Final filelist:%s length:%d", fileList, len(fileList)) - return fileList + if len(file_list) < 2 and len(args) >= 1: + file_list = args + + trace("Final file list: %s length: %d", file_list, len(file_list)) + return file_list def print_info_stderr(*args): + """ Print info to stderr """ printf_args = [] for i in range(1, len(args)): printf_args.append(args[i]) - - formatStr = args[0] - print >> sys.stderr, formatStr % tuple(printf_args) - + + format_str = args[0] + print (format_str % tuple(printf_args), file=sys.stderr) + def abort_script(*args): + """ Abort the script and print info to stderr """ printf_args = [] for i in range(1, len(args)): printf_args.append(args[i]) - - formatStr = "\nAborting script: " + args[0] - print >> sys.stderr, formatStr % tuple(printf_args) + + format_str = "\nAborting script: " + args[0] + print (format_str % tuple(printf_args), file=sys.stderr) sys.exit(2) + def exit_script(*args): + """ Exit the script and print info to stderr """ printf_args = [] for i in range(1, len(args)): printf_args.append(args[i]) - - formatStr = "\n" + args[0] - print >> sys.stderr, formatStr % tuple(printf_args) - sys.exit(0) -def verifyInputFileReadAcess(filename): - if os.access(filename, os.R_OK) == False: - abort_script("Cannot access input file: %s", filename) + format_str = "\n" + args[0] + print (format_str % tuple(printf_args), file=sys.stderr) + sys.exit(0) + + +def verify_input_file_read_access(file_name): + """ Verify if input file has read access """ + if not os.access(file_name, os.R_OK): + abort_script("Cannot access input file: %s", file_name) # An attempt to fix minidom pretty print functionality -# see http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/ -# however the result is still not nice..... (have replaced it with xmllint formatting) +# see http://ronrothman.com/public/leftbraned/ \ +# xml-dom-minidom-toprettyxml-and-silly-whitespace/ +# However the result is still not nice. +# (have replaced it with xmllint formatting) # This way the method gets replaced, worked when called from main() - # attempt to replace minidom's function with a "hacked" version - # to get decent prettyXml, however it does not look nice anyway.... - #xml.dom.minidom.Element.writexml = fixed_writexml - -def fixed_writexml(self, writer, indent="", addindent="", newl=""): +# Attempt to replace minidom's function with a "hacked" version to get decent +# prettyXml, however it does not look nice anyway. +# xml.dom.minidom.Element.writexml = fixed_writexml +def fixed_writexml(self, writer, indent="", add_indent="", new_line=""): + """ Customized pretty xml print function """ # indent = current indentation - # addindent = indentation to add to higher levels - # newl = newline string - writer.write(indent+"<" + self.tagName) + # add_indent = indentation to add to higher levels + # new_line = newline string + writer.write(indent + "<" + self.tagName) attrs = self._get_attributes() a_names = attrs.keys() @@ -230,12 +267,11 @@ def fixed_writexml(self, writer, indent="", addindent="", newl=""): and self.childNodes[0].nodeType == xml.dom.minidom.Node.TEXT_NODE: writer.write(">") self.childNodes[0].writexml(writer, "", "", "") - writer.write("</%s>%s" % (self.tagName, newl)) + writer.write("</%s>%s" % (self.tagName, new_line)) return - writer.write(">%s"%(newl)) + writer.write(">%s" % new_line) for node in self.childNodes: - node.writexml(writer,indent+addindent,addindent,newl) - writer.write("%s</%s>%s" % (indent,self.tagName,newl)) + node.writexml(writer, indent + add_indent, add_indent, new_line) + writer.write("%s</%s>%s" % (indent, self.tagName, new_line)) else: - writer.write("/>%s"%(newl)) - + writer.write("/>%s" % new_line) diff --git a/src/imm/tools/immxml-merge b/src/imm/tools/immxml-merge index e73955f..7489da3 100755 --- a/src/imm/tools/immxml-merge +++ b/src/imm/tools/immxml-merge @@ -1,30 +1,36 @@ #! /usr/bin/env python - -''' - -*- OpenSAF -*- - - (C) Copyright 2009 The OpenSAF Foundation - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed - under the GNU Lesser General Public License Version 2.1, February 1999. - The complete license can be accessed from the following location: - http://opensource.org/licenses/lgpl-license.php - See the Copying file included with the OpenSAF distribution for full - licensing terms. - - Author(s): Ericsson AB -''' - -import re, sys, getopt, shutil +# -*- OpenSAF -*- +# +# (C) Copyright 2009 The OpenSAF Foundation +# (C) Copyright Ericsson AB 2015, 2016, 2017. All rights reserved. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed +# under the GNU Lesser General Public License Version 2.1, February 1999. +# The complete license can be accessed from the following location: +# http://opensource.org/licenses/lgpl-license.php +# See the Copying file included with the OpenSAF distribution for full +# licensing terms. +# +# Author(s): Ericsson AB +# +# pylint: disable=invalid-name,unused-argument +""" immxml-merge tool """ +from __future__ import print_function +import os +import sys +import re +import getopt +import shutil import xml.dom.minidom -from baseimm import * # Base class and functions for Merge and validation utils -from subprocess import call from datetime import datetime - +from baseimm import BaseOptions, BaseImmDocument, trace, retrieve_file_names, \ + print_info_stderr, abort_script, verify_input_file_read_access + + class Options(BaseOptions): - #traceOn = False + """ immxml-merge options""" keepCommentElements = False ignoreVariants = False ignoreMissingClass = False @@ -38,14 +44,22 @@ class Options(BaseOptions): schemaFilename = None isXmlLintFound = True disableParentDnValidation = False - + @staticmethod - def printOptionSettings(): - return BaseOptions.printOptionSettings()+' keepCommentElements: %s \n ignoreVariants: %s \n grep_class: %s\n grep_dn: %s\n negateGrep_dn: %s\n negateGrep_class: %s \n sort:%s\n outputFilename: %s\n schemaFilename: %s\n' % (Options.keepCommentElements, Options.ignoreVariants, Options.grep_class, Options.grep_dn , Options.negateGrep_dn, Options.negateGrep_class, Options.sort, Options.outputFilename, Options.schemaFilename) + def print_option_settings(): + return BaseOptions.print_option_settings() + \ + ' keepCommentElements: %s\n ignoreVariants: %s\n ' \ + 'grep_class: %s\n grep_dn: %s\n negateGrep_dn: %s\n ' \ + 'negateGrep_class: %s\n sort:%s\n outputFilename: %s\n ' \ + 'schemaFilename: %s\n' % \ + (Options.keepCommentElements, Options.ignoreVariants, + Options.grep_class, Options.grep_dn, Options.negateGrep_dn, + Options.negateGrep_class, Options.sort, Options.outputFilename, + Options.schemaFilename) - class MergedImmDocument(BaseImmDocument): + """ This class contains methods to process the merging of IMM XML files """ def __init__(self): self.isResultDocInitialized = False self.document = xml.dom.minidom.Document() @@ -57,462 +71,540 @@ class MergedImmDocument(BaseImmDocument): self.firstSourceDoc = None self.objectList = [] self.objectDnNameDict = {} - self.classes_parsed=0 - self.objects_parsed=0 - + self.classes_parsed = 0 + self.objects_parsed = 0 def initialize(self): + """ Initialize matching patterns """ if Options.grep_class: - trace("matching classes with pattern:%s", Options.grep_class) + trace("Matching classes with pattern: %s", Options.grep_class) self.regexpObj = re.compile(Options.grep_class) if Options.grep_dn: - trace("grep_dn matching dn with pattern:%s", Options.grep_dn) + trace("grep_dn matching dn with pattern: %s", Options.grep_dn) self.dn_regexpObj = re.compile(Options.grep_dn) - - def initResultDocument(self, doc): + + def init_result_document(self, doc): + """ Initialize merge result document """ if self.isResultDocInitialized: return - + trace("Copy elements from first source document file") for element in doc.childNodes: - trace("nodeType:*%d*" ,element.nodeType) - trace("nodeValue:*%s*" , element.nodeValue) - trace("nodeName:*%s*" , element.nodeName) - trace("parent:*%s*", element.parentNode.nodeName) + trace("nodeType: *%d*", element.nodeType) + trace("nodeValue: *%s*", element.nodeValue) + trace("nodeName: *%s*", element.nodeName) + trace("parent: *%s*", element.parentNode.nodeName) clone = element.cloneNode(False) self.document.appendChild(clone) - if (element.nodeName == self.imm_content_element_name): - self.imm_content_element = clone - break - - if self.imm_content_element == None: - abort_script("did not find <imm:IMM-contents element in first source file") - - self.firstSourceDoc = doc + if element.nodeName == self.imm_content_element_name: + self.imm_content_element = clone + break + + if self.imm_content_element is None: + abort_script("Did not find <imm:IMM-contents element in first " + "source file") + + self.firstSourceDoc = doc self.isResultDocInitialized = True trace("Done copying elements from first source document.") - - - def addClass(self, classElement, otherNodes): - className = classElement.getAttribute("name") - self.classes_parsed=self.classes_parsed+1 - trace("className: %s nodeType:%d nodeName:%s", className, classElement.nodeType, classElement.nodeName) - - if self.regexpObj != None: - trace ("Check if class %s match pattern %s", className, Options.grep_class) - if re.search(self.regexpObj, className) == None: - if Options.negateGrep_class == False: - trace ("Class %s does not match pattern %s", className, Options.grep_class) + + def add_class(self, class_element, other_nodes): + """ Add classes matching pattern """ + class_name = class_element.getAttribute("name") + self.classes_parsed = self.classes_parsed + 1 + trace("className: %s nodeType:%d nodeName:%s", class_name, + class_element.nodeType, class_element.nodeName) + + if self.regexpObj is not None: + trace("Check if class %s match pattern %s", class_name, + Options.grep_class) + if re.search(self.regexpObj, class_name) is None: + if not Options.negateGrep_class: + trace("Class %s does not match pattern %s", class_name, + Options.grep_class) return elif Options.negateGrep_class: - trace ("Class %s does not match negated pattern %s", className, Options.grep_class) + trace("Class %s does not match negated pattern %s", class_name, + Options.grep_class) return - trace("match ok :%s", className); - - # check if class exist in dictionary map - if className in self.classDict: - # check if class clash is "acceptable" + + trace("match ok :%s", class_name) + + # Check if class exists in dictionary map + if class_name in self.classDict: + # Check if class clash is "acceptable" # i.e. should we bail out or not?? - if self.checkClassClashAcceptable(classElement, className): - trace("Ignore class definition duplicate %s. (Class is compatible)", className) - else: - self.classDict[className] = classElement - # add to ordered list - self.classList.append((className, (otherNodes, classElement))) - - - def addObject(self, objectElement, otherNodes): - className = objectElement.getAttribute("class") - trace("DUMPING Object with className: %s \n %s", className, objectElement.toxml()) - self.objects_parsed=self.objects_parsed+1 - - objectDnElement = None - ## find "dn" childnode - for childNode in objectElement.childNodes: - if childNode.nodeName == "dn": - objectDnElement = childNode + if self.check_class_clash_acceptable(class_element, class_name): + trace("Ignore class definition duplicate %s. " + "(Class is compatible)", class_name) + else: + self.classDict[class_name] = class_element + # Add to ordered list + self.classList.append((class_name, (other_nodes, class_element))) + + def add_object(self, object_element, other_nodes): + """ Add objects matching pattern """ + class_name = object_element.getAttribute("class") + trace("DUMPING Object with className: %s \n %s", class_name, + object_element.toxml()) + self.objects_parsed = self.objects_parsed + 1 + + object_dn_element = None + # Find "dn" child node + for child_node in object_element.childNodes: + if child_node.nodeName == "dn": + object_dn_element = child_node break - - if objectDnElement == None or className == None or objectDnElement.firstChild == None: - abort_script("Failed to find classname or the dn childnode in object:%s. " ,objectElement.toxml()) - - - # objectDnName = objectDnElement.nodeValue - ### NOTE dn name should be the nodeValue of objectDnName... - ## however minidom probably gets confused by the equal sign in dn value - # so it ends up adding a text node as a child...... - objectDnName = objectDnElement.firstChild.nodeValue - trace("objectDn key: %s", objectDnName) - - trace("objectDnElement: %s", objectDnElement.toxml()) - - # classname must exist in dictionary map (unless --ignore-noclass is specified) - if className not in self.classDict: - if Options.ignoreMissingClass == False: - if Options.grep_class != None: - trace("ignoring object with class not matching pattern classname:%s", className) + + if object_dn_element is None or class_name is None \ + or object_dn_element.firstChild is None: + abort_script("Failed to find class name or the dn child node in " + "object:%s. ", object_element.toxml()) + + # object_dn_name = object_dn_element.nodeValue + # NOTE: dn name should be the nodeValue of object_dn_name + # However minidom probably gets confused by the equal sign in dn value + # so it ends up adding a text node as a child. + object_dn_name = object_dn_element.firstChild.nodeValue + trace("objectDn key: %s", object_dn_name) + trace("objectDnElement: %s", object_dn_element.toxml()) + + # className must exist in dictionary map + # (unless --ignore-noclass is specified) + if class_name not in self.classDict: + if not Options.ignoreMissingClass: + if Options.grep_class is not None: + trace("Ignoring object with class not matching pattern " + "className:%s", class_name) return else: - abort_script("failed to find class referred in: \n %s", objectElement.toxml()) + abort_script("Failed to find class referred in: \n %s", + object_element.toxml()) trace("zzz") else: - trace("Continue processing object with missing class (--ignore-missing-class)") - - if self.dn_regexpObj != None: - trace ("grep_dn check if object dn %s match pattern %s", objectDnName, Options.grep_dn) - if re.search(self.dn_regexpObj, objectDnName) == None: - if Options.negateGrep_dn == False: - trace ("Object dn %s does not match pattern %s", objectDnName, Options.grep_dn) + trace("Continue processing object with missing class " + "(--ignore-missing-class)") + + if self.dn_regexpObj is not None: + trace("Grep_dn check if object dn %s matches pattern %s", + object_dn_name, Options.grep_dn) + if re.search(self.dn_regexpObj, object_dn_name) is None: + if not Options.negateGrep_dn: + trace("Object dn %s does not match pattern %s", + object_dn_name, Options.grep_dn) return elif Options.negateGrep_dn: - trace ("Object dn %s does not match negated pattern %s", objectDnName, Options.grep_dn) + trace("Object dn %s does not match negated pattern %s", + object_dn_name, Options.grep_dn) return - trace("match ok :%s", objectDnName); - - if objectDnName in self.objectDnNameDict: - if self.checkObjectClashAcceptable(objectDnName): - trace("ignore duplicate object: %s with dn: %s", className, objectDnName) + + trace("match ok :%s", object_dn_name) + + if object_dn_name in self.objectDnNameDict: + if self.check_object_clash_acceptable(object_dn_name): + trace("Ignore duplicate object: %s with dn: %s", class_name, + object_dn_name) return # TODO add code to check if this is valid clash - #self.validateAndStoreDn(objectDnName, objectDnElement) - self.objectDnNameDict[objectDnName] = objectDnElement - - # add to the complete object to ordered list (keyed by classname for sorting) - #self.objectList.append((className+objectDnName, objectElement)) - objectListKey = objectDnName # anything goes, its not used... - if Options.sort == True: - objectListKeyList = [self.getDnSortPrefix(objectDnName)] - objectListKeyList.append(className) - objectListKeyList.append(objectDnName) - objectListKey = ''.join(objectListKeyList) - #objectListKey = self.getDnSortPrefix(objectDnName)+className - trace("Object sort order key: %s", objectListKey) - - self.objectList.append((objectListKey, (otherNodes, objectElement))) - - def getDnSortPrefix(self, objectDnName): - sortKeyList = [ "A" ] + # self.validateAndStoreDn(object_dn_name, object_dn_element) + self.objectDnNameDict[object_dn_name] = object_dn_element + + # Add the complete object to ordered list + # (keyed by class_name for sorting) + # self.objectList.append((class_name + object_dn_name, object_element)) + object_list_key = object_dn_name # anything goes, it's not used... + if Options.sort: + object_list_key_list = [self.get_dn_sort_prefix(object_dn_name), + class_name, object_dn_name] + object_list_key = ''.join(object_list_key_list) + # objectListKey = self.get_dn_sort_prefix(objectDnName)+className + trace("Object sort order key: %s", object_list_key) + + self.objectList.append((object_list_key, + (other_nodes, object_element))) + + @staticmethod + def get_dn_sort_prefix(object_dn_name): + """ Get dn sort prefix """ + sort_key_list = ["A"] depth = 1 - maxDepth = 10 - # search for all unescaped comma (if any) - for i in range(0, len(objectDnName)): - if objectDnName[i] == ',': - if i > 1 and objectDnName[i-1] != '\\': - sortKeyList.append(",A") - depth = depth +1 - - if depth >= maxDepth: - excStr = "Object reference depth is higher than expected (maximum %d)" % maxDepth - raise Exception(excStr) - - for i in range(depth+1, maxDepth): - sortKeyList.append(" ") - - #sortKeyList.append("#") - key = ''.join(sortKeyList) - trace("getDnSortPrefix: %s", key) + max_depth = 10 + # Search for all unescaped commas (if any) + for i in range(0, len(object_dn_name)): + if object_dn_name[i] == ',': + if i > 1 and object_dn_name[i - 1] != '\\': + sort_key_list.append(",A") + depth = depth + 1 + + if depth >= max_depth: + exc_str = "Object reference depth is higher than expected " \ + "(maximum %d)" % max_depth + raise Exception(exc_str) + + for i in range(depth + 1, max_depth): + sort_key_list.append(" ") + + # sort_key_list.append("#") + key = ''.join(sort_key_list) + trace("get_dn_sort_prefix: %s", key) return key - - - def checkClassClashAcceptable(self, classElement, className): - # previousClassValue = self.classDict.get(className) - # TODO deep verify class Element clash is acceptable or not ????? - - if Options.ignoreVariants == False: - abort_script("failed to merge input files class with name:%s exists in multiple input files (use --ignore-variants to override)", className) - - return True - - def checkObjectClashAcceptable(self, objectDn): - if Options.ignoreVariants == False: - abort_script("failed to merge input files object with dn :%s exists in multiple input files (use --ignore-variants to override)", objectDn) - return True - - - def processInputfile(self, filename): + + @staticmethod + def check_class_clash_acceptable(class_element, class_name): + """ Check if class clash is acceptable """ + # previous_class_value = self.classDict.get(class_name) + # TODO deep verify class element clash is acceptable or not ????? + + if not Options.ignoreVariants: + abort_script("Failed to merge input files class with name: %s " + "exists in multiple input files " + "(use --ignore-variants to override)", class_name) + + return True + + @staticmethod + def check_object_clash_acceptable(object_dn): + """ Check if object clash is acceptable """ + if not Options.ignoreVariants: + abort_script("Failed to merge input files object with dn: %s " + "exists in multiple input files " + "(use --ignore-variants to override)", object_dn) + + return True + + def process_input_file(self, file_name): + """ Process input file """ trace("") - trace("processInputfile in file:%s", filename) - - if Options.isXmlLintFound and Options.schemaFilename != None: - if self.validateXmlFileWithSchema(filename, Options.schemaFilename) != 0: - abort_script("failed to validate input file %s:", filename) + trace("process_input_file in file: %s", file_name) + + if Options.isXmlLintFound and Options.schemaFilename is not None: + if self.validate_xml_file_with_schema(file_name, + Options.schemaFilename) != 0: + abort_script("Failed to validate input file %s:", file_name) else: - self.verifyInputXmlDocumentFileIsParsable(filename) - - doc = xml.dom.minidom.parse(filename) - #doc = self.openXmlDocumentFileAndCheckNamespace(filename) - - if self.isResultDocInitialized == False: - self.initResultDocument(doc) - - - ## Fast forward to imm:contents element + self.verify_input_xml_document_file_is_parsable(file_name) + + doc = xml.dom.minidom.parse(file_name) + # doc = self.open_xml_document_file_and_check_namespace(file_name) + + if not self.isResultDocInitialized: + self.init_result_document(doc) + + # Fast forward to imm:contents element for element in doc.childNodes: - otherNodes = [] - if (element.nodeName == self.imm_content_element_name): - for childElement in element.childNodes: - trace("imm:contents loop..... Nodename:%s NodeValue%s", childElement.nodeName, childElement.nodeValue) - if (childElement.nodeName == "class"): - self.addClass(childElement, otherNodes) - #for otherNode in : - # trace("OtherNode: %s", otherNode.toxml()) - otherNodes = [] - elif (childElement.nodeName == "object"): - self.addObject(childElement, otherNodes) - #for otherNode in otherNodes: - # trace("OtherNode: %s", otherNode.toxml()) - otherNodes = [] + other_nodes = [] + if element.nodeName == self.imm_content_element_name: + for child_element in element.childNodes: + trace("imm:contents loop.....nodeName: %s nodeValue: %s", + child_element.nodeName, child_element.nodeValue) + if child_element.nodeName == "class": + self.add_class(child_element, other_nodes) + # for other_node in other_nodes: + # trace("otherNode: %s", other_node.toxml()) + other_nodes = [] + elif child_element.nodeName == "object": + self.add_object(child_element, other_nodes) + # for other_node in other_nodes: + # trace("otherNode: %s", other_node.toxml()) + other_nodes = [] else: # probably text nodes....ignore if whitespace only - childElementStr = childElement.nodeValue.lstrip().rstrip() - #if len(childElementStr) > 1: - #otherNodes.append(childElement) - otherNodes.append(childElement) + # child_element_str = \ + # child_element.nodeValue.lstrip().rstrip() + # if len(child_element_str) > 1: + # other_nodes.append(child_element) + other_nodes.append(child_element) - return 0 - - def saveResult(self): + def save_result(self): + """ Save merge result to file or print to stdout """ trace("") - - #if len(self.classList) < 1 and Options.grep_class != None: - # exit_script("No classes matches pattern %s specified with --grep-class. No document is saved. Exiting!", Options.grep_class) - #if len(self.objectList) < 1 and Options.grep_dn != None: - # exit_script("No objects matches pattern %s specified with --grep-dn. No document is saved. Exiting!", Options.grep_dn) + # if len(self.classList) < 1 and Options.grep_class is not None: + # exit_script("No class matches pattern %s specified with + # --grep-class. No document is saved. Exiting!", Options.grep_class) - # Use a temp file when output file is not specified. When this script finishes it prints the file to stdout - # and removes the file + # if len(self.objectList) < 1 and Options.grep_dn is not None: + # exit_script("No object matches pattern %s specified with + # --grep-dn. No document is saved. Exiting!", Options.grep_dn) + + # Use a temp file when output file is not specified. + # When this script finishes it prints the file to stdout and removes + # the file if Options.stdout: Options.outputFilename = "/tmp/merge_result.xml" - if Options.sort == True: - trace("sorting the classes & objects in resulting xml document") + if Options.sort: + trace("Sorting the classes & objects in resulting xml document") self.classList.sort() self.objectList.sort() - trace("Number of classes in resulting xml document:%d", len(self.classList)) - - # iterate over all objects again to validate again when all objects are parsed from inputfiles - #if Options.disableParentDnValidation==False: - # self.postValidateObjectList() - #### I think there should not be imm validation functionality in merge tool (use validate_immfile instead) - - print_info_stderr("encoding in first source xml document:%s", self.firstSourceDoc.encoding) - tmpOutputFilename = Options.outputFilename+".tmp" - file_object = open(tmpOutputFilename, "w") - - if self.firstSourceDoc.encoding is not None and \ - self.firstSourceDoc.encoding.lower() == "utf-8": - encoding = "utf-8" - heading = "<?xml version=\"1.0\" encoding=\"utf-8\"?>" + trace("Number of classes in resulting xml document: %d", + len(self.classList)) + + # Iterate over all objects again to validate again when all objects are + # parsed from input files + # if not Options.disableParentDnValidation: + # self.postValidateObjectList() + # I think there should not be imm validation functionality in merge + # tool (use validate_immfile instead) + + print_info_stderr("encoding in first source xml document: %s", + self.firstSourceDoc.encoding) + tmp_output_file_name = Options.outputFilename+".tmp" + file_object = open(tmp_output_file_name, "w") + + if self.firstSourceDoc.encoding is not None \ + and self.firstSourceDoc.encoding.lower() == "utf-8": + encoding = "utf-8" + heading = "<?xml version=\"1.0\" encoding=\"utf-8\"?>" else: - encoding = None - heading = "<?xml version=\"1.0\"?>" + encoding = None + heading = "<?xml version=\"1.0\"?>" file_object.write(heading) - file_object.write(self.imm_content_element.toxml(encoding).replace("/>",">")+"\n") + file_object.write( + self.imm_content_element.toxml(encoding).replace("/>", ">")+"\n") for class_element_tuple in self.classList: - if (Options.keepCommentElements): + if Options.keepCommentElements: for textNode in class_element_tuple[1][0]: - file_object.write(textNode.toxml(encoding)+"\n") - file_object.write(class_element_tuple[1][1].toxml(encoding)+"\n") + file_object.write(textNode.toxml(encoding) + "\n") + file_object.write(class_element_tuple[1][1].toxml(encoding) + "\n") for object_element_tuple in self.objectList: - if (Options.keepCommentElements): + if Options.keepCommentElements: for textNode in object_element_tuple[1][0]: - file_object.write(textNode.toxml(encoding)+"\n") - file_object.write(object_element_tuple[1][1].toxml(encoding)+"\n") + file_object.write(textNode.toxml(encoding) + "\n") + file_object.write( + object_element_tuple[1][1].toxml(encoding) + "\n") file_object.write("</imm:IMM-contents>") file_object.close() - trace("Stored resulting xml document in tmp file: %s", tmpOutputFilename) + trace("Stored resulting xml document in tmp file: %s", + tmp_output_file_name) if Options.isXmlLintFound: - #command = "/bin/sh -c 'XMLLINT_INDENT=\" \" ; export XMLLINT_INDENT; /usr/bin/xmllint --format "+tmpOutputFilename+" --output "+Options.outputFilename +"'" - if self.formatXmlFileWithXmlLint(tmpOutputFilename, Options.outputFilename) != 0: - abort_script("failed to validate input file %s:", tmpOutputFilename) - trace("delete the tmp file: %s", tmpOutputFilename) - os.remove(tmpOutputFilename) + # command = "/bin/sh -c 'XMLLINT_INDENT=\" \"; + # export XMLLINT_INDENT; /usr/bin/xmllint --format " + + # tmp_output_file_name + " --output " + Options.outputFilename + + # "'" + if self.format_xml_file_with_xmllint(tmp_output_file_name, + Options.outputFilename) != 0: + abort_script("Failed to validate input file: %s", + tmp_output_file_name) + trace("Delete the tmp file: %s", tmp_output_file_name) + os.remove(tmp_output_file_name) else: - # at least we should move file to its real name - trace("shutil.move(%s, %s)", tmpOutputFilename, Options.outputFilename) - shutil.move(tmpOutputFilename, Options.outputFilename) + # At least we should move the file to its real name + trace("shutil.move(%s, %s)", tmp_output_file_name, + Options.outputFilename) + shutil.move(tmp_output_file_name, Options.outputFilename) + + trace("Number of classes parsed: %d stored: %d", self.classes_parsed, + len(self.classList)) + trace("Number of objects parsed: %d stored: %d", self.objects_parsed, + len(self.objectList)) + + diff_classes = self.classes_parsed - len(self.classList) + diff_objects = self.objects_parsed - len(self.objectList) + print_info_stderr("Note! Merge ignored %d classes " + "(parsed: %d stored: %d)", diff_classes, + self.classes_parsed, len(self.classList)) + print_info_stderr("Note! Merge ignored %d objects " + "(parsed: %d stored: %d)", diff_objects, + self.objects_parsed, len(self.objectList)) + + trace("Stored formatted xml document in file: %s", + Options.outputFilename) + + if Options.isXmlLintFound and Options.schemaFilename is not None: + self.validate_xml_file_with_schema(Options.outputFilename, + Options.schemaFilename) + if Options.stdout: + self.print_to_stdout_and_remove_tmp_file(Options.outputFilename) - trace("Number of classes parsed:%d stored:%d", self.classes_parsed, len(self.classList)) - trace("Number of objects parsed:%d stored:%d", self.objects_parsed, len(self.objectList)) +# End of MergedImmDocument class - diff_classes = self.classes_parsed-len(self.classList) - diff_objects = self.objects_parsed-len(self.objectList) - print_info_stderr("Note! Merge ignored %d classes (parsed:%d stored:%d)", diff_classes, self.classes_parsed, len(self.classList) ) - print_info_stderr("Note! Merge ignored %d objects (parsed:%d stored:%d)", diff_objects, self.objects_parsed, len(self.objectList) ) - - trace("Stored formatted xml document in file: %s", Options.outputFilename) - - if Options.isXmlLintFound and Options.schemaFilename != None: - self.validateXmlFileWithSchema(Options.outputFilename, Options.schemaFilename) +def print_usage(): + """ Print usage of immxml-merge tool """ + print ("usage: immxml-merge [options] filename[s]") + print (""" + -o, --output specified output file + (If option is omitted stdout is used) - if Options.stdout: - self.printToStdoutAndRemoveTmpFile(Options.outputFilename) - -#### end of class ResultImmDocument - - - -def printUsage(): - print "usage: immxml-merge [options] filename[s]" - print """ - -o, --output specified outputfile - (if option is omitted stdout is used) - --grep-class PATTERN pattern match the class names in source file(s) - (This option also sets disableParentDnValidation) - At end of processing a summary of processed/ignored - classes/objects is listed. - --grep-v-class PATTERN negated pattern matching. Similar to "grep -v PATTERN" - --grep-dn PATTERN pattern match the object dn names in source file(s) - (This option also sets disableParentDnValidation) - At end of processing a summary of processed/ignored - classes/objects is listed. - --grep-v-dn PATTERN negated pattern matching. Similar to "grep -v PATTERN" - --ignore-variants when merge tool finds several definitions of same - class or object the default behaviour is to abort the - merge processing and print out which class/object is - duplicated. With this option it is possible to - continue merge, keeping the first instance of a class - or object definition and ignore the other. - At end of processing a summary of processed/ignored - classes/objects is listed. - --ignore-missing-class do not require class definitions referred by an object - to exist in source file(s) (Is required by default) - --keepCommentElements keep embedded comment elements - (experimental support: associates comment elements - with subsequent class or object element which may be - correct association....) - --schema validate inputfiles and output(file) with the supplied - xsd schema file - -s, --sort sort the merged class and object names - -t, --trace print trace information to stderr - -v, --version print version information and exit - -h, --help display this help and exit - - See http://devel.opensaf.org/ for information and updates. -""" - -def printVersion(): - print "immxml-merge version 0.5.1" - - - - -def main(argv): + --grep-class PATTERN pattern match the class names in source file(s) + (This option also sets disableParentDnValidation) + At end of processing a summary of processed/ignored + classes/objects is listed. + + --grep-v-class PATTERN negated pattern matching + Similar to "grep -v PATTERN" + + --grep-dn PATTERN pattern match the object dn names in source file(s) + (This option also sets disableParentDnValidation) + At end of processing a summary of processed/ignored + classes/objects is listed. + + --grep-v-dn PATTERN negated pattern matching + Similar to "grep -v PATTERN" + + --ignore-variants when merge tool finds several definitions of same + class or object the default behaviour is to abort + the merge processing and print out which + class/object is duplicated. With this option it is + possible to continue merge, keeping the first + instance of a class or object definition and ignore + the other. At end of processing a summary of + processed/ignored classes/objects is listed. + + --ignore-missing-class do not require class definitions referred by an + object to exist in source file(s) + (Is required by default) + + --keepCommentElements keep embedded comment elements + (experimental support: associates comment elements + with subsequent class or object element which may + be correct association....) + + --schema validate input files and output(file) with the + supplied xsd schema file + + -s, --sort sort the merged class and object names + + -t, --trace print trace information to stderr + + -v, --version print version information and exit + + -h, --help display this help and exit + + See http://devel.opensaf.org/ for information and updates. + """) + + +def print_version(): + """ Print version of immxml-merge tool """ + print ("immxml-merge version 0.5.1") + + +def main(argv): + """ Main program """ try: - opts, args = getopt.getopt(argv, "thvso:", ["trace", "keep-comment-elements", "ignore-variants", "ignore-missing-class", "help", "version", "sort", "grep-class=", "grep-v-class=", "grep-dn=", "grep-v-dn=", "output=", "schema="]) - except getopt.GetoptError, err: - # print help information and exit: + opts, args = getopt.getopt(argv, "thvso:", + ["trace", "keep-comment-elements", + "ignore-variants", "ignore-missing-class", + "help", "version", "sort", "grep-class=", + "grep-v-class=", "grep-dn=", "grep-v-dn=", + "output=", "schema="]) + except getopt.GetoptError as err: + # Print help information and exit: print_info_stderr("%s", str(err)) - printUsage() + print_usage() sys.exit(2) - - foundGrep_dn = False - foundGrep_class = False - for o, v in opts: - if o in ["-t", "--trace"]: + found_grep_dn = False + found_grep_class = False + + for opt, value in opts: + if opt in ["-t", "--trace"]: BaseOptions.traceOn = True - if o in ["--keep-comment-elements"]: + if opt == "--keep-comment-elements": Options.keepCommentElements = True - if o in ["--ignore-variants"]: + if opt == "--ignore-variants": Options.ignoreVariants = True - if o in ["--ignore-missing-class"]: - Options.ignoreMissingClass = True - if o in ["--grep-class"]: - if foundGrep_class: - abort_script("Only one --grep-v-class or --grep-class option may be specified") - foundGrep_class = True - Options.grep_class = v - Options.disableParentDnValidation = True - if o in ["--grep-v-class"]: - if foundGrep_class: - abort_script("Only one --grep-v-class or --grep-class option may be specified") - foundGrep_class = True + if opt == "--ignore-missing-class": + Options.ignoreMissingClass = True + if opt == "--grep-class": + if found_grep_class: + abort_script("Only one --grep-v-class or --grep-class option " + "may be specified") + found_grep_class = True + Options.grep_class = value + Options.disableParentDnValidation = True + if opt == "--grep-v-class": + if found_grep_class: + abort_script("Only one --grep-v-class or --grep-class option " + "may be specified") + found_grep_class = True Options.negateGrep_class = True - Options.grep_class = v - Options.disableParentDnValidation = True - if o in ["--grep-v-dn"]: - if foundGrep_dn: - abort_script("Only one --grep-v-dn or --grep-dn option may be specified") - foundGrep_dn = True + Options.grep_class = value + Options.disableParentDnValidation = True + if opt == "--grep-v-dn": + if found_grep_dn: + abort_script("Only one --grep-v-dn or --grep-dn option may be " + "specified") + found_grep_dn = True Options.negateGrep_dn = True - Options.grep_dn = v - Options.disableParentDnValidation = True - if o in ["--grep-dn"]: - if foundGrep_dn: - abort_script("Only one --grep-v-dn or --grep-dn option may be specified") - foundGrep_dn = True - Options.grep_dn = v - Options.disableParentDnValidation = True - if o in ("-o", "--output"): + Options.grep_dn = value + Options.disableParentDnValidation = True + if opt == "--grep-dn": + if found_grep_dn: + abort_script("Only one --grep-v-dn or --grep-dn option may be " + "specified") + found_grep_dn = True + Options.grep_dn = value + Options.disableParentDnValidation = True + if opt in ["-o", "--output"]: Options.stdout = False - Options.outputFilename = v - if o in ("--schema"): - Options.schemaFilename = v - if o in ["-s", "--sort"]: + Options.outputFilename = value + if opt == "--schema": + Options.schemaFilename = value + if opt in ["-s", "--sort"]: Options.sort = True - - if o in ["-v", "--version"]: - printVersion() + if opt in ["-v", "--version"]: + print_version() sys.exit(0) - elif o in ["-h", "--help"]: - printUsage() + elif opt in ["-h", "--help"]: + print_usage() sys.exit(0) - - # cannot trace these until -t, Options.traceOn is effective (or not) - trace("opts:%s", opts) - trace("args:%s", args) - trace("sys.path:%s", sys.path) - - if len(args) == 0: - # print "Stdin processing not yet supported! (if ever!)" - printUsage() + + # Cannot trace these until -t, Options.traceOn is effective (or not) + trace("opts: %s", opts) + trace("args: %s", args) + trace("sys.path: %s", sys.path) + + if not args: + # print ("stdin processing not yet supported! (if ever!)") + print_usage() sys.exit(2) - - - trace("Option object: \n %s", Options.printOptionSettings()) - - - if os.path.exists('/usr/bin/xmllint') == False: - if (Options.schemaFilename == None ): - # It is possible to continue without xmllint (limited w.r.t formatting) + + trace("Option object:\n %s", Options.print_option_settings()) + + if not os.path.exists('/usr/bin/xmllint'): + if Options.schemaFilename is None: + # It is possible to continue without xmllint + # (limited w.r.t formatting) print_info_stderr("") - print_info_stderr("Cannot find the linux command /usr/bin/xmllint which is required for formatting!") - print_info_stderr("Script continues but the result file may for example lack linefeed characters.") + print_info_stderr("Cannot find the linux command /usr/bin/xmllint " + "which is required for formatting!") + print_info_stderr("Script continues but the result file may for " + "example lack linefeed characters.") print_info_stderr("") Options.isXmlLintFound = False else: - abort_script("Cannot find the required linux command /usr/bin/xmllint. --schema option requires xmllint, Exiting!") - - - - fileList = retrieveFilenames(args) - - trace("starting to process files::\n") - ## Create an Object to store classes and objects during process of input files - mergedDoc = MergedImmDocument() - mergedDoc.initialize(); - - for fileName in fileList: - verifyInputFileReadAcess(fileName) - mergedDoc.processInputfile(fileName) - trace("Done with file:%s", fileName) - - ## store the resulting document with collected class and objects elements - mergedDoc.saveResult() - print_info_stderr("{0} Successfully merged input files!".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3])) - return 0 - + abort_script("Cannot find the required linux command " + "/usr/bin/xmllint. --schema option requires xmllint, " + "Exiting!") + + file_list = retrieve_file_names(args) + trace("Starting to process files::\n") + # Create an Object to store classes and objects during process of + # input files + merged_doc = MergedImmDocument() + merged_doc.initialize() + + for file_name in file_list: + verify_input_file_read_access(file_name) + merged_doc.process_input_file(file_name) + trace("Done with file: %s", file_name) + + # Store the resulting document with collected class and object elements + merged_doc.save_result() + print_info_stderr("{0} Successfully merged input files!".format( + datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3])) + return 0 if __name__ == "__main__": diff --git a/src/imm/tools/immxml-validate b/src/imm/tools/immxml-validate index 89ce54e..e36ece0 100755 --- a/src/imm/tools/immxml-validate +++ b/src/imm/tools/immxml-validate @@ -1,50 +1,58 @@ #! /usr/bin/env python - -''' - -*- OpenSAF -*- - - (C) Copyright 2009 The OpenSAF Foundation - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed - under the GNU Lesser General Public License Version 2.1, February 1999. - The complete license can be accessed from the following location: - http://opensource.org/licenses/lgpl-license.php - See the Copying file included with the OpenSAF distribution for full - licensing terms. - - Author(s): Ericsson AB -''' - - -import re, os, sys, getopt, shutil -from baseimm import * # Base class and functions for Merge and validation utils +# -*- OpenSAF -*- +# +# (C) Copyright 2009 The OpenSAF Foundation +# (C) Copyright Ericsson AB 2017. All rights reserved. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed +# under the GNU Lesser General Public License Version 2.1, February 1999. +# The complete license can be accessed from the following location: +# http://opensource.org/licenses/lgpl-license.php +# See the Copying file included with the OpenSAF distribution for full +# licensing terms. +# +# Author(s): Ericsson AB +# +# pylint: disable=invalid-name,unused-argument +""" immxml-validate tool """ +from __future__ import print_function +import os +import sys +import getopt import xml.dom.minidom -from subprocess import call +from baseimm import BaseOptions, BaseImmDocument, trace, retrieve_file_names, \ + print_info_stderr, abort_script, verify_input_file_read_access - class Options(BaseOptions): - #traceOn = False + """ immxml-validate options """ schemaFilename = None isXmlLintFound = True ignoreAttributeRefs = False ignoreRdnAssociationRefs = False - + @staticmethod - def printOptionSettings(): - return BaseOptions.printOptionSettings()+' schemaFilename: %s\n' % (Options.schemaFilename) + def print_option_settings(): + return BaseOptions.print_option_settings() + ' schemaFilename: %s\n' \ + % Options.schemaFilename + class AbortFileException(Exception): + """ Exception to raise when aborting file """ def __init__(self, value): + Exception.__init__(self) self.value = value + def __str__(self): - return repr(self.value) + return repr(self.value) + class ImmDocument(BaseImmDocument): - + """ This class contains methods to validate an IMM XML file """ def __init__(self): + """ Class constructor """ self.imm_content_element = None self.classList = [] self.classDict = {} @@ -52,406 +60,464 @@ class ImmDocument(BaseImmDocument): self.objectDnNameDict = {} self.validateFailed = False - def initialize(self): + @staticmethod + def initialize(): + """ Initialize the validation process """ trace("initialize()") - - def addClass(self, classElement, otherNodes): - className = classElement.getAttribute("name") - trace("className: %s nodeType:%d nodeName:%s", className, classElement.nodeType, classElement.nodeName) - - # check if class exist in dictionary map - if className in self.classDict: - # check if class clash is "acceptable" + + @staticmethod + def check_class_clash_acceptable(class_element, class_name): + """ Check if class clash is acceptable. TBD """ + pass + + def add_class(self, class_element, other_nodes): + """ Add class to the list of classes to validate """ + class_name = class_element.getAttribute("name") + trace("className: %s nodeType: %d nodeName: %s", class_name, + class_element.nodeType, class_element.nodeName) + + # Check if class exists in dictionary map + if class_name in self.classDict: + # Check if class clash is "acceptable" # i.e. should we bail out or not?? - if self.checkClassClashAcceptable(classElement, className): - self.validate_failed("found duplicate class : %s", className) + if self.check_class_clash_acceptable(class_element, class_name): + self.validate_failed("Found duplicate class : %s", class_name) else: - attrlist = [] - attributeDict = {} - rdnType = None - for attribute in classElement.childNodes: - attrname = None - attrtype = None + attr_list = [] + attribute_dict = {} + rdn_type = None + for attribute in class_element.childNodes: + attr_name = None + attr_type = None if attribute.nodeName == "rdn": for element in attribute.childNodes: if element.nodeName == "type": - rdnType = element.firstChild.nodeValue + rdn_type = element.firstChild.nodeValue elif attribute.nodeName == "attr": for element in attribute.childNodes: if element.nodeName == "name": - attrname = element.firstChild.nodeValue - if attrname in attrlist: - raise Exception("Attribute '%s' is defined more than once in class '%s'" % (attrname, className)) - attrlist.append(attrname); - elif element.nodeName == "type": - attrtype = element.firstChild.nodeValue - - if attrname != None and attrtype != None: - attributeDict[attrname] = attrtype + attr_name = element.firstChild.nodeValue + if attr_name in attr_list: + raise Exception("Attribute '%s' is defined " + "more than once in class '%s'" + % (attr_name, class_name)) + attr_list.append(attr_name) + elif element.nodeName == "type": + attr_type = element.firstChild.nodeValue + + if attr_name is not None and attr_type is not None: + attribute_dict[attr_name] = attr_type break - del attrlist - - classDefDict = {} - self.classDict[className] = classDefDict - self.classDict[className]["attributes"] = attributeDict - self.classDict[className]["rdn"] = rdnType - # add to ordered list - self.classList.append((className, (otherNodes, classElement))) - - def addObject(self, objectElement, otherNodes): - className = objectElement.getAttribute("class") - trace("DUMPING Object with className: %s \n %s", className, objectElement.toxml()) - - attrList = [] - objectDnElement = None - ## find "dn" childnode - for childNode in objectElement.childNodes: - if childNode.nodeName == "dn" and objectDnElement == None: - objectDnElement = childNode - elif childNode.nodeName == "attr": - for attrChild in childNode.childNodes: - if attrChild.nodeName == "name": - if attrChild.firstChild.nodeValue in attrList: - raise Exception("Attribute '%s' is defined more than once in object" % (attrChild.firstChild.nodeValue)) - attrList.append(attrChild.firstChild.nodeValue) - - del attrList - - if objectDnElement == None or className == None or objectDnElement.firstChild == None: - self.validate_failed("Failed to find classname or the dn childnode in object:%s. " ,objectElement.toxml()) - - - - # objectDnName = objectDnElement.nodeValue - ### NOTE dn name should be the nodeValue of objectDnName... - ## however minidom probably gets confused by the equal sign in dn value - # so it ends up adding a text node as a child...... - objectDnName = objectDnElement.firstChild.nodeValue - trace("objectDn key: %s", objectDnName) - - trace("objectDnElement: %s", objectDnElement.toxml()) - - # classname must exist in dictionary map - if className not in self.classDict: - self.validate_failed("failed to find class '%s' in %s", className, objectElement.toxml()) - - - if objectDnName in self.objectDnNameDict: - self.validate_failed("found duplicate object : %s", objectElement.toxml()) - - self.validateAndStoreDn(objectDnName, objectDnElement, className) - - # add to the complete object to ordered list (keyed by classname for sorting) - #self.objectList.append((className+objectDnName, objectElement)) - self.objectList.append((objectDnName, (otherNodes, objectElement))) - - - def validateAndStoreDn(self, objectDnName, objectDnElement, className): - commaIndex = -1 - dnLen = len(objectDnName) - if dnLen > 256: - self.validate_failed("length of dn is %d (max 256): %s", len(objectDnName), objectDnName) - - - # search for first unescaped comma (if any) - for i in range(0, dnLen): - if objectDnName[i] == ',': - if i > 1 and objectDnName[i-1] != '\\': - commaIndex = i + del attr_list + + class_def_dict = {} + self.classDict[class_name] = class_def_dict + self.classDict[class_name]["attributes"] = attribute_dict + self.classDict[class_name]["rdn"] = rdn_type + # Add to ordered list + self.classList.append((class_name, (other_nodes, class_element))) + + def add_object(self, object_element, other_nodes): + """ Add object to the list of objects to validate """ + class_name = object_element.getAttribute("class") + trace("DUMPING Object with className: %s\n %s", class_name, + object_element.toxml()) + + attr_list = [] + object_dn_element = None + # Find "dn" child node + for child_node in object_element.childNodes: + if child_node.nodeName == "dn" and object_dn_element is None: + object_dn_element = child_node + elif child_node.nodeName == "attr": + for attr_child in child_node.childNodes: + if attr_child.nodeName == "name": + if attr_child.firstChild.nodeValue in attr_list: + raise Exception("Attribute '%s' is defined more " + "than once in object" % + attr_child.firstChild.nodeValue) + attr_list.append(attr_child.firstChild.nodeValue) + + del attr_list + + if object_dn_element is None or class_name is None \ + or object_dn_element.firstChild is None: + self.validate_failed("Failed to find class name or the dn child " + "node in object: %s. ", + object_element.toxml()) + + # object_dn_name = object_dn_element.nodeValue + # NOTE: dn name should be the nodeValue of object_dn_name + # However minidom probably gets confused by the equal sign in dn value + # so it ends up adding a text node as a child. + object_dn_name = object_dn_element.firstChild.nodeValue + trace("objectDn key: %s", object_dn_name) + trace("objectDnElement: %s", object_dn_element.toxml()) + + # class_name must exist in dictionary map + if class_name not in self.classDict: + self.validate_failed("Failed to find class '%s' in %s", + class_name, object_element.toxml()) + + if object_dn_name in self.objectDnNameDict: + self.validate_failed("Found duplicate object: %s", + object_element.toxml()) + + self.validate_and_store_dn(object_dn_name, object_dn_element, + class_name) + + # Add the complete object to ordered list + # (keyed by class_name for sorting) + # self.objectList.append((class_name + object_dn_name, object_element)) + self.objectList.append((object_dn_name, (other_nodes, object_element))) + + def validate_and_store_dn(self, object_dn_name, object_dn_element, + class_name): + """ Validate and store object's dn """ + comma_index = -1 + dn_len = len(object_dn_name) + if dn_len > 256: + self.validate_failed("Length of dn is %d (max 256): %s", dn_len, + object_dn_name) + + # Search for first unescaped comma (if any) + for i in range(0, dn_len): + if object_dn_name[i] == ',': + if i > 1 and object_dn_name[i - 1] != '\\': + comma_index = i break - - if commaIndex == -1: - trace("Found root element (no unescaped commas): %s", objectDnName) + + if comma_index == -1: + trace("Found root element (no unescaped commas): %s", + object_dn_name) # Looks alright, add element to "dn" dictionary - self.objectDnNameDict[objectDnName] = objectDnElement + self.objectDnNameDict[object_dn_name] = object_dn_element return - - objectOwnerPart = objectDnName[commaIndex+1:] - objectIdPart = objectDnName[:commaIndex] - trace("ObjectDN: %s objectOwner: %s objectIdPart:%s", objectDnName, objectOwnerPart, objectIdPart) - - - # store all dn's even if it is SA_NAME_T - # this means even the association references is stored as objects which could be parents - # move length checking (64 bytes) to postValidate... - self.objectDnNameDict[objectDnName] = objectDnElement - - + + object_owner_part = object_dn_name[comma_index + 1:] + object_id_part = object_dn_name[:comma_index] + trace("ObjectDN: %s objectOwner: %s objectIdPart: %s", object_dn_name, + object_owner_part, object_id_part) + + # Store all dn's even if it is SA_NAME_T + # This means even the association references are stored as objects + # which could be parents + # move length checking (64 bytes) to post_validate() + self.objectDnNameDict[object_dn_name] = object_dn_element + # NOTE for some imm.xml file (for example as result of immdump tool) - # the object dn's are not ordered in a way that it is possible to validate ownership - # while parsing the inputfiles. - # Instead it must be performed after the input files are completely parsed + # the object dn's are not ordered in a way that it is possible to + # validate ownership while parsing the input files. Instead it must be + # performed after the input files are completely parsed. # validate(): Do this type of IMM validation in one go afterwards - - return - + return + def validate_failed(self, *args): + """ Print validation failed message to stderr and + set validateFailed flag """ printf_args = [] for i in range(1, len(args)): printf_args.append(args[i]) - - formatStr = "\nValidation failed: " + args[0] - print >> sys.stderr, formatStr % tuple(printf_args) - #sys.exit(2) - # no more exit, set failedFlag True + + format_str = "\nValidation failed: " + args[0] + print (format_str % tuple(printf_args), file=sys.stderr) + # sys.exit(2) + # no more exit, set failedFlag to True self.validateFailed = True def abort_file(self, *args): + """ Abort file validation and raise exception """ self.validate_failed(*args) raise AbortFileException("Aborting current file!") - - - def postValidateObjectList(self): - - for tupleNode in self.objectList: - objectElement = tupleNode[1][1] - className = objectElement.getAttribute("class") - - objectDnElement = None - ## find "dn" childnode - for childNode in objectElement.childNodes: - if childNode.nodeName == "dn": - objectDnElement = childNode - if objectDnElement.firstChild == None: - # this is really a workaround for minidom bug?: assume dn name should be the nodeValue of objectDnName but with minidom that is not true... - raise Exception("Cannot find child element of dn element (required by minidom)") - objectDnName = objectDnElement.firstChild.nodeValue - trace("objectDnElement: %s", objectDnElement.toxml()) - trace("objectDn key: %s", objectDnName) - # validate dn w.r.t ownership - self.postValidateDn(objectDnName, objectDnElement, className) - #break - elif childNode.nodeName == "attr": - nameOfAttributeToValidate = None - for element in childNode.childNodes: + + def post_validate_object_list(self): + """ Post-validate object list """ + for tuple_node in self.objectList: + object_element = tuple_node[1][1] + class_name = object_element.getAttribute("class") + attributes = self.classDict[class_name]["attributes"] + + object_dn_element = None + # Find "dn" child node + for child_node in object_element.childNodes: + if child_node.nodeName == "dn": + object_dn_element = child_node + if object_dn_element.firstChild is None: + # This is really a workaround for minidom bug?: + # Assume dn name should be the nodeValue of + # object_dn_name but with minidom that is not true. + raise Exception("Cannot find child element of dn " + "element (required by minidom)") + object_dn_name = object_dn_element.firstChild.nodeValue + trace("objectDnElement: %s", object_dn_element.toxml()) + trace("objectDn key: %s", object_dn_name) + # Validate dn w.r.t ownership + self.post_validate_dn(object_dn_name, object_dn_element, + class_name) + # break + elif child_node.nodeName == "attr": + name_of_attribute_to_validate = None + for element in child_node.childNodes: if element.nodeName == "name": - attrname = element.firstChild.nodeValue - if attrname in self.classDict[className]["attributes"]: - if self.classDict[className]["attributes"][attrname] == "SA_NAME_T": - nameOfAttributeToValidate = attrname + attr_name = element.firstChild.nodeValue + if attr_name in attributes: + if attributes[attr_name] == "SA_NAME_T": + name_of_attribute_to_validate = attr_name else: - # attribute exist in classDict but type is not further validated + # Attribute exists in classDict but type is + # not further validated break else: - self.postValidateMissingAttribute(attrname, className, objectDnElement) - elif element.nodeName == "value": - nodeValue = element.firstChild.nodeValue - self.postValidateAttributeSaNameT(nameOfAttributeToValidate, nodeValue, className, objectDnElement) - # multiple values allowed....no break - - - trace("postValidateObjectList() complete!") - - def postValidateAttributeSaNameT(self, attributeName, attributeValue, className, objectElement): - if Options.ignoreAttributeRefs == True: + self.post_validate_missing_attribute( + attr_name, class_name, object_dn_element) + elif element.nodeName == "value": + node_value = element.firstChild.nodeValue + self.post_validate_attribute_sa_name_t( + name_of_attribute_to_validate, node_value, + class_name, object_dn_element) + # Multiple values allowed....no break + + trace("post_validate_object_list() complete!") + + def post_validate_attribute_sa_name_t(self, attribute_name, + attribute_value, class_name, + object_element): + """ Post-validate SaNameT attribute """ + if Options.ignoreAttributeRefs: return - - if attributeValue not in self.objectDnNameDict: - self.validate_failed("NOTE: The object with rdn '%s' referred in attribute %s does not exist (The attribute is element in object with class: %s dn: %s)", attributeValue, attributeName, className, objectElement.toxml()) - - def postValidateMissingAttribute(self, attributeName, className, objectElement): - self.validate_failed("NOTE: The attribute %s does not exist in class definition (The attribute is element in object with class: %s dn: %s)", attributeName, className, objectElement.toxml()) - - - def postValidateDn(self, objectDnName, objectDnElement, className): - commaIndex = -1 - dnLen = len(objectDnName) - - # search for first unescaped comma (if any) - for i in range(0, dnLen): - if objectDnName[i] == ',': - if i > 1 and objectDnName[i-1] != '\\': - commaIndex = i + + if attribute_value not in self.objectDnNameDict: + self.validate_failed("NOTE: The object with rdn '%s' referred in " + "attribute %s does not exist (The attribute " + "is element in object with class: %s dn: %s)", + attribute_value, attribute_name, class_name, + object_element.toxml()) + + def post_validate_missing_attribute(self, attribute_name, class_name, + object_element): + """ Post-validate missing attribute """ + self.validate_failed("NOTE: The attribute %s does not exist in class " + "definition (The attribute is element in object " + "with class: %s dn: %s)", attribute_name, + class_name, object_element.toxml()) + + def post_validate_dn(self, object_dn_name, object_dn_element, class_name): + """ Post-validate dn """ + comma_index = -1 + dn_len = len(object_dn_name) + + # Search for first unescaped comma (if any) + for i in range(0, dn_len): + if object_dn_name[i] == ',': + if i > 1 and object_dn_name[i - 1] != '\\': + comma_index = i break - - if commaIndex == -1: - trace("Found root element (no unescaped commas): %s", objectDnName) - return - - objectOwnerPart = objectDnName[commaIndex+1:] - objectPart = objectDnName[:commaIndex] - trace("ObjectDN: %s objectOwner: %s objectPart:%s", objectDnName, objectOwnerPart, objectPart) - - # owner should exist for both SA_NAME_T and SA_STRING_T - if objectOwnerPart not in self.objectDnNameDict: - print_info_stderr("validate Dn in %s", objectDnElement.toxml()) - self.validate_failed("Parent to %s is not found %s", objectDnName, objectDnElement.toxml()) - - trace("postValidateDn() OK parentPart %s found in objectDnNameDict", objectOwnerPart) - - # But in case dn is a SA_NAME_T also the objectIdPart - # should exist in dictionary.... - if self.classDict[className]["rdn"] == "SA_NAME_T": - # find value of association (remove association name) - equalSignIndex = objectPart.find("=") - objectName = objectPart[:equalSignIndex] - objectValue = objectPart[equalSignIndex+1:] - objectValueEqualSignIndex = objectValue.find("=") - - # x=y vs x=y=z - if objectValueEqualSignIndex != -1: - if Options.ignoreRdnAssociationRefs == False: - associationValue = objectValue - # remove escaping - unescapedDN = associationValue.replace('\\','') - if unescapedDN not in self.objectDnNameDict: - print_info_stderr("validate Dn in %s", objectDnElement.toxml()) - self.validate_failed("The associated object %s is not found %s", unescapedDN, objectDnElement.toxml()) - trace("postValidateDn() OK The associated object %s is found in objectDnNameDict (Dn has type SA_NAME_T)", unescapedDN) - else: - if len(objectValue) > 64: - print_info_stderr("validate Dn in %s", objectDnElement.toxml()) - self.validate_failed("length of object value is %d (max 64): %s", len(objectPart), objectValue) - - return + if comma_index == -1: + trace("Found root element (no unescaped commas): %s", + object_dn_name) + return + object_owner_part = object_dn_name[comma_index + 1:] + object_part = object_dn_name[:comma_index] + trace("ObjectDN: %s objectOwner: %s objectPart: %s", object_dn_name, + object_owner_part, object_part) + + # Owner should exist for both SA_NAME_T and SA_STRING_T + if object_owner_part not in self.objectDnNameDict: + print_info_stderr("Validate dn in %s", object_dn_element.toxml()) + self.validate_failed("Parent to %s is not found in %s", + object_dn_name, object_dn_element.toxml()) + + trace("post_validate_dn() OK parentPart %s found in objectDnNameDict", + object_owner_part) + + # But in case dn is a SA_NAME_T also the objectIdPart should exist + # in dictionary + if self.classDict[class_name]["rdn"] == "SA_NAME_T": + # Find value of association (remove association name) + equal_sign_index = object_part.find("=") + # object_name = object_part[:equal_sign_index] + object_value = object_part[equal_sign_index + 1:] + object_value_equal_sign_index = object_value.find("=") - - def processInputfile(self, filename): + # x=y vs x=y=z + if object_value_equal_sign_index != -1: + if not Options.ignoreRdnAssociationRefs: + association_value = object_value + # Remove escape characters + unescaped_dn = association_value.replace('\\', '') + if unescaped_dn not in self.objectDnNameDict: + print_info_stderr("Validate dn in %s", + object_dn_element.toxml()) + self.validate_failed("The associated object %s is not " + "found in %s", unescaped_dn, + object_dn_element.toxml()) + trace("post_validate_dn() OK The associated object %s is " + "found in objectDnNameDict (dn has type SA_NAME_T)", + unescaped_dn) + else: + if len(object_value) > 64: + print_info_stderr("Validate dn in %s", + object_dn_element.toxml()) + self.validate_failed("Length of object value is %d " + "(max 64): %s", len(object_part), + object_value) + return + + def process_input_file(self, file_name): + """ Process IMM XML input file """ trace("") - trace("processInputfile() :%s", filename) - - if Options.isXmlLintFound and Options.schemaFilename != None: - if self.validateXmlFileWithSchema(filename, Options.schemaFilename) != 0: - self.abort_file("failed to validate input file %s: with xml schema", filename) + trace("process_input_file(): %s", file_name) + + if Options.isXmlLintFound and Options.schemaFilename is not None: + if self.validate_xml_file_with_schema(file_name, + Options.schemaFilename) != 0: + self.abort_file("Failed to validate input file %s with xml " + "schema", file_name) else: - self.verifyInputXmlDocumentFileIsParsable(filename) + self.verify_input_xml_document_file_is_parsable(file_name) + + doc = xml.dom.minidom.parse(file_name) - doc = xml.dom.minidom.parse(filename) - - - ## Fast forward to imm:contents element + # Fast forward to imm:contents element for element in doc.childNodes: - otherNodes = [] - if (element.nodeName == self.imm_content_element_name): - for childElement in element.childNodes: - trace("imm:contents loop..... Nodename:%s NodeValue%s", childElement.nodeName, childElement.nodeValue) - if (childElement.nodeName == "class"): - self.addClass(childElement, otherNodes) - otherNodes = [] - elif (childElement.nodeName == "object"): - self.addObject(childElement, otherNodes) - otherNodes = [] + other_nodes = [] + if element.nodeName == self.imm_content_element_name: + for child_element in element.childNodes: + trace("imm:contents loop.....NodeName: %s NodeValue: %s", + child_element.nodeName, child_element.nodeValue) + if child_element.nodeName == "class": + self.add_class(child_element, other_nodes) + other_nodes = [] + elif child_element.nodeName == "object": + self.add_object(child_element, other_nodes) + other_nodes = [] else: - # probably text nodes....ignore if whitespace only - childElementStr = childElement.nodeValue.lstrip().rstrip() - #if len(childElementStr) > 1: - #otherNodes.append(childElement) - otherNodes.append(childElement) + # Probably text nodes + # Ignore if whitespace only + # child_element_str = \ + # child_element.nodeValue.lstrip().rstrip() + # if len(child_element_str) > 1: + # other_nodes.append(child_element) + other_nodes.append(child_element) - return 0 - - - def postProcessValidate(self): - # iterate over all objects again to validate again when all objects are parsed from inputfiles - self.postValidateObjectList() - -#### end of class ImmDocument - -def printUsage(): - print "usage: immxml-validate [options] filename[s]" - print """ - --schema validate inputfiles with the supplied - xsd schema file - -t, --trace print trace information to stderr - --ignore-attribute-refs - specifying this option then the tool skips to - validate that SA_NAME_T attributes references - existing objects - --ignore-rdn-association-refs - specifying this option then the tool skips to - validate that SA_NAME_T rdn association references - existing objects - -v, --version print version information and exit - -h, --help display this help and exit - + + def post_process_validate(self): + """ Post validation process """ + # Iterate over all objects again to validate again when all objects are + # parsed from input files + self.post_validate_object_list() + +# End of ImmDocument class + + +def print_usage(): + """ Print usage of immxml-validate tool """ + print ("usage: immxml-validate [options] filename[s]") + print (""" + --schema validate input files with the supplied xsd schema file + + -t, --trace print trace information to stderr + + --ignore-attribute-refs + if this option is specified, the tool skips to validate + that SA_NAME_T attributes references existing objects + + --ignore-rdn-association-refs + if this option is specified, the tool skips to validate + that SA_NAME_T rdn association references existing + objects + + -v, --version print version information and exit + + -h, --help display this help and exit + See http://devel.opensaf.org/ for information and updates. -""" - -def printVersion(): - print "immxml-validate version 0.5.1" + """) + +def print_version(): + """ Print version of immxml-validate tool """ + print ("immxml-validate version 0.5.1") -def main(argv): +def main(argv): + """ Main program """ try: - opts, args = getopt.getopt(argv, "thvso", ["trace", "help", "version", "schema=", "ignore-attribute-refs", "ignore-rdn-association-refs"]) - except getopt.GetoptError, err: - # print help information and exit: + opts, args = getopt.getopt(argv, "thv", + ["trace", "help", "version", "schema=", + "ignore-attribute-refs", + "ignore-rdn-association-refs"]) + except getopt.GetoptError as err: + # Print help information and exit print_info_stderr("%s", str(err)) - printUsage() + print_usage() sys.exit(2) - - - for o, v in opts: - if o in ["-t", "--trace"]: + + for opt, value in opts: + if opt in ["-t", "--trace"]: BaseOptions.traceOn = True - if o in ("--schema"): - Options.schemaFilename = v - if o in ("--ignore-attribute-refs"): + if opt == "--schema": + Options.schemaFilename = value + if opt == "--ignore-attribute-refs": Options.ignoreAttributeRefs = True - if o in ("--ignore-rdn-association-refs"): + if opt == "--ignore-rdn-association-refs": Options.ignoreRdnAssociationRefs = True - if o in ["-v", "--version"]: - printVersion() + if opt in ["-v", "--version"]: + print_version() sys.exit(0) - elif o in ["-h", "--help"]: - printUsage() + elif opt in ["-h", "--help"]: + print_usage() sys.exit(0) - - # cannot trace these until -t, Options.traceOn is effective (or not) + + # Cannot trace these until -t, Options.traceOn is effective (or not) trace("opts:%s", opts) trace("args:%s", args) trace("sys.path:%s", sys.path) - - if len(args) == 0: - printUsage() + + if not args: + print_usage() sys.exit(2) - - - trace("Option object: \n %s", Options.printOptionSettings()) - - - if os.path.exists('/usr/bin/xmllint') == False: - if (Options.schemaFilename != None ): - abort_script("Cannot find the required linux command /usr/bin/xmllint. --schema option requires xmllint, Exiting!") - - fileList = retrieveFilenames(args) - - trace("starting to process files::\n") - atLeastOneFileFailed = False - for fileName in fileList: + + trace("Option object:\n %s", Options.print_option_settings()) + + if not os.path.exists('/usr/bin/xmllint'): + if Options.schemaFilename is not None: + abort_script("Cannot find the required linux command " + "/usr/bin/xmllint. '--schema' option requires " + "xmllint, Exiting!") + + file_list = retrieve_file_names(args) + + trace("Starting to process files...\n") + at_least_one_file_failed = False + for file_name in file_list: try: doc = ImmDocument() - verifyInputFileReadAcess(fileName) - doc.initialize(); - - doc.processInputfile(fileName) - if doc.validateFailed == False: - doc.postProcessValidate() + verify_input_file_read_access(file_name) + doc.initialize() + doc.process_input_file(file_name) + if not doc.validateFailed: + doc.post_process_validate() except AbortFileException: doc.validateFailed = True - - - if doc.validateFailed == True: - atLeastOneFileFailed = True - print_info_stderr("Validation failed of file:%s", fileName) + + if doc.validateFailed: + at_least_one_file_failed = True + print_info_stderr("Validation failed for file: %s", file_name) else: - print_info_stderr("Validation succeded of file:%s", fileName) - - trace("Done with file:%s", fileName) - - if atLeastOneFileFailed == True: + print_info_stderr("Validation succeeded for file: %s", file_name) + + trace("Done with file: %s", file_name) + + if at_least_one_file_failed: sys.exit(2) else: return 0 - - - if __name__ == "__main__": main(sys.argv[1:]) diff --git a/tools/devel/dot/trace2dot b/tools/devel/dot/trace2dot index dac0cf3..160617c 100755 --- a/tools/devel/dot/trace2dot +++ b/tools/devel/dot/trace2dot @@ -2,7 +2,7 @@ # # # (C) Copyright 2015 The OpenSAF Foundation -# Copyright Ericsson AB 2015, 2016, 2017 - All Rights Reserved. +# (C) Copyright Ericsson AB 2015, 2016, 2017. All rights reserved. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -15,24 +15,25 @@ # # Author(s): Ericsson AB # -# """ - trace2dot creates a runtime call graph using an opensaf trace file as input - and produces a dot file. The generated dot file can be viewed graphically using e.g. dotty. - - Example: - Create a dot file, amfd_trace.dot from osafamfd trace file. Start from function saClmDispatch. - $ trace2dot -t osafamfd -f saClmDispatch -d amfd_trace.dot - $ dotty amfd_trace.dot - +trace2dot creates a runtime call graph using an opensaf trace file as input and +produces a dot file. The generated dot file can be viewed graphically using +tool such as dotty. + +Example: +Create a dot file, amfd_trace.dot from osafamfd trace file. Start from function +saClmDispatch. +$ trace2dot -t osafamfd -f saClmDispatch -d amfd_trace.dot +$ dotty amfd_trace.dot """ +from __future__ import print_function import sys import os import argparse def run(trace_file, from_function, dot_file): - ''' TBD ''' + """ TBD """ infile = open(trace_file) if dot_file: @@ -50,9 +51,9 @@ def run(trace_file, from_function, dot_file): def check_infile(trace_file): - ''' Rudimentary check for missing TRACE_ENTER/TRACE_LEAVE. Will not - check for e.g. returns before TRACE_LEAVE. - ''' + """ Rudimentary check for missing TRACE_ENTER/TRACE_LEAVE. + Will not check for, e.g. return statements, before TRACE_LEAVE. + """ infile = open(trace_file) trace_enter = set() trace_leave = set() @@ -79,7 +80,7 @@ def check_infile(trace_file): def process_infile(infile, from_function, outfile): - ''' TBD ''' + """ TBD """ function_names = [] from_func_found = False @@ -96,12 +97,16 @@ def process_infile(infile, from_function, outfile): function_names.append(func_enter) - if len(function_names) > 1: + if function_names: outfile.write( - func_enter + ' [ordering=out, color=grey, shape=box, label="' + func_enter + '"];\n') + func_enter + + ' [ordering=out, color=grey, shape=box, label="' + + func_enter + '"];\n') else: outfile.write( - func_enter + ' [ordering=out, color=red, shape=box, label="' + func_enter + '"];\n') + func_enter + + ' [ordering=out, color=red, shape=box, label="' + + func_enter + '"];\n') if items[5] == '<<': func_leave = items[6].rstrip(':') @@ -109,22 +114,24 @@ def process_infile(infile, from_function, outfile): if from_function == func_leave: break - if len(function_names) > 0: + if function_names: func_enter = function_names.pop() if func_enter != func_leave: - print('%s %s %s' % (func_enter, ' has no matching TRACE_LEAVE, found ', func_leave)) + print('%s %s %s' % + (func_enter, + ' has no matching TRACE_LEAVE, found ', + func_leave)) outfile.write(func_leave + ' -> ' + func_enter + '\n') else: - if len(function_names) > 0: + if function_names: caller = function_names[len(function_names) - 1] outfile.write(caller + ' -> ' + func_enter + '\n') def file_exists(filename): - '''Check if arg is a valid file that already exists on the file - system. - ''' + """ Check if arg is a valid file that already exists on the file system. + """ if not os.path.exists(filename): raise argparse.ArgumentTypeError( "The file %s does not exist!" % filename) @@ -133,18 +140,20 @@ def file_exists(filename): def main(): - ''' program main ''' + """ Main program """ parser = argparse.ArgumentParser( - description="Create runtime callgraph from OpenSAF trace file") + description="Create runtime call graph from OpenSAF trace file") parser.add_argument('-t', '--tracefile', nargs='+', type=file_exists, help='OpenSAF trace file') parser.add_argument('-f', '--fromfunction', nargs='+', - help='Show runtime callgraph from function') - parser.add_argument('-d', '--dotfile', nargs='+', help='Result file in dot format') - parser.add_argument('-c', '--checkfile', nargs='+', - type=file_exists, help='Check infile for matching trace_enter trace_leave') + help='Show runtime call graph from function') + parser.add_argument('-d', '--dotfile', nargs='+', + help='Result file in dot format') + parser.add_argument('-c', '--checkfile', nargs='+', type=file_exists, + help='Check infile for matching ' + 'trace_enter trace_leave') args = parser.parse_args() @@ -166,5 +175,6 @@ def main(): trace_file = args.tracefile[0] run(trace_file, from_function, dot_file) + if __name__ == '__main__': main() diff --git a/tools/devel/review/patch-tokenize.py b/tools/devel/review/patch-tokenize.py index 8350517..30a5188 100755 --- a/tools/devel/review/patch-tokenize.py +++ b/tools/devel/review/patch-tokenize.py @@ -2,6 +2,7 @@ # -*- OpenSAF -*- # # (C) Copyright 2010 The OpenSAF Foundation +# (C) Copyright Ericsson AB 2015, 2016, 2017. All rights reserved. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY @@ -13,16 +14,19 @@ # licensing terms. # # Author(s): Wind River Systems +# Ericsson AB # - +# pylint: disable=invalid-name +""" Tokenize alphanumeric words in patch file """ +from __future__ import print_function import re import sys -# Validates only on lines that are patch addition (e.g. + foo) -pattern = re.compile("\+\s.*") +# Validate only on lines that are patch additions (e.g. + foo) +pattern = re.compile(r"\+\s.*") m = pattern.match(sys.argv[1]) if m: - # Tokenize all alphanumeric words for banned word lookup - for word in re.findall("\s*(\w+).?\(", m.group(0)): - print word + # Tokenize all alphanumeric words for banned word lookup + for word in re.findall(r"\s*(\w+).?\(", m.group(0)): + print (word) -- 2.7.4 ------------------------------------------------------------------------------ Check out the vibrant tech community on one of the world's most engaging tech sites, Slashdot.org! http://sdm.link/slashdot _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel
