Repository: oozie Updated Branches: refs/heads/branch-4.3 3db0cf4c3 -> 62eb28b7b
Fix parsing issue (satishsaley) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a9791858 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a9791858 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a9791858 Branch: refs/heads/branch-4.3 Commit: a9791858657c65b91a8a622d6a2a0335149287f0 Parents: 3db0cf4 Author: satishsaley <[email protected]> Authored: Mon Dec 18 13:06:58 2017 -0800 Committer: satishsaley <[email protected]> Committed: Mon Dec 18 13:06:58 2017 -0800 ---------------------------------------------------------------------- .../command/bundle/BundleSubmitXCommand.java | 3 +- .../command/coord/CoordSubmitXCommand.java | 3 +- .../oozie/service/ConfigurationService.java | 24 +++-- .../oozie/service/HadoopAccessorService.java | 2 +- .../org/apache/oozie/service/SchemaService.java | 27 ++++- .../apache/oozie/servlet/V2ValidateServlet.java | 6 +- .../org/apache/oozie/util/XConfiguration.java | 106 ++++++++++++++----- .../java/org/apache/oozie/util/XmlUtils.java | 65 +++++------- .../workflow/lite/LiteWorkflowAppParser.java | 44 ++++---- .../apache/oozie/service/TestSchemaService.java | 23 ++-- .../apache/oozie/util/TestXConfiguration.java | 41 +++---- .../org/apache/oozie/util/TestXmlUtils.java | 12 ++- 12 files changed, 213 insertions(+), 143 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java index 3df37fa..f3742eb 100644 --- a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java @@ -348,9 +348,8 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { * @throws BundleJobException thrown if failed to validate xml */ private void validateXml(String xmlContent) throws BundleJobException { - javax.xml.validation.Schema schema = Services.get().get(SchemaService.class).getSchema(SchemaName.BUNDLE); - Validator validator = schema.newValidator(); try { + Validator validator = Services.get().get(SchemaService.class).getValidator(SchemaName.BUNDLE); validator.validate(new StreamSource(new StringReader(xmlContent))); } catch (SAXException ex) { http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java index 969336d..c75774b 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java @@ -507,9 +507,8 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { * @throws CoordinatorJobException thrown if unable to validate coordinator xml */ private void validateXml(String xmlContent) throws CoordinatorJobException { - javax.xml.validation.Schema schema = Services.get().get(SchemaService.class).getSchema(SchemaName.COORDINATOR); - Validator validator = schema.newValidator(); try { + Validator validator = Services.get().get(SchemaService.class).getValidator(SchemaName.COORDINATOR); validator.validate(new StreamSource(new StringReader(xmlContent))); } catch (SAXException ex) { http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/service/ConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java index 524d4d6..919bda6 100644 --- a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java +++ b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java @@ -18,13 +18,15 @@ package org.apache.oozie.service; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.ErrorCode; import org.apache.oozie.util.ConfigUtils; import org.apache.oozie.util.Instrumentable; import org.apache.oozie.util.Instrumentation; -import org.apache.oozie.util.XLog; import org.apache.oozie.util.XConfiguration; -import org.apache.oozie.ErrorCode; +import org.apache.oozie.util.XLog; +import org.apache.oozie.util.ZKUtils; import java.io.File; import java.io.FileInputStream; @@ -33,15 +35,11 @@ import java.io.InputStream; import java.io.StringWriter; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.Arrays; - -import org.apache.oozie.util.ZKUtils; - -import com.google.common.annotations.VisibleForTesting; /** * Built in service that initializes the services configuration. @@ -127,6 +125,13 @@ public class ConfigurationService implements Service, Instrumentable { // Not supported getPasswordMethod = null; } + + try { + Method method = Configuration.class.getDeclaredMethod("setRestrictSystemPropertiesDefault", boolean + .class); + method.invoke(null, true); + } catch( NoSuchMethodException | InvocationTargetException | IllegalAccessException ignore) { + } } public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml"; @@ -299,6 +304,7 @@ public class ConfigurationService implements Service, Instrumentable { private XConfiguration loadConfig(InputStream inputStream, boolean defaultConfig) throws IOException, ServiceException { XConfiguration configuration; configuration = new XConfiguration(inputStream); + configuration.setRestrictSystemProperties(false); for(Map.Entry<String,String> entry: configuration) { if (defaultConfig) { defaultConfigs.put(entry.getKey(), entry.getValue()); @@ -318,6 +324,10 @@ public class ConfigurationService implements Service, Instrumentable { setValue(entry.getKey(), entry.getValue()); } } + if(conf instanceof XConfiguration) { + this.setRestrictParser(((XConfiguration)conf).getRestrictParser()); + this.setRestrictSystemProperties(((XConfiguration)conf).getRestrictSystemProperties()); + } } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 23a9d92..7358209 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -254,7 +254,7 @@ public class HadoopAccessorService implements Service { File f = new File(dir, file); if (f.exists()) { InputStream is = new FileInputStream(f); - Configuration conf = new XConfiguration(is); + Configuration conf = new XConfiguration(is, false); is.close(); XConfiguration.copy(conf, hadoopConf); } http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/service/SchemaService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/SchemaService.java b/core/src/main/java/org/apache/oozie/service/SchemaService.java index 3181495..1780eb1 100644 --- a/core/src/main/java/org/apache/oozie/service/SchemaService.java +++ b/core/src/main/java/org/apache/oozie/service/SchemaService.java @@ -28,8 +28,8 @@ import javax.xml.XMLConstants; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; +import javax.xml.validation.Validator; -import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; import org.apache.oozie.util.IOUtils; import org.xml.sax.SAXException; @@ -183,4 +183,29 @@ public class SchemaService implements Service { return id; } } + + /** + * Returns validator for schema + * @param schemaName + * @return + * @throws SAXException + */ + public Validator getValidator(SchemaName schemaName) throws SAXException { + return getValidator(getSchema(schemaName)); + } + + /** + * Returns validator for schema + * @param schema + * @return + * @throws SAXException + */ + public static Validator getValidator(Schema schema) throws SAXException { + Validator validator = schema.newValidator(); + validator.setFeature("http://apache.org/xml/features/disallow-doctype-decl";, true); + validator.setFeature("http://xml.org/sax/features/external-general-entities";, false); + validator.setFeature("http://xml.org/sax/features/external-parameter-entities";, false); + return validator; + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java index dbb3d49..42688e9 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2ValidateServlet.java @@ -137,8 +137,8 @@ public class V2ValidateServlet extends JsonRestServlet { } private void validateSchema(Schema schema, Reader src) throws SAXException, IOException, OozieCLIException{ - Validator validator = schema.newValidator(); - validator.validate(new StreamSource(src)); + Validator validator = SchemaService.getValidator(schema); + validator.validate(new StreamSource(src)); } private JSONObject createJSON(String content) { @@ -147,4 +147,4 @@ public class V2ValidateServlet extends JsonRestServlet { return jsonObject; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/util/XConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/XConfiguration.java b/core/src/main/java/org/apache/oozie/util/XConfiguration.java index bd62cb1..e3591db 100644 --- a/core/src/main/java/org/apache/oozie/util/XConfiguration.java +++ b/core/src/main/java/org/apache/oozie/util/XConfiguration.java @@ -50,6 +50,8 @@ public class XConfiguration extends Configuration { public static final String CONFIGURATION_SUBSTITUTE_DEPTH = "oozie.configuration.substitute.depth"; + private boolean restrictSystemProperties = true; + private boolean restrictParser = true; /** * Create an empty configuration. <p> Default values are not loaded. */ @@ -62,25 +64,49 @@ public class XConfiguration extends Configuration { * Create a configuration from an InputStream. <p> Code canibalized from <code>Configuration.loadResource()</code>. * * @param is inputstream to read the configuration from. + * @param restrictParser whether to restrict the parser * @throws IOException thrown if the configuration could not be read. */ - public XConfiguration(InputStream is) throws IOException { + public XConfiguration(InputStream is, boolean restrictParser) throws IOException { this(); + this.restrictParser = restrictParser; parse(is); } /** + * Create a configuration from an InputStream. <p> Code canibalized from <code>Configuration.loadResource()</code>. + * + * @param is inputstream to read the configuration from. + * @throws IOException thrown if the configuration could not be read. + */ + public XConfiguration(InputStream is) throws IOException { + this(is, true); + } + + /** * Create a configuration from an Reader. <p> Code canibalized from <code>Configuration.loadResource()</code>. * * @param reader reader to read the configuration from. + * @param restrictParser whether to restrict the parser * @throws IOException thrown if the configuration could not be read. */ - public XConfiguration(Reader reader) throws IOException { + public XConfiguration(Reader reader, boolean restrictParser) throws IOException { this(); + this.restrictParser = restrictParser; parse(reader); } /** + * Create a configuration from an Reader. <p> Code canibalized from <code>Configuration.loadResource()</code>. + * + * @param reader reader to read the configuration from. + * @throws IOException thrown if the configuration could not be read. + */ + public XConfiguration(Reader reader) throws IOException { + this(reader, true); + } + + /** * Create an configuration from a Properties instance. * * @param props Properties instance to get all properties from. @@ -90,7 +116,6 @@ public class XConfiguration extends Configuration { for (Map.Entry entry : props.entrySet()) { set((String) entry.getKey(), (String) entry.getValue()); } - } /** @@ -123,7 +148,7 @@ public class XConfiguration extends Configuration { */ @Override public String get(String name) { - return substituteVars(getRaw(name)); + return substituteVars(getRaw(name)); } /** @@ -173,8 +198,9 @@ public class XConfiguration extends Configuration { var = var.substring(2, var.length() - 1); // remove ${ .. } String val = getRaw(var); - if (val == null) { - val = System.getProperty(var); + + if (val == null && !this.restrictSystemProperties) { + val = System.getProperty(var); } if (val == null) { @@ -253,16 +279,7 @@ public class XConfiguration extends Configuration { // Canibalized from Hadoop <code>Configuration.loadResource()</code>. private void parse(InputStream is) throws IOException { try { - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); - // support for includes in the xml file - docBuilderFactory.setNamespaceAware(true); - docBuilderFactory.setXIncludeAware(true); - // ignore all comments inside the xml file - docBuilderFactory.setIgnoringComments(true); - docBuilderFactory.setExpandEntityReferences(false); - docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl";, true); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(is); + Document doc = getDocumentBuilder().parse(is); parseDocument(doc); } @@ -277,16 +294,7 @@ public class XConfiguration extends Configuration { // Canibalized from Hadoop <code>Configuration.loadResource()</code>. private void parse(Reader reader) throws IOException { try { - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); - // support for includes in the xml file - docBuilderFactory.setNamespaceAware(true); - docBuilderFactory.setXIncludeAware(true); - // ignore all comments inside the xml file - docBuilderFactory.setIgnoringComments(true); - docBuilderFactory.setExpandEntityReferences(false); - docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl";, true); - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = builder.parse(new InputSource(reader)); + Document doc = getDocumentBuilder().parse(new InputSource(reader)); parseDocument(doc); } catch (SAXException e) { @@ -306,7 +314,7 @@ public class XConfiguration extends Configuration { processNodes(root); } - // Canibalized from Hadoop <code>Configuration.loadResource()</code>. + // Cannibalized from Hadoop <code>Configuration.loadResource()</code>. private void processNodes(Element root) throws IOException { try { NodeList props = root.getChildNodes(); @@ -392,4 +400,48 @@ public class XConfiguration extends Configuration { } return values; } + + private DocumentBuilder getDocumentBuilder() throws ParserConfigurationException { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); + docBuilderFactory.setNamespaceAware(true); + docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl",true); + docBuilderFactory.setXIncludeAware(true); + if(this.restrictParser) { + docBuilderFactory.setXIncludeAware(false); + //Redundant with disallow-doctype, but just in case + docBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities";, false); + docBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities";, false); + docBuilderFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd";, false); + } + docBuilderFactory.setExpandEntityReferences(false); + // ignore all comments inside the xml file + docBuilderFactory.setIgnoringComments(true); + return docBuilderFactory.newDocumentBuilder(); + } + + /** + * Restrict the parser + * @param restrictParser + */ + public void setRestrictParser(boolean restrictParser) { + this.restrictParser = restrictParser; + } + + public boolean getRestrictParser() { + return restrictParser; + } + /** + * Restrict reading property from System.getProperty() + * @param restrictSystemProperties + */ + public void setRestrictSystemProperties(boolean restrictSystemProperties) { + this.restrictSystemProperties = restrictSystemProperties; + } + + public boolean getRestrictSystemProperties() { + return restrictSystemProperties; + } + + + } http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/util/XmlUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/XmlUtils.java b/core/src/main/java/org/apache/oozie/util/XmlUtils.java index f850236..993ae7f 100644 --- a/core/src/main/java/org/apache/oozie/util/XmlUtils.java +++ b/core/src/main/java/org/apache/oozie/util/XmlUtils.java @@ -32,7 +32,9 @@ import java.util.Map; import java.util.Properties; import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Result; import javax.xml.transform.Source; import javax.xml.transform.Transformer; @@ -56,8 +58,6 @@ import org.jdom.Namespace; import org.jdom.input.SAXBuilder; import org.jdom.output.Format; import org.jdom.output.XMLOutputter; -import org.xml.sax.EntityResolver; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; /** @@ -65,23 +65,12 @@ import org.xml.sax.SAXException; */ public class XmlUtils { - private static class NoExternalEntityEntityResolver implements EntityResolver { - - @Override - public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException { - return new InputSource(new ByteArrayInputStream(new byte[0])); - } - - } - private static SAXBuilder createSAXBuilder() { SAXBuilder saxBuilder = new SAXBuilder(); - - //THIS IS NOT WORKING - //saxBuilder.setFeature("http://xml.org/sax/features/external-general-entities";, false); - - //INSTEAD WE ARE JUST SETTING AN EntityResolver that does not resolve entities - saxBuilder.setEntityResolver(new NoExternalEntityEntityResolver()); + saxBuilder.setFeature("http://apache.org/xml/features/disallow-doctype-decl",true); + saxBuilder.setFeature("http://xml.org/sax/features/external-general-entities";, false); + saxBuilder.setFeature("http://xml.org/sax/features/external-parameter-entities";, false); + saxBuilder.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd";, false); return saxBuilder; } @@ -279,28 +268,10 @@ public class XmlUtils { * @param xml to be validated */ public static void validateXml(Schema schema, String xml) throws SAXException, IOException { - - Validator validator = schema.newValidator(); + Validator validator = SchemaService.getValidator(schema); validator.validate(new StreamSource(new ByteArrayInputStream(xml.getBytes()))); } - /** - * Create schema object for the given xsd - * - * @param is inputstream to schema. - * @return the schema object. - */ - public static Schema createSchema(InputStream is) { - SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - StreamSource src = new StreamSource(is); - try { - return factory.newSchema(src); - } - catch (SAXException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - public static void validateData(String xmlData, SchemaName xsdFile) throws SAXException, IOException { if (xmlData == null || xmlData.length() == 0) { return; @@ -318,7 +289,7 @@ public class XmlUtils { */ public static String writePropToString(Properties props) throws IOException { try { - org.w3c.dom.Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); + org.w3c.dom.Document doc = getDocumentBuilder().newDocument(); org.w3c.dom.Element conf = doc.createElement("configuration"); doc.appendChild(conf); conf.appendChild(doc.createTextNode("\n")); @@ -362,6 +333,26 @@ public class XmlUtils { } /** + * Returns a DocumentBuilder + * @return DocumentBuilder + * @throws ParserConfigurationException + */ + private static DocumentBuilder getDocumentBuilder() throws ParserConfigurationException { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); + docBuilderFactory.setNamespaceAware(true); + docBuilderFactory.setXIncludeAware(false); + docBuilderFactory.setExpandEntityReferences(false); + docBuilderFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl",true); + //Redundant with disallow-doctype, but just in case + docBuilderFactory.setFeature("http://xml.org/sax/features/external-general-entities";, false); + docBuilderFactory.setFeature("http://xml.org/sax/features/external-parameter-entities";, false); + docBuilderFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd";, false); + // ignore all comments inside the xml file + docBuilderFactory.setIgnoringComments(true); + return docBuilderFactory.newDocumentBuilder(); + } + + /** * Escape characters for text appearing as XML data, between tags. * <p> * The following characters are replaced with corresponding character entities : http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java index a74e5c7..918996e 100644 --- a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java +++ b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java @@ -18,27 +18,6 @@ package org.apache.oozie.workflow.lite; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.zip.Deflater; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.Inflater; -import java.util.zip.InflaterInputStream; - -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import javax.xml.validation.Validator; - import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -49,6 +28,7 @@ import org.apache.oozie.action.hadoop.FsActionExecutor; import org.apache.oozie.action.oozie.SubWorkflowActionExecutor; import org.apache.oozie.service.ActionService; import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.service.SchemaService; import org.apache.oozie.service.Services; import org.apache.oozie.util.ELUtils; import org.apache.oozie.util.IOUtils; @@ -63,6 +43,26 @@ import org.jdom.JDOMException; import org.jdom.Namespace; import org.xml.sax.SAXException; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.Validator; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + /** * Class to parse and validate workflow xml */ @@ -165,7 +165,7 @@ public class LiteWorkflowAppParser { String strDef = writer.toString(); if (schema != null) { - Validator validator = schema.newValidator(); + Validator validator = SchemaService.getValidator(schema); validator.validate(new StreamSource(new StringReader(strDef))); } http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/test/java/org/apache/oozie/service/TestSchemaService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java index 88a10db..db6d0a2 100644 --- a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java +++ b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java @@ -119,25 +119,25 @@ public class TestSchemaService extends XTestCase { public void testWfSchema() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); validator.validate(new StreamSource(new StringReader(APP1))); } public void testWfMultipleJavaOpts() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); validator.validate(new StreamSource(new StringReader(WF_4_MULTIPLE_JAVA_OPTS))); } public void testWfSchemaV2() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); validator.validate(new StreamSource(new StringReader(APP_V2))); } public void testWfSchemaV25() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); validator.validate(new StreamSource(new StringReader(APP_V25))); } @@ -146,19 +146,19 @@ public class TestSchemaService extends XTestCase { setSystemProperty(SchemaService.WF_CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); new Services().init(); SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); validator.validate(new StreamSource(new StringReader(APP2))); } public void testWfSLASchema() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); validator.validate(new StreamSource(new StringReader(WF_SLA_APP))); } public void testWfSLASchemaNW() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + Validator validator = wss.getValidator(SchemaName.WORKFLOW); try { validator.validate(new StreamSource(new StringReader(WF_SLA_APP_NW))); fail("Schema service check does not work"); @@ -170,7 +170,7 @@ public class TestSchemaService extends XTestCase { public void testCoordSchema() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); + Validator validator = wss.getValidator(SchemaName.COORDINATOR); String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.1' xmlns:sla='uri:oozie:sla:0.1'> " + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> </controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </p roperty></configuration> </workflow> " + "</action> </coordinator-app>"; @@ -182,7 +182,7 @@ public class TestSchemaService extends XTestCase { public void testCoordSchema2() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); + Validator validator = wss.getValidator(SchemaName.COORDINATOR); String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1'> " + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> <throttle>3</throttle></controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut( 'LOCAL_A')}</value> </property></configuration> </workflow> " + "</action> </coordinator-app>"; @@ -194,8 +194,7 @@ public class TestSchemaService extends XTestCase { public void testCoordSLASchema() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.COORDINATOR) - .newValidator(); + Validator validator = wss.getValidator(SchemaName.COORDINATOR); String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1'> " + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> </controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </p roperty></configuration> </workflow> " + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " @@ -211,7 +210,7 @@ public class TestSchemaService extends XTestCase { public void testBundleSchema() throws Exception { SchemaService wss = Services.get().get(SchemaService.class); - Validator validator = wss.getSchema(SchemaName.BUNDLE).newValidator(); + Validator validator = wss.getValidator(SchemaName.BUNDLE); String BUNDLE_APP = "<bundle-app name='NAME' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:bundle:0.1'> " + "<controls> <kick-off-time>2009-02-02T00:00Z</kick-off-time> </controls> " + "<coordinator name='c12'> " http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java b/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java index bd8e077..7a9d780 100644 --- a/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java +++ b/core/src/test/java/org/apache/oozie/util/TestXConfiguration.java @@ -56,32 +56,24 @@ public class TestXConfiguration extends XTestCase { String parentXml = "parentXml"; prepareXmlWithInclude(parentXml); try { - XConfiguration conf = new XConfiguration(new FileInputStream(new File(getTestCaseDir(), parentXml))); - assertEquals("DEFAULT", conf.get("oozie.dummy")); - // verify the properties from include file - assertEquals("bar", conf.get("foo")); - assertEquals("def", conf.get("abc")); - } catch (IOException e) { - e.printStackTrace(); - fail("XInclude failed"); + new XConfiguration(new FileInputStream(new File(getTestCaseDir(), parentXml))); + fail("XInclude should not be allowed"); + } + catch (IOException e) { + assertEquals("bad conf file: element not <property>", e.getMessage()); } - } public void testAddXIncludeFromReader() throws IOException { String parentXml = "parentXml"; prepareXmlWithInclude(parentXml); try { - XConfiguration conf = new XConfiguration(new FileReader(new File(getTestCaseDir(), parentXml))); - assertEquals("DEFAULT", conf.get("oozie.dummy")); - // verify the properties from include file - assertEquals("bar", conf.get("foo")); - assertEquals("def", conf.get("abc")); - } catch (IOException e) { - e.printStackTrace(); - fail("XInclude failed"); + new XConfiguration(new FileReader(new File(getTestCaseDir(), parentXml))); + fail("XInclude should not be allowed"); + } + catch (IOException e) { + assertEquals("bad conf file: element not <property>", e.getMessage()); } - } // Copy the parent xml to testCaseDir and add the include element @@ -184,6 +176,7 @@ public class TestXConfiguration extends XTestCase { public void testResolve() { XConfiguration conf = new XConfiguration(); + conf.setRestrictSystemProperties(false); conf.set("a", "A"); conf.set("b", "${a}"); assertEquals("A", conf.getRaw("a")); @@ -205,14 +198,9 @@ public class TestXConfiguration extends XTestCase { assertEquals("${aa}", conf.getRaw("c")); assertEquals("A", conf.get("a")); assertEquals("A", conf.get("b")); - assertEquals("foo", conf.get("c")); assertEquals("${aaa}", conf.get("d")); - conf.set("un","${user.name}"); - assertEquals(System.getProperty("user.name"), conf.get("un")); - setSystemProperty("user.name", "foo"); - assertEquals("foo", conf.get("un")); - + assertEquals("${user.name}", conf.get("un")); } public void testSubstituteVar() throws ServiceException { @@ -235,7 +223,8 @@ public class TestXConfiguration extends XTestCase { try { conf.get(String.valueOf("key" + (depth))); fail("Fail to apply substitution depth"); - } catch (IllegalStateException e) { + } + catch (IllegalStateException e) { assertTrue(e.getMessage().contains("Variable substitution depth too large: " + depth)); } @@ -264,4 +253,4 @@ public class TestXConfiguration extends XTestCase { services.destroy(); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/a9791858/core/src/test/java/org/apache/oozie/util/TestXmlUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestXmlUtils.java b/core/src/test/java/org/apache/oozie/util/TestXmlUtils.java index 4ceaede..7e11d5f 100644 --- a/core/src/test/java/org/apache/oozie/util/TestXmlUtils.java +++ b/core/src/test/java/org/apache/oozie/util/TestXmlUtils.java @@ -19,7 +19,7 @@ package org.apache.oozie.util; import org.apache.oozie.test.XTestCase; -import org.jdom.Element; +import org.jdom.input.JDOMParseException; public class TestXmlUtils extends XTestCase { @@ -27,8 +27,14 @@ public class TestXmlUtils extends XTestCase { + "<foo>&xxe;</foo>"; public void testExternalEntity() throws Exception { - Element e = XmlUtils.parseXml(EXTERNAL_ENTITY_XML); - assertEquals(0, e.getText().length()); + try { + XmlUtils.parseXml(EXTERNAL_ENTITY_XML); + fail("DOCTYPE should not be allowed"); + } catch (JDOMParseException e) { + assertTrue("Exception has different message.", e.getMessage(). + contains("DOCTYPE is disallowed when the feature \"http://apache."; + + "org/xml/features/disallow-doctype-decl\" set to true")); + } } public void testRemoveComments() throws Exception {
