http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/apex_development_setup/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/apex_development_setup/index.html b/content/docs/apex-3.3/apex_development_setup/index.html index 60350b2..c3029c0 100644 --- a/content/docs/apex-3.3/apex_development_setup/index.html +++ b/content/docs/apex-3.3/apex_development_setup/index.html @@ -135,6 +135,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="../compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div>
http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/application_development/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/application_development/index.html b/content/docs/apex-3.3/application_development/index.html index 5297074..d0bc30b 100644 --- a/content/docs/apex-3.3/application_development/index.html +++ b/content/docs/apex-3.3/application_development/index.html @@ -201,6 +201,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="../compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/application_packages/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/application_packages/index.html b/content/docs/apex-3.3/application_packages/index.html index 41e9a29..4ca9434 100644 --- a/content/docs/apex-3.3/application_packages/index.html +++ b/content/docs/apex-3.3/application_packages/index.html @@ -147,6 +147,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="../compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/autometrics/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/autometrics/index.html b/content/docs/apex-3.3/autometrics/index.html index 6fd3fd8..d5ffea8 100644 --- a/content/docs/apex-3.3/autometrics/index.html +++ b/content/docs/apex-3.3/autometrics/index.html @@ -146,6 +146,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="../compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/compatibility/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/compatibility/index.html b/content/docs/apex-3.3/compatibility/index.html new file mode 100644 index 0000000..fc88b80 --- /dev/null +++ b/content/docs/apex-3.3/compatibility/index.html @@ -0,0 +1,280 @@ +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + + + <title>Compatibility - Apache Apex Documentation</title> + + + <link rel="shortcut icon" href="../favicon.ico"> + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + <link rel="stylesheet" href="../css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" /> + <link rel="stylesheet" href="../css/highlight.css"> + + + <script> + // Current page data + var mkdocs_page_name = "Compatibility"; + var mkdocs_page_input_path = "compatibility.md"; + var mkdocs_page_url = "/compatibility/"; + </script> + + <script src="../js/jquery-2.1.1.min.js"></script> + <script src="../js/modernizr-2.8.3.min.js"></script> + <script type="text/javascript" src="../js/highlight.pack.js"></script> + <script src="../js/theme.js"></script> + + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> + <div class="wy-side-nav-search"> + <a href=".." class="icon icon-home"> Apache Apex Documentation</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + <ul class="current"> + + <li> + <li class="toctree-l1 "> + <a class="" href="..">Apache Apex</a> + + </li> +<li> + + <li> + <ul class="subnav"> + <li><span>Development</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="../apex_development_setup/">Development Setup</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../application_development/">Applications</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../application_packages/">Packages</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../operator_development/">Operators</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../autometrics/">AutoMetric API</a> + + </li> + + + </ul> +<li> + + <li> + <ul class="subnav"> + <li><span>Operations</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="../dtcli/">dtCli</a> + + </li> + + + </ul> +<li> + + <li> + <li class="toctree-l1 current"> + <a class="current" href="./">Compatibility</a> + + <ul> + + <li class="toctree-l3"><a href="#apache-apex-compatibility">Apache Apex Compatibility</a></li> + + <li><a class="toctree-l4" href="#purpose">Purpose</a></li> + + <li><a class="toctree-l4" href="#compatibility-types">Compatibility types</a></li> + + + </ul> + + </li> +<li> + + </ul> + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="..">Apache Apex Documentation</a> + </nav> + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="..">Docs</a> »</li> + + + + <li>Compatibility</li> + <li class="wy-breadcrumbs-aside"> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + <div class="section"> + + <h1 id="apache-apex-compatibility">Apache Apex Compatibility</h1> +<h2 id="purpose">Purpose</h2> +<p>This document captures the compatibility goals of the Apache Apex project. The different types of compatibility between Apex releases that affect contributors, downstream projects, and end-users are enumerated. For each type of compatibility we:</p> +<ul> +<li>describe the impact on downstream projects or end-users</li> +<li>where applicable, call out the policy adopted when incompatible changes are permitted.</li> +</ul> +<p>Apache Apex follows <a href="http://semver.org/";>semantic versioning</a>. Depending on the compatibility type, there may be different tools or mechanisms to ensure compatibility, for example by comparing artifacts during the build process.</p> +<p>The type of change will inform the required target version number. Given a version number MAJOR.MINOR.PATCH, increment the:</p> +<ul> +<li>MAJOR version when you make incompatible API changes,</li> +<li>MINOR version when you add functionality in a backward-compatible manner, and</li> +<li>PATCH version when you make backward-compatible bug fixes.</li> +</ul> +<p>Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.</p> +<p>The overall goal is to avoid backward incompatible changes and major release upgrades. Accordingly we attempt to release new features with minor versions that are incremental to the prior release and offer our users a frictionless upgrade path. When planning contributions, please consider compatibility and release road map upfront. Specifically, certain changes that conflict with the versioning may need to be documented in JIRA and deferred until a future major release. </p> +<h2 id="compatibility-types">Compatibility types</h2> +<h3 id="java-api">Java API</h3> +<p>Public API compatibility is required to ensure end-user programs and downstream projects continue to work without modification. +The public API consists of:</p> +<ul> +<li>apex-core: all interfaces and classes in <code>api</code> and <code>common</code> modules</li> +<li>apex-malhar: all interfaces and classes in all modules except <code>demos</code>, <code>samples</code>, <code>benchmark</code> </li> +</ul> +<p>Interfaces and classes that are part of the public API and are annotated with <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html";>interface stability</a> are treated according to the rules defined by the annotation. </p> +<p>Policy</p> +<p>Changes to the public API must follow semantic versioning. +Public APIs must be deprecated for at least one minor release prior to their removal in a major release. +The <a href="https://github.com/siom79/japicmp";>japicmp Maven plugin</a> is used to enforce compatibility as part of the Travis pre-commit builds.</p> +<h3 id="semantic-compatibility">Semantic compatibility</h3> +<p>The behavior of APIs needs to remain consistent over versions, though changes for correctness may result in changes in behavior. Tests and javadocs specify the behavior. Over time, test suites should be expanded to verify compliance with the specification, effectively creating a formal specification for the subset of behaviors that can be easily tested.</p> +<p>Policy</p> +<p>The behavior of existing API cannot be modified as it would break existing user code. There are exceptional circumstances that may justify such changes, in which cases they should be discussed on the mailing list before implementation. Examples are bug fixes related to security issues, data corruption/consistency or to correct an unintended change from previous release that violated semantic compatibility. Such changes should be accompanied by test coverage for the exact behavior.</p> +<h3 id="rest-api">REST API</h3> +<p>REST API compatibility corresponds to both the URLs and request/response content over the wire. REST APIs are specifically meant for stable use by clients across releases, even major releases. </p> +<p>Policy</p> +<p>The REST API is separately versioned. This is to allow for co-existence of old and new API should there be a need for backward incompatible changes in the future.</p> +<h3 id="command-line-interface-cli">Command Line Interface (CLI)</h3> +<p>The CLI may be used either directly via the system shell or via shell scripts. Changing the path, removing or renaming command line options, the order of arguments, or the command return code and output break compatibility and may adversely affect users.</p> +<p>Policy</p> +<p>CLI commands are to be deprecated (warning when used) in a prior minor release before they are removed or incompatibly modified in a subsequent major release.</p> +<h3 id="configuration-files">Configuration Files</h3> +<p>Configuration files are used for engine or application settings. Changes to keys and default values directly affect users and are hard to diagnose (compared to a compile error, for example).</p> +<p>Policy</p> +<p>Name, location, format, keys of configuration files should be deprecated in a prior minor release and can only be changed in major release. Best effort should be made to support the deprecated behavior for one more major release (not guaranteed). It is also desirable to provide the user with a migration tool.</p> +<h3 id="internal-wire-compatibility">Internal Wire compatibility</h3> +<p>Apex containers internally use RPC communication and netlet for the data flow. The protocols are private and user components are not exposed to it. Apex is a YARN application and automatically deployed. There is currently no situation where containers of different Apex engine versions need to be interoperable. Should such a scenario become relevant in the future, wire compatibility needs to be specified.</p> +<p>Policy</p> +<p>N/A</p> +<h3 id="internal-file-formats">Internal File formats</h3> +<p>Apex engine stores data in the file system for recovery and the data is typically obtained from serialization (from Kryo, Java etc.). Changes to internal classes may affect the ability to relaunch an application with upgraded engine code from previous state. This is currently not supported. In the future, the serialization mechanism should guarantee backward compatibility.</p> +<p>Policy</p> +<p>Currently no compatibility guarantee. User to cold-restart application on engine upgrade.</p> +<h3 id="java-classpath">Java Classpath</h3> +<p>Apex applications should not bundle Hadoop dependencies or Apex engine dependencies but use the dependencies provided in the target environment to avoid conflicts. The Apex application archetype can be used to generate a compliant project. </p> +<p>Policy</p> +<p>Apex engine dependencies can change as per semantic versioning. Following above guidelines automatically maintains the backward compatibility based on semantic versioning of Apex.</p> +<h3 id="maven-build-artifacts">Maven Build Artifacts</h3> +<p>Downstream projects reference the Apex engine dependencies and Malhar operator libraries for application development etc. Changes to the packaging (which classes are in which jar), the groupId, artifactId and which artifacts are deployed to Maven central impact upgrades.</p> +<p>Policy</p> +<p>The artifacts that contain the classes that form the public API as specified above cannot change in patch releases and should stay compatible within a major release. Patch releases can change dependencies, but only at the patch level and following semantic versioning.</p> +<h3 id="hardwaresoftware-requirements">Hardware/Software Requirements</h3> +<p>Apex depends on Apache Hadoop. The community intends to support all major Hadoop distros and current versions. Apex currently supports Hadoop 2.2.0 and higher and Java 7 and higher. Apex is written in Java and has been tested on Linux based Hadoop clusters. There are no additional restrictions on the hardware architecture. </p> +<p>To keep up with the latest advances in hardware, operating systems, JVMs, Hadoop and other software, new Apex releases may require higher versions. Upgrading Apex may require upgrading other dependent software components.</p> +<p>Policy</p> +<p>The JVM and Hadoop minimum version requirements are not expected to change outside major releases.</p> + + </div> + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + + <a href="../dtcli/" class="btn btn-neutral" title="dtCli"><span class="icon icon-circle-arrow-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <!-- Copyright etc --> + + </div> + + Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. +</footer> + + </div> + </div> + + </section> + + </div> + +<div class="rst-versions" role="note" style="cursor: pointer"> + <span class="rst-current-version" data-toggle="rst-current-version"> + + + <span><a href="../dtcli/" style="color: #fcfcfc;">« Previous</a></span> + + + </span> +</div> + +</body> +</html> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/dtcli/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/dtcli/index.html b/content/docs/apex-3.3/dtcli/index.html index fc33b6f..bb63fab 100644 --- a/content/docs/apex-3.3/dtcli/index.html +++ b/content/docs/apex-3.3/dtcli/index.html @@ -131,6 +131,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="../compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> @@ -419,6 +426,8 @@ they must be part of the jar files that were deployed at application launch time <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + <a href="../compatibility/" class="btn btn-neutral float-right" title="Compatibility">Next <span class="icon icon-circle-arrow-right"></span></a> + <a href="../autometrics/" class="btn btn-neutral" title="AutoMetric API"><span class="icon icon-circle-arrow-left"></span> Previous</a> @@ -449,6 +458,8 @@ they must be part of the jar files that were deployed at application launch time <span><a href="../autometrics/" style="color: #fcfcfc;">« Previous</a></span> + <span style="margin-left: 15px"><a href="../compatibility/" style="color: #fcfcfc">Next »</a></span> + </span> </div> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/index.html b/content/docs/apex-3.3/index.html index 95952ca..766851f 100644 --- a/content/docs/apex-3.3/index.html +++ b/content/docs/apex-3.3/index.html @@ -127,6 +127,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> @@ -218,5 +225,5 @@ <!-- MkDocs version : 0.15.3 -Build Date UTC : 2016-03-10 00:39:42.856605 +Build Date UTC : 2016-03-19 02:17:55.330407 --> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/mkdocs/search_index.json ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/mkdocs/search_index.json b/content/docs/apex-3.3/mkdocs/search_index.json index 34dd42f..cc08c8a 100644 --- a/content/docs/apex-3.3/mkdocs/search_index.json +++ b/content/docs/apex-3.3/mkdocs/search_index.json @@ -829,6 +829,76 @@ "location": "/dtcli/#examples", "text": "An example of defining a custom macro. The macro updates a running application by inserting a new operator. It takes three parameters and executes a logical plan changes. dt begin-macro add-console-output\nmacro begin-logical-plan-change\nmacro create-operator $1 com.datatorrent.lib.io.ConsoleOutputOperator\nmacro create-stream stream_$1 $2 $3 $1 in\nmacro submit Then execute the add-console-output macro like this dt add-console-output xyz opername portname This macro then expands to run the following command begin-logical-plan-change\ncreate-operator xyz com.datatorrent.lib.io.ConsoleOutputOperator\ncreate-stream stream_xyz opername portname xyz in\nsubmit Note : To perform runtime logical plan changes, like ability to add new operators,\nthey must be part of the jar files that were deployed at application launch time.", "title": "Examples" + }, + { + "location": "/compatibility/", + "text": "Apache Apex Compatibility\n\n\nPurpose\n\n\nThis document captures the compatibility goals of the Apache Apex project. The different types of compatibility between Apex releases that affect contributors, downstream projects, and end-users are enumerated. For each type of compatibility we:\n\n\n\n\ndescribe the impact on downstream projects or end-users\n\n\nwhere applicable, call out the policy adopted when incompatible changes are permitted.\n\n\n\n\nApache Apex follows \nsemantic versioning\n. Depending on the compatibility type, there may be different tools or mechanisms to ensure compatibility, for example by comparing artifacts during the build process.\n\n\nThe type of change will inform the required target version number. Given a version number MAJOR.MINOR.PATCH, increment the:\n\n\n\n\nMAJOR version when you make incompatible API changes,\n\n\nMINOR version when you add functionality in a backward-compatible manner, and\n\n\nPATCH version when you make b ackward-compatible bug fixes.\n\n\n\n\nAdditional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.\n\n\nThe overall goal is to avoid backward incompatible changes and major release upgrades. Accordingly we attempt to release new features with minor versions that are incremental to the prior release and offer our users a frictionless upgrade path. When planning contributions, please consider compatibility and release road map upfront. Specifically, certain changes that conflict with the versioning may need to be documented in JIRA and deferred until a future major release. \n\n\nCompatibility types\n\n\nJava API\n\n\nPublic API compatibility is required to ensure end-user programs and downstream projects continue to work without modification.\nThe public API consists of:\n\n\n\n\napex-core: all interfaces and classes in \napi\n and \ncommon\n modules\n\n\napex-malhar: all interfaces and classes in all modules except \ndemos\n, \ns amples\n, \nbenchmark\n \n\n\n\n\nInterfaces and classes that are part of the public API and are annotated with \ninterface stability\n are treated according to the rules defined by the annotation. \n\n\nPolicy\n\n\nChanges to the public API must follow semantic versioning. \nPublic APIs must be deprecated for at least one minor release prior to their removal in a major release.\nThe \njapicmp Maven plugin\n is used to enforce compatibility as part of the Travis pre-commit builds.\n\n\nSemantic compatibility\n\n\nThe behavior of APIs needs to remain consistent over versions, though changes for correctness may result in changes in behavior. Tests and javadocs specify the behavior. Over time, test suites should be expanded to verify compliance with the specification, effectively creating a formal specification for the subset of behaviors that can be easily tested.\n\n\nPolicy\n\n\nThe behavior of existing API cannot be modified as it would break existing user code. There are exceptio nal circumstances that may justify such changes, in which cases they should be discussed on the mailing list before implementation. Examples are bug fixes related to security issues, data corruption/consistency or to correct an unintended change from previous release that violated semantic compatibility. Such changes should be accompanied by test coverage for the exact behavior.\n\n\nREST API\n\n\nREST API compatibility corresponds to both the URLs and request/response content over the wire. REST APIs are specifically meant for stable use by clients across releases, even major releases. \n\n\nPolicy\n\n\nThe REST API is separately versioned. This is to allow for co-existence of old and new API should there be a need for backward incompatible changes in the future.\n\n\nCommand Line Interface (CLI)\n\n\nThe CLI may be used either directly via the system shell or via shell scripts. Changing the path, removing or renaming command line options, the order of arguments, or the command ret urn code and output break compatibility and may adversely affect users.\n\n\nPolicy\n\n\nCLI commands are to be deprecated (warning when used) in a prior minor release before they are removed or incompatibly modified in a subsequent major release.\n\n\nConfiguration Files\n\n\nConfiguration files are used for engine or application settings. Changes to keys and default values directly affect users and are hard to diagnose (compared to a compile error, for example).\n\n\nPolicy\n\n\nName, location, format, keys of configuration files should be deprecated in a prior minor release and can only be changed in major release. Best effort should be made to support the deprecated behavior for one more major release (not guaranteed). It is also desirable to provide the user with a migration tool.\n\n\nInternal Wire compatibility\n\n\nApex containers internally use RPC communication and netlet for the data flow. The protocols are private and user components are not exposed to it. Apex is a YARN application and automatically deployed. There is currently no situation where containers of different Apex engine versions need to be interoperable. Should such a scenario become relevant in the future, wire compatibility needs to be specified.\n\n\nPolicy\n\n\nN/A\n\n\nInternal File formats\n\n\nApex engine stores data in the file system for recovery and the data is typically obtained from serialization (from Kryo, Java etc.). Changes to internal classes may affect the ability to relaunch an application with upgraded engine code from previous state. This is currently not supported. In the future, the serialization mechanism should guarantee backward compatibility.\n\n\nPolicy\n\n\nCurrently no compatibility guarantee. User to cold-restart application on engine upgrade.\n\n\nJava Classpath\n\n\nApex applications should not bundle Hadoop dependencies or Apex engine dependencies but use the dependencies provided in the target environment to avoid conflicts. The Apex application arche type can be used to generate a compliant project. \n\n\nPolicy\n\n\nApex engine dependencies can change as per semantic versioning. Following above guidelines automatically maintains the backward compatibility based on semantic versioning of Apex.\n\n\nMaven Build Artifacts\n\n\nDownstream projects reference the Apex engine dependencies and Malhar operator libraries for application development etc. Changes to the packaging (which classes are in which jar), the groupId, artifactId and which artifacts are deployed to Maven central impact upgrades.\n\n\nPolicy\n\n\nThe artifacts that contain the classes that form the public API as specified above cannot change in patch releases and should stay compatible within a major release. Patch releases can change dependencies, but only at the patch level and following semantic versioning.\n\n\nHardware/Software Requirements\n\n\nApex depends on Apache Hadoop. The community intends to support all major Hadoop distros and current versions. Apex c urrently supports Hadoop 2.2.0 and higher and Java 7 and higher. Apex is written in Java and has been tested on Linux based Hadoop clusters. There are no additional restrictions on the hardware architecture. \n\n\nTo keep up with the latest advances in hardware, operating systems, JVMs, Hadoop and other software, new Apex releases may require higher versions. Upgrading Apex may require upgrading other dependent software components.\n\n\nPolicy\n\n\nThe JVM and Hadoop minimum version requirements are not expected to change outside major releases.", + "title": "Compatibility" + }, + { + "location": "/compatibility/#apache-apex-compatibility", + "text": "", + "title": "Apache Apex Compatibility" + }, + { + "location": "/compatibility/#purpose", + "text": "This document captures the compatibility goals of the Apache Apex project. The different types of compatibility between Apex releases that affect contributors, downstream projects, and end-users are enumerated. For each type of compatibility we: describe the impact on downstream projects or end-users where applicable, call out the policy adopted when incompatible changes are permitted. Apache Apex follows semantic versioning . Depending on the compatibility type, there may be different tools or mechanisms to ensure compatibility, for example by comparing artifacts during the build process. The type of change will inform the required target version number. Given a version number MAJOR.MINOR.PATCH, increment the: MAJOR version when you make incompatible API changes, MINOR version when you add functionality in a backward-compatible manner, and PATCH version when you make backward-compatible bug fixes. Additional labels for pre-release and build metadat a are available as extensions to the MAJOR.MINOR.PATCH format. The overall goal is to avoid backward incompatible changes and major release upgrades. Accordingly we attempt to release new features with minor versions that are incremental to the prior release and offer our users a frictionless upgrade path. When planning contributions, please consider compatibility and release road map upfront. Specifically, certain changes that conflict with the versioning may need to be documented in JIRA and deferred until a future major release.", + "title": "Purpose" + }, + { + "location": "/compatibility/#compatibility-types", + "text": "", + "title": "Compatibility types" + }, + { + "location": "/compatibility/#java-api", + "text": "Public API compatibility is required to ensure end-user programs and downstream projects continue to work without modification.\nThe public API consists of: apex-core: all interfaces and classes in api and common modules apex-malhar: all interfaces and classes in all modules except demos , samples , benchmark Interfaces and classes that are part of the public API and are annotated with interface stability are treated according to the rules defined by the annotation. Policy Changes to the public API must follow semantic versioning. \nPublic APIs must be deprecated for at least one minor release prior to their removal in a major release.\nThe japicmp Maven plugin is used to enforce compatibility as part of the Travis pre-commit builds.", + "title": "Java API" + }, + { + "location": "/compatibility/#semantic-compatibility", + "text": "The behavior of APIs needs to remain consistent over versions, though changes for correctness may result in changes in behavior. Tests and javadocs specify the behavior. Over time, test suites should be expanded to verify compliance with the specification, effectively creating a formal specification for the subset of behaviors that can be easily tested. Policy The behavior of existing API cannot be modified as it would break existing user code. There are exceptional circumstances that may justify such changes, in which cases they should be discussed on the mailing list before implementation. Examples are bug fixes related to security issues, data corruption/consistency or to correct an unintended change from previous release that violated semantic compatibility. Such changes should be accompanied by test coverage for the exact behavior.", + "title": "Semantic compatibility" + }, + { + "location": "/compatibility/#rest-api", + "text": "REST API compatibility corresponds to both the URLs and request/response content over the wire. REST APIs are specifically meant for stable use by clients across releases, even major releases. Policy The REST API is separately versioned. This is to allow for co-existence of old and new API should there be a need for backward incompatible changes in the future.", + "title": "REST API" + }, + { + "location": "/compatibility/#command-line-interface-cli", + "text": "The CLI may be used either directly via the system shell or via shell scripts. Changing the path, removing or renaming command line options, the order of arguments, or the command return code and output break compatibility and may adversely affect users. Policy CLI commands are to be deprecated (warning when used) in a prior minor release before they are removed or incompatibly modified in a subsequent major release.", + "title": "Command Line Interface (CLI)" + }, + { + "location": "/compatibility/#configuration-files", + "text": "Configuration files are used for engine or application settings. Changes to keys and default values directly affect users and are hard to diagnose (compared to a compile error, for example). Policy Name, location, format, keys of configuration files should be deprecated in a prior minor release and can only be changed in major release. Best effort should be made to support the deprecated behavior for one more major release (not guaranteed). It is also desirable to provide the user with a migration tool.", + "title": "Configuration Files" + }, + { + "location": "/compatibility/#internal-wire-compatibility", + "text": "Apex containers internally use RPC communication and netlet for the data flow. The protocols are private and user components are not exposed to it. Apex is a YARN application and automatically deployed. There is currently no situation where containers of different Apex engine versions need to be interoperable. Should such a scenario become relevant in the future, wire compatibility needs to be specified. Policy N/A", + "title": "Internal Wire compatibility" + }, + { + "location": "/compatibility/#internal-file-formats", + "text": "Apex engine stores data in the file system for recovery and the data is typically obtained from serialization (from Kryo, Java etc.). Changes to internal classes may affect the ability to relaunch an application with upgraded engine code from previous state. This is currently not supported. In the future, the serialization mechanism should guarantee backward compatibility. Policy Currently no compatibility guarantee. User to cold-restart application on engine upgrade.", + "title": "Internal File formats" + }, + { + "location": "/compatibility/#java-classpath", + "text": "Apex applications should not bundle Hadoop dependencies or Apex engine dependencies but use the dependencies provided in the target environment to avoid conflicts. The Apex application archetype can be used to generate a compliant project. Policy Apex engine dependencies can change as per semantic versioning. Following above guidelines automatically maintains the backward compatibility based on semantic versioning of Apex.", + "title": "Java Classpath" + }, + { + "location": "/compatibility/#maven-build-artifacts", + "text": "Downstream projects reference the Apex engine dependencies and Malhar operator libraries for application development etc. Changes to the packaging (which classes are in which jar), the groupId, artifactId and which artifacts are deployed to Maven central impact upgrades. Policy The artifacts that contain the classes that form the public API as specified above cannot change in patch releases and should stay compatible within a major release. Patch releases can change dependencies, but only at the patch level and following semantic versioning.", + "title": "Maven Build Artifacts" + }, + { + "location": "/compatibility/#hardwaresoftware-requirements", + "text": "Apex depends on Apache Hadoop. The community intends to support all major Hadoop distros and current versions. Apex currently supports Hadoop 2.2.0 and higher and Java 7 and higher. Apex is written in Java and has been tested on Linux based Hadoop clusters. There are no additional restrictions on the hardware architecture. To keep up with the latest advances in hardware, operating systems, JVMs, Hadoop and other software, new Apex releases may require higher versions. Upgrading Apex may require upgrading other dependent software components. Policy The JVM and Hadoop minimum version requirements are not expected to change outside major releases.", + "title": "Hardware/Software Requirements" } ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/operator_development/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/operator_development/index.html b/content/docs/apex-3.3/operator_development/index.html index deef122..cae1749 100644 --- a/content/docs/apex-3.3/operator_development/index.html +++ b/content/docs/apex-3.3/operator_development/index.html @@ -179,6 +179,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="../compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/search.html ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/search.html b/content/docs/apex-3.3/search.html index aaad72b..6fa6c09 100644 --- a/content/docs/apex-3.3/search.html +++ b/content/docs/apex-3.3/search.html @@ -116,6 +116,13 @@ </ul> <li> + <li> + <li class="toctree-l1 "> + <a class="" href="compatibility/">Compatibility</a> + + </li> +<li> + </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex-3.3/sitemap.xml ---------------------------------------------------------------------- diff --git a/content/docs/apex-3.3/sitemap.xml b/content/docs/apex-3.3/sitemap.xml index edd34f3..262a94b 100644 --- a/content/docs/apex-3.3/sitemap.xml +++ b/content/docs/apex-3.3/sitemap.xml @@ -4,7 +4,7 @@ <url> <loc>/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> @@ -13,31 +13,31 @@ <url> <loc>/apex_development_setup/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> <url> <loc>/application_development/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> <url> <loc>/application_packages/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> <url> <loc>/operator_development/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> <url> <loc>/autometrics/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> @@ -47,10 +47,18 @@ <url> <loc>/dtcli/</loc> - <lastmod>2016-03-09</lastmod> + <lastmod>2016-03-18</lastmod> <changefreq>daily</changefreq> </url> + + <url> + <loc>/compatibility/</loc> + <lastmod>2016-03-18</lastmod> + <changefreq>daily</changefreq> + </url> + + </urlset> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex/__init__.py ---------------------------------------------------------------------- diff --git a/content/docs/apex/__init__.py b/content/docs/apex/__init__.py deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex/__init__.pyc ---------------------------------------------------------------------- diff --git a/content/docs/apex/__init__.pyc b/content/docs/apex/__init__.pyc deleted file mode 100644 index f478a23..0000000 Binary files a/content/docs/apex/__init__.pyc and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-apex-site/blob/2ea69c2b/content/docs/apex/apex_development_setup/index.html ---------------------------------------------------------------------- diff --git a/content/docs/apex/apex_development_setup/index.html b/content/docs/apex/apex_development_setup/index.html deleted file mode 100644 index 60350b2..0000000 --- a/content/docs/apex/apex_development_setup/index.html +++ /dev/null @@ -1,330 +0,0 @@ -<!DOCTYPE html> -<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> -<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> -<head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - - - - <title>Development Setup - Apache Apex Documentation</title> - - - <link rel="shortcut icon" href="../favicon.ico"> - - - - <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> - - <link rel="stylesheet" href="../css/theme.css" type="text/css" /> - <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" /> - <link rel="stylesheet" href="../css/highlight.css"> - - - <script> - // Current page data - var mkdocs_page_name = "Development Setup"; - var mkdocs_page_input_path = "apex_development_setup.md"; - var mkdocs_page_url = "/apex_development_setup/"; - </script> - - <script src="../js/jquery-2.1.1.min.js"></script> - <script src="../js/modernizr-2.8.3.min.js"></script> - <script type="text/javascript" src="../js/highlight.pack.js"></script> - <script src="../js/theme.js"></script> - - -</head> - -<body class="wy-body-for-nav" role="document"> - - <div class="wy-grid-for-nav"> - - - <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> - <div class="wy-side-nav-search"> - <a href=".." class="icon icon-home"> Apache Apex Documentation</a> - <div role="search"> - <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get"> - <input type="text" name="q" placeholder="Search docs" /> - </form> -</div> - </div> - - <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> - <ul class="current"> - - <li> - <li class="toctree-l1 "> - <a class="" href="..">Apache Apex</a> - - </li> -<li> - - <li> - <ul class="subnav"> - <li><span>Development</span></li> - - - - <li class="toctree-l1 current"> - <a class="current" href="./">Development Setup</a> - - <ul> - - <li class="toctree-l3"><a href="#apache-apex-development-environment-setup">Apache Apex Development Environment Setup</a></li> - - <li><a class="toctree-l4" href="#development-tools">Development Tools</a></li> - - <li><a class="toctree-l4" href="#creating-new-apex-project">Creating New Apex Project</a></li> - - <li><a class="toctree-l4" href="#building-apex-demos">Building Apex Demos</a></li> - - <li><a class="toctree-l4" href="#sandbox">Sandbox</a></li> - - - </ul> - - </li> - - - - <li class="toctree-l1 "> - <a class="" href="../application_development/">Applications</a> - - </li> - - - - <li class="toctree-l1 "> - <a class="" href="../application_packages/">Packages</a> - - </li> - - - - <li class="toctree-l1 "> - <a class="" href="../operator_development/">Operators</a> - - </li> - - - - <li class="toctree-l1 "> - <a class="" href="../autometrics/">AutoMetric API</a> - - </li> - - - </ul> -<li> - - <li> - <ul class="subnav"> - <li><span>Operations</span></li> - - - - <li class="toctree-l1 "> - <a class="" href="../dtcli/">dtCli</a> - - </li> - - - </ul> -<li> - - </ul> - </div> - - </nav> - - <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> - - - <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="..">Apache Apex Documentation</a> - </nav> - - - <div class="wy-nav-content"> - <div class="rst-content"> - <div role="navigation" aria-label="breadcrumbs navigation"> - <ul class="wy-breadcrumbs"> - <li><a href="..">Docs</a> »</li> - - - - <li>Development »</li> - - - - <li>Development Setup</li> - <li class="wy-breadcrumbs-aside"> - - </li> - </ul> - <hr/> -</div> - <div role="main"> - <div class="section"> - - <h1 id="apache-apex-development-environment-setup">Apache Apex Development Environment Setup</h1> -<p>This document discusses the steps needed for setting up a development environment for creating applications that run on the Apache Apex platform.</p> -<h2 id="development-tools">Development Tools</h2> -<p>There are a few tools that will be helpful when developing Apache Apex applications, including:</p> -<ol> -<li> -<p><strong>git</strong> - A revision control system (version 1.7.1 or later). There are multiple git clients available for Windows (<a href="http://git-scm.com/download/win";>http://git-scm.com/download/win</a> for example), so download and install a client of your choice.</p> -</li> -<li> -<p><strong>java JDK</strong> (not JRE) - Includes the Java Runtime Environment as well as the Java compiler and a variety of tools (version 1.7.0_79 or later). Can be downloaded from the Oracle website.</p> -</li> -<li> -<p><strong>maven</strong> - Apache Maven is a build system for Java projects (version 3.0.5 or later). It can be downloaded from <a href="https://maven.apache.org/download.cgi";>https://maven.apache.org/download.cgi</a>.</p> -</li> -<li> -<p><strong>IDE</strong> (Optional) - If you prefer to use an IDE (Integrated Development Environment) such as <em>NetBeans</em>, <em>Eclipse</em> or <em>IntelliJ</em>, install that as well.</p> -</li> -</ol> -<p>After installing these tools, make sure that the directories containing the executable files are in your PATH environment variable.</p> -<ul> -<li><strong>Windows</strong> - Open a console window and enter the command <code>echo %PATH%</code> to see the value of the <code>PATH</code> variable and verify that the above directories for Java, git, and maven executables are present. JDK executables like <em>java</em> and <em>javac</em>, the directory might be something like <code>C:\\Program Files\\Java\\jdk1.7.0\_80\\bin</code>; for <em>git</em> it might be <code>C:\\Program Files\\Git\\bin</code>; and for maven it might be <code>C:\\Users\\user\\Software\\apache-maven-3.3.3\\bin</code>. If not, you can change its value clicking on the button at <em>Control Panel</em> ⇨ <em>Advanced System Settings</em> ⇨ <em>Advanced tab</em> ⇨ <em>Environment Variables</em>.</li> -<li><strong>Linux and Mac</strong> - Open a console/terminal window and enter the command <code>echo $PATH</code> to see the value of the <code>PATH</code> variable and verify that the above directories for Java, git, and maven executables are present. If not, make sure software is downloaded and installed, and optionally PATH reference is added and exported in a <code>~/.profile</code> or <code>~/.bash_profile</code>. For example to add maven located in <code>/sfw/maven/apache-maven-3.3.3</code> to PATH add the line: <code>export PATH=$PATH:/sfw/maven/apache-maven-3.3.3/bin</code></li> -</ul> -<p>Confirm by running the following commands and comparing with output that show in the table below:</p> -<table> -<colgroup> -<col width="30%" /> -<col width="70%" /> -</colgroup> -<tbody> -<tr class="odd"> -<td align="left"><p>Command</p></td> -<td align="left"><p>Output</p></td> -</tr> -<tr class="even"> -<td align="left"><p><tt>javac -version</tt></p></td> -<td align="left"><p>javac 1.7.0_80</p></td> -</tr> -<tr class="odd"> -<td align="left"><p><tt>java -version</tt></p></td> -<td align="left"><p>java version "1.7.0_80"</p> -<p>Java(TM) SE Runtime Environment (build 1.7.0_80-b15)</p> -<p>Java HotSpot(TM) 64-Bit Server VM (build 24.80-b11, mixed mode)</p></td> -</tr> -<tr class="even"> -<td align="left"><p><tt>git --version</tt></p></td> -<td align="left"><p>git version 2.6.1.windows.1</p></td> -</tr> -<tr class="odd"> -<td align="left"><p><tt>mvn --version</tt></p></td> -<td align="left"><p>Apache Maven 3.3.3 (7994120775791599e205a5524ec3e0dfe41d4a06; 2015-04-22T06:57:37-05:00)</p> -<p>...</p> -</td> -</tr> -</tbody> -</table> - -<h2 id="creating-new-apex-project">Creating New Apex Project</h2> -<p>After development tools are configured, you can now use the maven archetype to create a basic Apache Apex project. <strong>Note:</strong> When executing the commands below, replace <code>3.3.0-incubating</code> by <a href="http://apex.apache.org/downloads.html";>latest available version</a> of Apache Apex.</p> -<ul> -<li> -<p><strong>Windows</strong> - Create a new Windows command file called <code>newapp.cmd</code> by copying the lines below, and execute it. When you run this file, the properties will be displayed and you will be prompted with <code>Y: :</code>; just press <strong>Enter</strong> to complete the project generation. The caret (^) at the end of some lines indicates that a continuation line follows. </p> -<pre><code>@echo off -@rem Script for creating a new application -setlocal -mvn archetype:generate ^ - -DarchetypeGroupId=org.apache.apex ^ - -DarchetypeArtifactId=apex-app-archetype -DarchetypeVersion=3.3.0-incubating ^ - -DgroupId=com.example -Dpackage=com.example.myapexapp -DartifactId=myapexapp ^ - -Dversion=1.0-SNAPSHOT -endlocal -</code></pre> -</li> -<li> -<p><strong>Linux</strong> - Execute the lines below in a terminal window. New project will be created in the curent working directory. The backslash (\) at the end of the lines indicates continuation.</p> -<pre><code>mvn archetype:generate \ - -DarchetypeGroupId=org.apache.apex \ - -DarchetypeArtifactId=apex-app-archetype -DarchetypeVersion=3.2.0-incubating \ - -DgroupId=com.example -Dpackage=com.example.myapexapp -DartifactId=myapexapp \ - -Dversion=1.0-SNAPSHOT -</code></pre> -</li> -</ul> -<p>When the run completes successfully, you should see a new directory named <code>myapexapp</code> containing a maven project for building a basic Apache Apex application. It includes 3 source files:<strong>Application.java</strong>, <strong>RandomNumberGenerator.java</strong> and <strong>ApplicationTest.java</strong>. You can now build the application by stepping into the new directory and running the maven package command:</p> -<pre><code>cd myapexapp -mvn clean package -DskipTests -</code></pre> -<p>The build should create the application package file <code>myapexapp/target/myapexapp-1.0-SNAPSHOT.apa</code>. This application package can then be used to launch example application via <strong>dtCli</strong>, or other visual management tools. When running, this application will generate a stream of random numbers and print them out, each prefixed by the string <code>hello world:</code>.</p> -<h2 id="building-apex-demos">Building Apex Demos</h2> -<p>If you want to see more substantial Apex demo applications and the associated source code, you can follow these simple steps to check out and build them.</p> -<ol> -<li> -<p>Check out the source code repositories:</p> -<pre><code>git clone https://github.com/apache/incubator-apex-core -git clone https://github.com/apache/incubator-apex-malhar -</code></pre> -</li> -<li> -<p>Switch to the appropriate release branch and build each repository:</p> -<pre><code>cd incubator-apex-core -mvn clean install -DskipTests - -cd incubator-apex-malhar -mvn clean install -DskipTests -</code></pre> -</li> -</ol> -<p>The <code>install</code> argument to the <code>mvn</code> command installs resources from each project to your local maven repository (typically <code>.m2/repository</code> under your home directory), and <strong>not</strong> to the system directories, so Administrator privileges are not required. The <code>-DskipTests</code> argument skips running unit tests since they take a long time. If this is a first-time installation, it might take several minutes to complete because maven will download a number of associated plugins.</p> -<p>After the build completes, you should see the demo application package files in the target directory under each demo subdirectory in <code>incubator-apex-malhar/demos</code>.</p> -<h2 id="sandbox">Sandbox</h2> -<p>To jump start development with an Apache Hadoop single node cluster, <a href="https://www.datatorrent.com/download";>DataTorrent Sandbox</a> powered by VirtualBox is available on Windows, Linux, or Mac platforms. The sandbox is configured by default to run with 6GB RAM; if your development machine has 16GB or more, you can increase the sandbox RAM to 8GB or more using the VirtualBox console. This will yield better performance and support larger applications. The advantage of developing in the sandbox is that most of the tools (e.g. <em>jdk</em>, <em>git</em>, <em>maven</em>), Hadoop YARN and HDFS, and a distribution of Apache Apex and DataTorrent RTS are pre-installed. The disadvantage is that the sandbox is a memory-limited environment, and requires settings changes and restarts to adjust memory available for development and testing.</p> - - </div> - </div> - <footer> - - <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> - - <a href="../application_development/" class="btn btn-neutral float-right" title="Applications">Next <span class="icon icon-circle-arrow-right"></span></a> - - - <a href=".." class="btn btn-neutral" title="Apache Apex"><span class="icon icon-circle-arrow-left"></span> Previous</a> - - </div> - - - <hr/> - - <div role="contentinfo"> - <!-- Copyright etc --> - - </div> - - Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. -</footer> - - </div> - </div> - - </section> - - </div> - -<div class="rst-versions" role="note" style="cursor: pointer"> - <span class="rst-current-version" data-toggle="rst-current-version"> - - - <span><a href=".." style="color: #fcfcfc;">« Previous</a></span> - - - <span style="margin-left: 15px"><a href="../application_development/" style="color: #fcfcfc">Next »</a></span> - - </span> -</div> - -</body> -</html>
