Repository: incubator-samza Updated Branches: refs/heads/master dd13c3d20 -> 9652e6412
SAMZA-285: Use Apache Rat for release audit tests Project: http://git-wip-us.apache.org/repos/asf/incubator-samza/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-samza/commit/9652e641 Tree: http://git-wip-us.apache.org/repos/asf/incubator-samza/tree/9652e641 Diff: http://git-wip-us.apache.org/repos/asf/incubator-samza/diff/9652e641 Branch: refs/heads/master Commit: 9652e6412f668bcd738b02caf77fe685fd56b46f Parents: dd13c3d Author: David Chen <[email protected]> Authored: Wed Jul 9 13:06:56 2014 -0700 Committer: Jakob Homan <[email protected]> Committed: Wed Jul 9 13:06:56 2014 -0700 ---------------------------------------------------------------------- build.gradle | 53 ++++- .../documentation/0.7.0/jobs/reprocessing.md | 16 ++ gradle/buildscript.gradle | 6 - gradle/license.gradle | 32 --- gradle/rat.gradle | 109 ++++++++++ gradle/resources/rat-output-to-html.xsl | 206 +++++++++++++++++++ 6 files changed, 377 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index f1a458b..c853fc0 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + buildscript { repositories { mavenCentral() @@ -35,11 +36,50 @@ allprojects { } } -apply from: file('gradle/license.gradle') apply from: file('gradle/environment.gradle') apply from: file("gradle/dependency-versions.gradle") apply from: file("gradle/dependency-versions-scala-" + scalaVersion + ".gradle") apply from: file("gradle/release.gradle") +apply from: file("gradle/rat.gradle") + +rat { + excludes = [ + '**/.git/**', + '**/.gradle/**', + '**/.project', + '**/.settings', + '**/*.iml', + '**/*.iws', + '**/*.ipr', + 'gradle/wrapper/**', + '.reviewboardrc', + 'gradlew', + 'README.md', + 'RELEASE.md', + '**/.DS_Store/**', + '**/.gitignore', + '**/build/**', + 'docs/Gemfile.lock', + 'docs/sitemap.xml', + 'docs/_site/**', + 'samza-test/state/mystore/**', + '**/bootstrap.min.js', + '**/jquery-1.11.1.min.js', + '**/jquery.tablesorter.min.js', + '**/bootstrap.css.map', + '**/bootstrap.min.css', + '**/ropa-sans.css', + '**/font-awesome.min.css', + '**/syntax.css', + '**/non-responsive.less', + '**/*.graffle', + '**/*.otf', + '**/*.woff', + '**/*.eot', + '**/*.svg', + '**/*.ttf' + ] +} allprojects { apply plugin: 'idea' @@ -200,8 +240,8 @@ project(":samza-shell") { gradleShell "log4j:log4j:1.2.16" } - task shellTarGz(type: Tar) { - compression = Compression.GZIP + task shellTarGz(type: Tar) { + compression = Compression.GZIP classifier = 'dist' from 'src/main/bash' from 'src/main/resources' @@ -284,16 +324,15 @@ project(":samza-test_$scalaVersion") { } test { - // Bump up the heap so we can start ZooKeeper and Kafka brokers. Also - // required for TestSamzaContainerPerformance when a high thread count + // Bump up the heap so we can start ZooKeeper and Kafka brokers. Also + // required for TestSamzaContainerPerformance when a high thread count // with a lot of inputs is used. minHeapSize = "4096m" maxHeapSize = "4096m" jvmArgs = ["-XX:+UseConcMarkSweepGC", "-server", "-d64"] - // Forward all samza.* system properties to test subprocesses. This is + // Forward all samza.* system properties to test subprocesses. This is // useful for configuring TestSamzaContainerPerformance from the CLI. systemProperties = System.properties.findAll { it.key.startsWith("samza") } } } - http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/docs/learn/documentation/0.7.0/jobs/reprocessing.md ---------------------------------------------------------------------- diff --git a/docs/learn/documentation/0.7.0/jobs/reprocessing.md b/docs/learn/documentation/0.7.0/jobs/reprocessing.md index 4c92ef9..28d9925 100644 --- a/docs/learn/documentation/0.7.0/jobs/reprocessing.md +++ b/docs/learn/documentation/0.7.0/jobs/reprocessing.md @@ -2,6 +2,22 @@ layout: page title: Reprocessing previously processed data --- +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> From time to time you may want to deploy a new version of your Samza job that computes results differently. Perhaps you fixed a bug or introduced a new feature. For example, say you have a Samza job that classifies messages as spam or not-spam, using a machine learning model that you train offline. Periodically you want to deploy an updated version of your Samza job which includes the latest classification model. http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/buildscript.gradle ---------------------------------------------------------------------- diff --git a/gradle/buildscript.gradle b/gradle/buildscript.gradle index 739a9c1..a2bdb35 100644 --- a/gradle/buildscript.gradle +++ b/gradle/buildscript.gradle @@ -18,11 +18,6 @@ */ repositories { repositories { - // For license plugin. - maven { - url 'http://dl.bintray.com/content/netflixoss/external-gradle-plugins/' - } - // For gradle-nexus-plugin maven { url 'http://jcenter.bintray.com/' @@ -31,7 +26,6 @@ repositories { } dependencies { - classpath 'nl.javadude.gradle.plugins:license-gradle-plugin:0.6.1' classpath 'de.obqo.gradle:gradle-lesscss-plugin:1.0-1.3.3' classpath 'org.gradle.api.plugins:gradle-nexus-plugin:0.7.1' } http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/license.gradle ---------------------------------------------------------------------- diff --git a/gradle/license.gradle b/gradle/license.gradle deleted file mode 100644 index 82aca21..0000000 --- a/gradle/license.gradle +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -subprojects { - apply plugin: 'license' - - license { - header rootProject.file('HEADER') - // Skip Twitter bootstrap JS and CSS. - skipExistingHeaders = true - mapping { - // Treat integration scripts (with .samsa extension) as - // regular scripts, as they use the same comment style - samsa='SCRIPT_STYLE' - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/rat.gradle ---------------------------------------------------------------------- diff --git a/gradle/rat.gradle b/gradle/rat.gradle new file mode 100644 index 0000000..8c8ac5e --- /dev/null +++ b/gradle/rat.gradle @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.internal.project.IsolatedAntBuilder + +apply plugin: RatPlugin + +class RatTask extends DefaultTask { + @Input + List<String> excludes + + def reportPath = 'build/rat' + def stylesheet = 'gradle/resources/rat-output-to-html.xsl' + def xmlReport = reportPath + '/rat-report.xml' + def htmlReport = reportPath + '/rat-report.html' + + def generateXmlReport(File reportDir) { + def antBuilder = services.get(IsolatedAntBuilder) + def ratClasspath = project.configurations.rat + antBuilder.withClasspath(ratClasspath).execute { + ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml') + ant.report(format: 'xml', reportFile: xmlReport) { + fileset(dir: ".") { + patternset { + excludes.each { + exclude(name: it) + } + } + } + } + } + } + + def printUnknownFiles() { + def ratXml = new XmlParser().parse(xmlReport) + ratXml.resource.each { resource -> + if (resource.'license-approval'.@name[0] == "false") { + println('Unknown license: ' + resource.@name) + } + } + } + + def generateHtmlReport() { + def antBuilder = services.get(IsolatedAntBuilder) + def ratClasspath = project.configurations.rat + antBuilder.withClasspath(ratClasspath).execute { + ant.xslt( + in: xmlReport, + style: stylesheet, + out: htmlReport, + classpath: ratClasspath) + } + println('Rat report: ' + htmlReport) + } + + @TaskAction + def rat() { + File reportDir = new File(reportPath) + if (!reportDir.exists()) { + reportDir.mkdirs() + } + generateXmlReport(reportDir) + printUnknownFiles() + generateHtmlReport() + } +} + +class RatPlugin implements Plugin<Project> { + void apply(Project project) { + configureDependencies(project) + project.plugins.apply(JavaBasePlugin); + Task ratTask = project.task("rat", + type: RatTask, + group: 'Build', + description: 'Runs Apache Rat checks.') + project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn ratTask + } + + void configureDependencies(final Project project) { + project.configurations { + rat + } + project.repositories { + mavenCentral() + } + project.dependencies { + rat 'org.apache.rat:apache-rat-tasks:0.8' + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/9652e641/gradle/resources/rat-output-to-html.xsl ---------------------------------------------------------------------- diff --git a/gradle/resources/rat-output-to-html.xsl b/gradle/resources/rat-output-to-html.xsl new file mode 100644 index 0000000..97ea7a1 --- /dev/null +++ b/gradle/resources/rat-output-to-html.xsl @@ -0,0 +1,206 @@ +<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="2.0"> + +<!--*********************************************************** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + ***********************************************************--> + +<!-- This style sheet converts any rat-report.xml file. --> + +<xsl:template match="/"> + + <html> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <style type="text/css"> + <!-- +body {margin-top: 0px;font-size: 0.8em;background-color: #F9F7ED;} + +h1 {color:red;} +h2 {color:blue;} +h3 {color:green;} +h4 {color:orange;} + +/* Table Design */ + +table,tr,td {text-align:center;font-weight:bold;border:1px solid #000;} +caption {color:blue;text-align:left;} +.notes, .binaries, .archives, .standards {width:25%;} +.notes {background:#D7EDEE;} +.binaries {background:#D0F2F4;} +.archives {background:#ABE7E9;} +.standards {background:#A0F0F4;} +.licenced, .generated {width:50%;} +.licenced {background:#C6EBDD;} +.generated {background:#ABE9D2;} +.java_note {background:#D6EBC6;} +.generated_note {background:#C9E7A9;} +.unknown {width:100%;background:#E92020;} +.unknown-zero {color:#00CC00;} +.center{text-align:center;margin:0 auto;} +--> + </style> + </head> + <body> + <xsl:apply-templates/> + <xsl:call-template name="generated"/> + </body> + </html> +</xsl:template> + +<xsl:template match="rat-report"> + + <h1>Rat Report</h1> + <p>This HTML version (yes, it is!) is generated from the RAT xml reports using Saxon9B. All the outputs required are displayed below, similar to the .txt version. + This is obviously a work in progress; and a prettier, easier to read and manage version will be available soon</p> +<div class="center"> +<table id="rat-reports summary" cellspacing="0" summary="A snapshot summary of this rat report"> +<caption> +Table 1: A snapshot summary of this rat report. +</caption> + <tr> + <td colspan="1" class="notes">Notes: <xsl:value-of select="count(descendant::type[attribute::name="notice"])"/></td> + <td colspan="1" class="binaries">Binaries: <xsl:value-of select="count(descendant::type[attribute::name="binary"])"/></td> + <td colspan="1" class="archives">Archives: <xsl:value-of select="count(descendant::type[attribute::name="archive"])"/></td> + <td colspan="1" class="standards">Standards: <xsl:value-of select="count(descendant::type[attribute::name="standard"])"/></td> + </tr> + <tr> + <td colspan="2" class="licenced">Apache Licensed: <xsl:value-of select="count(descendant::header-type[attribute::name="AL "])"/></td> + <td colspan="2" class="generated">Generated Documents: <xsl:value-of select="count(descendant::header-type[attribute::name="GEN "])"/></td> + </tr> + <tr> + <td colspan="2" class="java_note">Note: JavaDocs are generated and so license header is optional</td> + <td colspan="2" class="generated_note">Note: Generated files do not require license headers</td> + </tr> + <tr> +<xsl:choose> + <xsl:when test="count(descendant::header-type[attribute::name="?????"]) > 0"> + <td colspan="4" class="unknown"><xsl:value-of select="count(descendant::header-type[attribute::name="?????"])"/> Unknown Licenses - or files without a license.</td> + </xsl:when> + <xsl:otherwise> + <td colspan="4" class="unknown-zero"><xsl:value-of select="count(descendant::header-type[attribute::name="?????"])"/> Unknown Licenses - or files without a license.</td> + </xsl:otherwise> +</xsl:choose> + </tr> +</table> +</div> +<hr/> + <h3>Unapproved Licenses:</h3> + + <xsl:for-each select="descendant::resource[license-approval/@name="false"]"> + <xsl:text> </xsl:text> + <xsl:value-of select="@name"/><br/> + <xsl:text> +</xsl:text> +</xsl:for-each> +<hr/> + +<h3>Archives:</h3> + +<xsl:for-each select="descendant::resource[type/@name="archive"]"> + + <xsl:value-of select="@name"/> + <br/> + </xsl:for-each> + <hr/> + + <p> + Files with Apache License headers will be marked AL<br/> + Binary files (which do not require AL headers) will be marked B<br/> + Compressed archives will be marked A<br/> + Notices, licenses etc will be marked N<br/> + </p> + + <xsl:for-each select="descendant::resource"> + <xsl:choose> + <xsl:when test="license-approval/@name="false"">!</xsl:when> + <xsl:otherwise><xsl:text> </xsl:text></xsl:otherwise> + </xsl:choose> + <xsl:choose> + <xsl:when test="type/@name="notice"">N </xsl:when> + <xsl:when test="type/@name="archive"">A </xsl:when> + <xsl:when test="type/@name="binary"">B </xsl:when> + <xsl:when test="type/@name="standard""><xsl:value-of select="header-type/@name"/></xsl:when> + <xsl:otherwise>!!!!!</xsl:otherwise> + </xsl:choose> + <xsl:text> </xsl:text> + <xsl:value-of select="@name"/><br/> + <xsl:text> + </xsl:text> + </xsl:for-each> + <hr/> + + <h3>Printing headers for files without AL header...</h3> + + <xsl:for-each select="descendant::resource[header-type/@name="?????"]"> + + <h4><xsl:value-of select="@name"/></h4> + <xsl:value-of select="header-sample"/> + <hr/> +</xsl:for-each> +<br/> + + <!-- <xsl:apply-templates select="resource"/> + <xsl:apply-templates select="header-sample"/> + <xsl:apply-templates select="header-type"/> + <xsl:apply-templates select="license-family"/> + <xsl:apply-templates select="license-approval"/> + <xsl:apply-templates select="type"/> --> + +</xsl:template> + +<xsl:template match="resource"> + <div> + <h3>Resource: <xsl:value-of select="@name"/></h3> + <xsl:apply-templates/> + </div> +</xsl:template> + +<xsl:template match="header-sample"> + <xsl:if test="normalize-space(.) != ''"> + <h4>First few lines of non-compliant file</h4> + <p> + <xsl:value-of select="."/> + </p> + </xsl:if> + <h4>Other Info:</h4> +</xsl:template> + +<xsl:template match="header-type"> + Header Type: <xsl:value-of select="@name"/> + <br/> +</xsl:template> + +<xsl:template match="license-family"> + License Family: <xsl:value-of select="@name"/> + <br/> +</xsl:template> + +<xsl:template match="license-approval"> + License Approval: <xsl:value-of select="@name"/> + <br/> +</xsl:template> + +<xsl:template match="type"> + Type: <xsl:value-of select="@name"/> + <br/> +</xsl:template> + +<xsl:template name="generated"> +</xsl:template> +</xsl:transform>
