Add release audit capability.

From: Aaron Kimball <aaron@cloudera.com>

git-svn-id: https://svn.apache.org/repos/asf/incubator/sqoop/trunk@1149923 13f79535-47bb-0310-9956-ffa450edef68

COMPILING.txt

@@ -3,6 +3,24 @@
 
 This document explains how to compile Sqoop.
 
+////
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+////
+
+
 == Build Dependencies
 
 Compiling Sqoop requires the following tools:

build.xml

@@ -58,6 +58,10 @@
   <property name="test.log.dir" location="${build.dir}/test/logs"/>
   <property name="dist.dir" location="${build.dir}/${artifact.name}" />
   <property name="tar.file" location="${build.dir}/${artifact.name}.tar.gz" />
+  <property name="build.docs.timestamp"
+      location="${build.dir}/docs.timestamp" />
+  <property name="build.webdocs.timestamp"
+      location="${build.dir}/webdocs.timestamp" />
 
   <!-- compilation -->
   <property name="javac.deprecation" value="off"/>
@@ -96,6 +100,8 @@
       value="${test.dir}/checkstyle-noframes.xsl" />
   <property name="checkstyle.report.dir" value="${build.dir}" />
 
+  <!-- Release audit -->
+  <property name="rat.reporting.classname" value="rat.Report"/>
 
   <!-- When testing with non-free JDBC drivers, override this parameter
        to contain the path to the driver library dir.
@@ -554,7 +560,43 @@
     </junit>
   </target>
 
-  <target name="webdocs" description="Build website documentation">
+  <target name="releaseaudit" depends="package,ivy-retrieve-releaseaudit"
+      description="Audit license headers for release">
+    <fail unless="rat.present"
+        message="Failed to load class [${rat.reporting.classname}]." />
+    <java classname="${rat.reporting.classname}" fork="true" 
+        output="${build.dir}/rat.log">
+      <classpath refid="${name}.releaseaudit.classpath" />
+      <arg value="${dist.dir}" />
+    </java>
+    <exec executable="${script.src.dir}/rat-violations.sh" failOnError="true">
+        <arg value="${build.dir}/rat.log" />
+        <arg value="${dist.dir}" />
+    </exec>
+    <echo message="Release audit appears okay. Full results are in " />
+    <echo message="${build.dir}/rat.log" />
+  </target>
+
+  <target name="docs-uptodate" depends="init">
+    <uptodate property="docs.uptodate">
+      <srcfiles dir="${basedir}/src/docs/">
+        <include name="**/*.txt" />
+      </srcfiles>
+      <mapper type="merge" to="${build.docs.timestamp}" />
+    </uptodate>
+  </target>
+
+  <target name="webdocs-uptodate" depends="init">
+    <uptodate property="webdocs.uptodate">
+      <srcfiles dir="${basedir}/src/docs/">
+        <include name="**/*.txt" />
+      </srcfiles>
+      <mapper type="merge" to="${build.webdocs.timestamp}" />
+    </uptodate>
+  </target>
+
+  <target name="webdocs" depends="webdocs-uptodate" unless="webdocs.uptodate"
+      description="Build website documentation">
     <exec executable="make" failonerror="true">
       <arg value="-C" />
       <arg value="${basedir}/src/docs" />
@@ -562,15 +604,18 @@
       <arg value="VERSION=${version}" />
       <arg value="website" />
     </exec>
+    <touch file="${build.webdocs.timestamp}" />
   </target>
 
-  <target name="docs" description="Build documentation">
+  <target name="docs" depends="docs-uptodate" unless="docs.uptodate"
+      description="Build documentation">
     <exec executable="make" failonerror="true">
       <arg value="-C" />
       <arg value="${basedir}/src/docs" />
       <arg value="BUILDROOT=${build.dir}" />
       <arg value="VERSION=${version}" />
     </exec>
+    <touch file="${build.docs.timestamp}" />
   </target>
 
   <target name="javadoc-uptodate" depends="init">
@@ -813,4 +858,17 @@
     <ivy:cachepath pathid="${name}.checkstyle.classpath" conf="checkstyle" />
   </target>
 
+  <!-- retrieve ivy-managed artifacts for releaseaudit -->
+  <target name="ivy-resolve-releaseaudit" depends="ivy-init">
+    <ivy:resolve settingsRef="${name}.ivy.settings" conf="releaseaudit" />
+  </target>
+  <target name="ivy-retrieve-releaseaudit" depends="ivy-resolve-releaseaudit">
+    <ivy:retrieve settingsRef="${name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" sync="true" />
+    <ivy:cachepath pathid="${name}.releaseaudit.classpath" conf="releaseaudit" />
+    <available classname="${rat.reporting.classname}"
+        classpathref="${name}.releaseaudit.classpath" property="rat.present"
+        value="true" />
+  </target>
+
 </project>

ivy.xml

@@ -61,6 +61,9 @@
 
     <!-- configuration for running checkstyle -->
     <conf name="checkstyle" visibility="private" />
+
+    <!-- configuration for running rat -->
+    <conf name="releaseaudit" visibility="private" />
   </configurations>
 
   <publications>
@@ -119,5 +122,13 @@
     <!-- dependencies for static analysis -->
     <dependency org="checkstyle" name="checkstyle" rev="${checkstyle.version}"
       conf="checkstyle->default" />
+
+    <dependency org="com.google.code.p.arat" name="rat-lib"
+          rev="${rats-lib.version}" conf="releaseaudit->default" />
+    <dependency org="commons-lang" name="commons-lang"
+      rev="${commons-lang.version}" conf="releaseaudit->default"/>
+    <dependency org="commons-collections" name="commons-collections"
+      rev="${commons-collections.version}" conf="releaseaudit->default"/>
+
   </dependencies>
 </ivy-module>

ivy/libraries.properties

@@ -19,7 +19,9 @@
 checkstyle.version=5.0
 
 commons-cli.version=1.2
+commons-collections.version=3.1
 commons-io.version=1.4
+commons-lang.version=2.4
 commons-logging.version=1.0.4
 
 # Apache Hadoop dependency version: use trunk.
@@ -43,3 +45,5 @@ junit.version=4.5
 
 log4j.version=1.2.15
 
+rats-lib.version=0.5.1
+

lib/.gitignore

@@ -1 +1,15 @@
+#   Licensed to Cloudera, Inc. under one or more
+#   contributor license agreements.  See the NOTICE file distributed with
+#   this work for additional information regarding copyright ownership.
+#   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+#   (the "License"); you may not use this file except in compliance with
+#   the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
 /ivy-*.jar

src/docs/man/import-args.txt

@@ -1,4 +1,22 @@
 
+////
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+////
+
+
 Import control options
 ~~~~~~~~~~~~~~~~~~~~~~
 

src/docs/web/breadcrumbs.xsl

@@ -3,6 +3,23 @@
                 version="1.0"
                 exclude-result-prefixes="exsl">
 
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
 <xsl:template name="breadcrumbs">
   <xsl:param name="this.node" select="."/>
   <div class="breadcrumbs">

src/docs/web/docbook.css

@@ -1,4 +1,25 @@
 /*
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ *
+ *
+ * This file was provided to this project under these terms:
+ *
  * Copyright (c) 2001, 2003 The FreeBSD Documentation Project
  * All rights reserved.
  *

src/docs/web/footer.xsl

@@ -3,6 +3,24 @@
                 version="1.0"
                 exclude-result-prefixes="exsl">
 
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+
 <xsl:template name="user.footer.content">
 
   <script type="text/javascript">

src/docs/web/header.xsl

@@ -3,6 +3,24 @@
                 version="1.0"
                 exclude-result-prefixes="exsl">
 
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+
 <xsl:import href="breadcrumbs.xsl"/>
 
 <xsl:template name="user.head.content">

src/docs/web/html.xsl

@@ -1,5 +1,23 @@
 <?xml version='1.0'?>
 
+
+<!--
+   Licensed to Cloudera, Inc. under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
 <!-- $FreeBSD: doc/share/xsl/freebsd-html.xsl,v 1.1 2003/01/03 05:06:14 trhodes Exp $ -->
 
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"

src/scripts/rat-violations.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Licensed to Cloudera, Inc. under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# rat-violations.sh
+#   Given an audit report generated by Apache RAT, determine which violations
+#   may be genuine.
+#
+# Arguments:
+#   rat-violations.sh <audit-log-file> <basedir>
+#
+# audit-log-file should the filename containing the output of RAT.
+# basedir should be the base directory where the audit was run.
+
+auditlog=$1
+basedir=$2
+
+if [ ! -f "${auditlog}" ]; then
+  echo "Could not read audit log: ${auditlog}"
+  exit 1
+fi
+
+auditbase=`dirname $auditlog`
+filtered=${auditbase}/filtered-release-audit.log
+
+sed -i -e "s|${basedir}||" ${auditlog}
+
+# Exclude paths that don't count.
+# Anything in /docs is auto-generated.
+# Anything in /testdata is a file that is supposed to represent exact output.
+grep '!?????' ${auditlog} \
+    | grep -v ' \/docs\/' \
+    | grep -v ' \/testdata\/' \
+    > ${filtered}
+
+# Check: did we find any violations after filtering?
+grep '!?????' ${filtered}
+status=$?
+
+if [ "$status" == "0" ]; then
+  # We found something that looks genuine.
+  echo Possible violations found.
+  echo See ${filtered}
+  exit 1
+fi
+
+
+
+