SQOOP-2827: Sqoop2: Doc: Provide doc preprocessor facilities

(Jarek Jarcec Cecho via Colin Ma)

docs/pom.xml

@@ -32,6 +32,12 @@ limitations under the License.
   <name>Sqoop Documentation</name>
 
   <dependencies>
+    <!-- To get all libraries and all connector on classpath, they will be used for doc generation -->
+    <dependency>
+      <groupId>org.apache.sqoop</groupId>
+      <artifactId>sqoop-server</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.testng</groupId>
       <artifactId>testng</artifactId>
@@ -47,34 +53,66 @@ limitations under the License.
       </activation>
       <build>
         <plugins>
+          <!-- Preprocess documentation - generate dynamic pages and such -->
           <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
+            <groupId>org.codehaus.mojo</groupId>
-            <artifactId>maven-site-plugin</artifactId>
+            <artifactId>exec-maven-plugin</artifactId>
-
+            <configuration>
-            <!-- Configure generating documentation alongside with package -->
+              <mainClass>org.apache.sqoop.docs.generator.DocPreprocessor</mainClass>
+              <arguments>
+                <argument>${project.basedir}/src/site/sphinx/</argument>
+                <argument>${project.build.directory}/docs-preprocessor/</argument>
+              </arguments>
+            </configuration>
             <executions>
               <execution>
-                <id>packaging-documentation</id>
+                <id>generate-docs-package</id>
                 <phase>package</phase>
                 <goals>
-                  <goal>site</goal>
+                  <goal>java</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>generate-docs-site</id>
+                <phase>site</phase>
+                <goals>
+                  <goal>java</goal>
                 </goals>
               </execution>
             </executions>
-
           </plugin>
-        </plugins>
+
-      </build>
+          <!-- Generate documentation using sphinx and reStructuredText -->
-      <reporting>
-        <plugins>
           <plugin>
             <groupId>org.tomdz.maven</groupId>
             <artifactId>sphinx-maven-plugin</artifactId>
             <version>1.0.3</version>
             <configuration>
+              <!-- Configuration options available at http://tomdz.github.io/sphinx-maven/configuration.html -->
               <warningsAsErrors>true</warningsAsErrors>
+              <sourceDirectory>${project.build.directory}/docs-preprocessor/</sourceDirectory>
             </configuration>
+            <executions>
+              <execution>
+                <id>sphinx-package</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>generate</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>sphinx-site</id>
+                <phase>site</phase>
+                <goals>
+                  <goal>generate</goal>
+                </goals>
+              </execution>
+            </executions>
           </plugin>
+        </plugins>
+      </build>
+      <reporting>
+        <plugins>
           <!-- Turning off standard reports as they collide with sphinx -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>

docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java

@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.docs.generator;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.docs.generator.plugins.AbstractPlugin;
+import org.apache.sqoop.docs.generator.plugins.CopySourceToDestination;
+import org.apache.sqoop.utils.ClassUtils;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Documentation preprocessing that generates some of the RST files dynamically.
+ */
+public class DocPreprocessor {
+
+  private static final Logger LOG = Logger.getLogger(DocPreprocessor.class);
+
+  private static List<Class<? extends AbstractPlugin>> plugins;
+  static {
+    plugins = new LinkedList<>();
+    plugins.add(CopySourceToDestination.class);
+  }
+
+  public static void main(String []args) {
+    LOG.info("Documentation preprocessor started");
+
+    // Parameter handling
+    if(args.length != 2) {
+      throw new RuntimeException("Expected two arguments - source and destination - but " + args.length + " given.");
+    }
+    String source = args[0];
+    String destination = args[1];
+    LOG.info("Source directory: " + source);
+    LOG.info("Destination directory: " + destination);
+
+    // Plugin execution
+    for(Class<? extends AbstractPlugin> pluginClass : plugins) {
+      LOG.info("Running plugin " + pluginClass.getCanonicalName());
+
+      AbstractPlugin plugin = (AbstractPlugin) ClassUtils.instantiate(pluginClass);
+      plugin.setSource(source);
+      plugin.setDestination(destination);
+
+      plugin.run();
+    }
+
+    LOG.info("Documentation preprocessor finished");
+  }
+}

docs/src/main/java/org/apache/sqoop/docs/generator/plugins/AbstractPlugin.java

@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.docs.generator.plugins;
+
+/**
+ * Abstract plugin class to abstract steps during docs pre-processing
+ */
+abstract public class AbstractPlugin {
+
+  /**
+   * Source directory
+   */
+  private String source;
+
+  /**
+   * Destination directory;
+   */
+  private String destination;
+
+  /**
+   * Do the plugin step.
+   *
+   * Throws RuntimeException aborting the build on error.
+   */
+  abstract public void run();
+
+  public void setSource(String source) {
+    this.source = source;
+  }
+
+  public String getSource() {
+    return source;
+  }
+
+  public void setDestination(String destination) {
+    this.destination = destination;
+  }
+
+  public String getDestination() {
+    return destination;
+  }
+
+}

docs/src/main/java/org/apache/sqoop/docs/generator/plugins/CopySourceToDestination.java

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.docs.generator.plugins;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Copy Source files to destination directory preserving directory structure.
+ */
+public class CopySourceToDestination extends AbstractPlugin {
+  @Override
+  public void run() {
+    try {
+      FileUtils.copyDirectory(new File(getSource()), new File(getDestination()));
+    } catch (IOException e) {
+      throw new RuntimeException("Can't copy source artifacts to destination", e);
+    }
+  }
+}

docs/src/main/resources/log4j.properties

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=INFO, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=[DOC GEN] %-5p %c %x: %m%n

pom.xml

@@ -751,6 +751,11 @@ limitations under the License.
             <target>${maven.compile.target}</target>
           </configuration>
         </plugin>
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>exec-maven-plugin</artifactId>
+          <version>1.4.0</version>
+        </plugin>
         <plugin>
           <groupId>org.apache.rat</groupId>
           <artifactId>apache-rat-plugin</artifactId>