SQOOP-916: Add an abort validation handler

(Venkatesh Seetharam via Jarek Jarcec Cecho)

src/docs/user/validation.txt

@@ -90,7 +90,7 @@ described below.
  Description:      Responsible for handling failures, must implement
                    org.apache.sqoop.validation.ValidationFailureHandler
  Supported values: The value has to be a fully qualified class name.
- Default value:    org.apache.sqoop.validation.LogOnFailureHandler
+ Default value:    org.apache.sqoop.validation.AbortOnFailureHandler
 
 
 Limitations
@@ -132,5 +132,5 @@ $ sqoop import --connect jdbc:mysql://db.foo.com/corp --table EMPLOYEES \
     --validation-threshold \
           org.apache.sqoop.validation.AbsoluteValidationThreshold \
     --validation-failurehandler \
-          org.apache.sqoop.validation.LogOnFailureHandler
+          org.apache.sqoop.validation.AbortOnFailureHandler
 ----

src/java/org/apache/sqoop/SqoopOptions.java

@@ -41,8 +41,8 @@
 import com.cloudera.sqoop.util.StoredAsProperty;
 import org.apache.sqoop.util.CredentialsUtil;
 import org.apache.sqoop.util.LoggingUtils;
+import org.apache.sqoop.validation.AbortOnFailureHandler;
 import org.apache.sqoop.validation.AbsoluteValidationThreshold;
-import org.apache.sqoop.validation.LogOnFailureHandler;
 import org.apache.sqoop.validation.RowCountValidator;
 
 /**
@@ -870,7 +870,7 @@ private void initDefaults(Configuration baseConfiguration) {
     this.isValidationEnabled = false; // validation is disabled by default
     this.validatorClass = RowCountValidator.class;
     this.validationThresholdClass = AbsoluteValidationThreshold.class;
-    this.validationFailureHandlerClass = LogOnFailureHandler.class;
+    this.validationFailureHandlerClass = AbortOnFailureHandler.class;
   }
 
   /**

src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java

@@ -0,0 +1,55 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+/**
+ * A specific implementation of ValidationFailureHandler that aborts the
+ * processing by throwing an exception with failure message and the reason.
+ *
+ * This is used as the default handler unless overridden in configuration.
+ */
+public class AbortOnFailureHandler implements ValidationFailureHandler {
+
+  static final ValidationFailureHandler INSTANCE = new AbortOnFailureHandler();
+
+  /**
+   * Method that handles the validation failure.
+   *
+   * @param validationContext validation context
+   * @return if failure was handled or not
+   * @throws ValidationException
+   */
+  @Override
+  public boolean handle(ValidationContext validationContext)
+    throws ValidationException {
+
+    StringBuilder messageBuffer = new StringBuilder();
+    messageBuffer.append("Validation failed by ");
+    messageBuffer.append(validationContext.getMessage());
+    messageBuffer.append(". Reason: ").append(validationContext.getReason());
+    messageBuffer.append(", Row Count at Source: ");
+    messageBuffer.append(validationContext.getSourceRowCount());
+    messageBuffer.append(", Row Count at Target: ");
+    messageBuffer.append(validationContext.getTargetRowCount());
+
+    throw new ValidationException(messageBuffer.toString());
+  }
+}

src/java/org/apache/sqoop/validation/LogOnFailureHandler.java

@@ -23,9 +23,10 @@
 
 /**
  * A specific implementation of ValidationFailureHandler that logs the failure
- * message and the reason with the configured logger.
+ * message and the reason with the configured logger. A note of caution in
+ * using this since it fails silently by logging the failure to a log file.
  *
- * This is used as the default handler unless overridden in configuration.
+ * This is mostly used for testing purposes since this fails silently.
  */
 public class LogOnFailureHandler implements ValidationFailureHandler {
   private static final Log LOG =

src/java/org/apache/sqoop/validation/RowCountValidator.java

@@ -37,7 +37,7 @@ public class RowCountValidator implements Validator {
   public boolean validate(ValidationContext context)
     throws ValidationException {
     return validate(context,
-      AbsoluteValidationThreshold.INSTANCE, LogOnFailureHandler.INSTANCE);
+      AbsoluteValidationThreshold.INSTANCE, AbortOnFailureHandler.INSTANCE);
   }
 
   @Override

src/java/org/apache/sqoop/validation/ValidationException.java

@@ -24,8 +24,8 @@
  */
 public class ValidationException extends Exception {
 
-  public ValidationException(String s, Throwable throwable) {
-    super(s, throwable);
+  public ValidationException(String message) {
+    super(message);
   }
 
   @Override

src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java

@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+import com.cloudera.sqoop.SqoopOptions;
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Tests for AbortOnFailureHandler.
+ */
+public class AbortOnFailureHandlerTest extends TestCase {
+
+  public void testAbortOnFailureHandlerIsDefaultOption() {
+    assertEquals(AbortOnFailureHandler.class,
+      new SqoopOptions(new Configuration()).getValidationFailureHandlerClass());
+  }
+
+  /**
+   * Positive case.
+   */
+  public void testAbortOnFailureHandlerAborting() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 90));
+      fail("AbortOnFailureHandler should have thrown an exception");
+    } catch (ValidationException e) {
+      assertEquals("Validation failed by RowCountValidator. "
+        + "Reason: The expected counter value was 100 but the actual value "
+        + "was 90, Row Count at Source: 100, Row Count at Target: 90",
+        e.getMessage());
+    }
+  }
+
+  /**
+   * Negative case.
+   */
+  public void testAbortOnFailureHandlerNotAborting() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 100));
+    } catch (ValidationException e) {
+      fail("AbortOnFailureHandler should NOT have thrown an exception");
+    }
+  }
+}

src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+import junit.framework.TestCase;
+
+/**
+ * Tests for AbsoluteValidationThreshold.
+ */
+public class AbsoluteValidationThresholdTest extends TestCase {
+
+  /**
+   * Test the implementation for AbsoluteValidationThreshold.
+   * Both arguments should be same else fail.
+   */
+  public void testAbsoluteValidationThreshold() {
+    ValidationThreshold validationThreshold = new AbsoluteValidationThreshold();
+    assertTrue(validationThreshold.compare(100, 100));
+    assertFalse(validationThreshold.compare(100, 90));
+    assertFalse(validationThreshold.compare(90, 100));
+  }
+}

src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java

@@ -18,8 +18,10 @@
 
 package org.apache.sqoop.validation;
 
+import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.testutil.ImportJobTestCase;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.tool.ImportTool;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -37,22 +39,11 @@ protected List<String> getExtraArgs(Configuration conf) {
   }
 
   /**
-   * Test the implementation for AbsoluteValidationThreshold.
-   * Both arguments should be same else fail.
-   */
-  public void testAbsoluteValidationThreshold() {
-    ValidationThreshold validationThreshold = new AbsoluteValidationThreshold();
-    assertTrue(validationThreshold.compare(100, 100));
-    assertFalse(validationThreshold.compare(100, 90));
-    assertFalse(validationThreshold.compare(90, 100));
-  }
-
-  /**
-   * Test if teh --validate flag actually made it through the options.
+   * Test if the --validate flag actually made it through the options.
    *
    * @throws Exception
    */
-  public void testValidateOptionIsEnabled() throws Exception {
+  public void testValidateOptionIsEnabledInCLI() throws Exception {
     String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
     String[] insertVals = {"1", "'Bob'", "'sales'"};
 
@@ -68,6 +59,112 @@ public void testValidateOptionIsEnabled() throws Exception {
     }
   }
 
+  public void testValidationOptionsParsedCorrectly() throws Exception {
+    String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
+    String[] insertVals = {"1", "'Bob'", "'sales'"};
+
+    try {
+      createTableWithColTypes(types, insertVals);
+
+      String[] args = getArgv(true, null, getConf());
+      ArrayList<String> argsList = new ArrayList<String>();
+      argsList.add("--validator");
+      argsList.add("org.apache.sqoop.validation.RowCountValidator");
+      argsList.add("--validation-threshold");
+      argsList.add("org.apache.sqoop.validation.AbsoluteValidationThreshold");
+      argsList.add("--validation-failurehandler");
+      argsList.add("org.apache.sqoop.validation.AbortOnFailureHandler");
+      Collections.addAll(argsList, args);
+
+      assertTrue("Validate option missing.", argsList.contains("--validate"));
+      assertTrue("Validator option missing.", argsList.contains("--validator"));
+
+      String[] optionArgs = toStringArray(argsList);
+
+      SqoopOptions validationOptions = new ImportTool().parseArguments(
+        optionArgs, getConf(), getSqoopOptions(getConf()), true);
+      assertEquals(RowCountValidator.class,
+        validationOptions.getValidatorClass());
+      assertEquals(AbsoluteValidationThreshold.class,
+        validationOptions.getValidationThresholdClass());
+      assertEquals(AbortOnFailureHandler.class,
+        validationOptions.getValidationFailureHandlerClass());
+    } catch (Exception e) {
+      fail("The validation options are passed correctly: " + e.getMessage());
+    } finally {
+      dropTableIfExists(getTableName());
+    }
+  }
+
+  public void testInvalidValidationOptions() throws Exception {
+    String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
+    String[] insertVals = {"1", "'Bob'", "'sales'"};
+
+    try {
+      createTableWithColTypes(types, insertVals);
+
+      String[] args = getArgv(true, null, getConf());
+      ArrayList<String> argsList = new ArrayList<String>();
+      argsList.add("--validator");
+      argsList.add("org.apache.sqoop.validation.NullValidator");
+      argsList.add("--validation-threshold");
+      argsList.add("org.apache.sqoop.validation.NullValidationThreshold");
+      argsList.add("--validation-failurehandler");
+      argsList.add("org.apache.sqoop.validation.NullFailureHandler");
+      Collections.addAll(argsList, args);
+
+      String[] optionArgs = toStringArray(argsList);
+
+      new ImportTool().parseArguments(optionArgs, getConf(),
+        getSqoopOptions(getConf()), true);
+      fail("The validation options are incorrect and must throw an exception");
+    } catch (Exception e) {
+      System.out.println("e.getMessage() = " + e.getMessage());
+      System.out.println("e.getClass() = " + e.getClass());
+      assertEquals(
+        com.cloudera.sqoop.SqoopOptions.InvalidOptionsException.class,
+        e.getClass());
+    } finally {
+      dropTableIfExists(getTableName());
+    }
+  }
+
+  private String[] toStringArray(ArrayList<String> argsList) {
+    String[] optionArgs = new String[argsList.size()];
+    for (int i = 0; i < argsList.size(); i++) {
+      optionArgs[i] = argsList.get(i);
+    }
+    return optionArgs;
+  }
+
+  /**
+   * Negative case where the row counts do NOT match.
+   */
+  public void testValidatorWithDifferentRowCounts() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 90));
+      fail("FailureHandler should have thrown an exception");
+    } catch (ValidationException e) {
+      assertEquals("Validation failed by RowCountValidator. "
+        + "Reason: The expected counter value was 100 but the actual value "
+        + "was 90, Row Count at Source: 100, Row Count at Target: 90",
+        e.getMessage());
+    }
+  }
+
+  /**
+   * Positive case where the row counts match.
+   */
+  public void testValidatorWithMatchingRowCounts() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 100));
+    } catch (ValidationException e) {
+      fail("FailureHandler should NOT have thrown an exception");
+    }
+  }
+
   /**
    * Test the validation for a sample import, positive case.
    *