From ef093399bf01fed4a7c17caebd05676f1e566b8a Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Fri, 5 Apr 2013 11:07:50 -0700 Subject: [PATCH] SQOOP-916: Add an abort validation handler (Venkatesh Seetharam via Jarek Jarcec Cecho) --- src/docs/user/validation.txt | 4 +- src/java/org/apache/sqoop/SqoopOptions.java | 4 +- .../validation/AbortOnFailureHandler.java | 55 ++++++++ .../sqoop/validation/LogOnFailureHandler.java | 5 +- .../sqoop/validation/RowCountValidator.java | 2 +- .../sqoop/validation/ValidationException.java | 4 +- .../validation/AbortOnFailureHandlerTest.java | 62 +++++++++ .../AbsoluteValidationThresholdTest.java | 38 ++++++ .../RowCountValidatorImportTest.java | 123 ++++++++++++++++-- 9 files changed, 275 insertions(+), 22 deletions(-) create mode 100644 src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java create mode 100644 src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java create mode 100644 src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java diff --git a/src/docs/user/validation.txt b/src/docs/user/validation.txt index f28b4208..282cfd6b 100644 --- a/src/docs/user/validation.txt +++ b/src/docs/user/validation.txt @@ -90,7 +90,7 @@ described below. Description: Responsible for handling failures, must implement org.apache.sqoop.validation.ValidationFailureHandler Supported values: The value has to be a fully qualified class name. - Default value: org.apache.sqoop.validation.LogOnFailureHandler + Default value: org.apache.sqoop.validation.AbortOnFailureHandler Limitations @@ -132,5 +132,5 @@ $ sqoop import --connect jdbc:mysql://db.foo.com/corp --table EMPLOYEES \ --validation-threshold \ org.apache.sqoop.validation.AbsoluteValidationThreshold \ --validation-failurehandler \ - org.apache.sqoop.validation.LogOnFailureHandler + org.apache.sqoop.validation.AbortOnFailureHandler ---- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 906542bf..f423c0fd 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -41,8 +41,8 @@ import com.cloudera.sqoop.util.StoredAsProperty; import org.apache.sqoop.util.CredentialsUtil; import org.apache.sqoop.util.LoggingUtils; +import org.apache.sqoop.validation.AbortOnFailureHandler; import org.apache.sqoop.validation.AbsoluteValidationThreshold; -import org.apache.sqoop.validation.LogOnFailureHandler; import org.apache.sqoop.validation.RowCountValidator; /** @@ -870,7 +870,7 @@ private void initDefaults(Configuration baseConfiguration) { this.isValidationEnabled = false; // validation is disabled by default this.validatorClass = RowCountValidator.class; this.validationThresholdClass = AbsoluteValidationThreshold.class; - this.validationFailureHandlerClass = LogOnFailureHandler.class; + this.validationFailureHandlerClass = AbortOnFailureHandler.class; } /** diff --git a/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java b/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java new file mode 100644 index 00000000..32f4fd9d --- /dev/null +++ b/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java @@ -0,0 +1,55 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.validation; + +/** + * A specific implementation of ValidationFailureHandler that aborts the + * processing by throwing an exception with failure message and the reason. + * + * This is used as the default handler unless overridden in configuration. + */ +public class AbortOnFailureHandler implements ValidationFailureHandler { + + static final ValidationFailureHandler INSTANCE = new AbortOnFailureHandler(); + + /** + * Method that handles the validation failure. + * + * @param validationContext validation context + * @return if failure was handled or not + * @throws ValidationException + */ + @Override + public boolean handle(ValidationContext validationContext) + throws ValidationException { + + StringBuilder messageBuffer = new StringBuilder(); + messageBuffer.append("Validation failed by "); + messageBuffer.append(validationContext.getMessage()); + messageBuffer.append(". Reason: ").append(validationContext.getReason()); + messageBuffer.append(", Row Count at Source: "); + messageBuffer.append(validationContext.getSourceRowCount()); + messageBuffer.append(", Row Count at Target: "); + messageBuffer.append(validationContext.getTargetRowCount()); + + throw new ValidationException(messageBuffer.toString()); + } +} diff --git a/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java b/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java index 3ded6adf..3c20136c 100644 --- a/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java +++ b/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java @@ -23,9 +23,10 @@ /** * A specific implementation of ValidationFailureHandler that logs the failure - * message and the reason with the configured logger. + * message and the reason with the configured logger. A note of caution in + * using this since it fails silently by logging the failure to a log file. * - * This is used as the default handler unless overridden in configuration. + * This is mostly used for testing purposes since this fails silently. */ public class LogOnFailureHandler implements ValidationFailureHandler { private static final Log LOG = diff --git a/src/java/org/apache/sqoop/validation/RowCountValidator.java b/src/java/org/apache/sqoop/validation/RowCountValidator.java index 2896192a..ca212327 100644 --- a/src/java/org/apache/sqoop/validation/RowCountValidator.java +++ b/src/java/org/apache/sqoop/validation/RowCountValidator.java @@ -37,7 +37,7 @@ public class RowCountValidator implements Validator { public boolean validate(ValidationContext context) throws ValidationException { return validate(context, - AbsoluteValidationThreshold.INSTANCE, LogOnFailureHandler.INSTANCE); + AbsoluteValidationThreshold.INSTANCE, AbortOnFailureHandler.INSTANCE); } @Override diff --git a/src/java/org/apache/sqoop/validation/ValidationException.java b/src/java/org/apache/sqoop/validation/ValidationException.java index 0c102416..118f1ca7 100644 --- a/src/java/org/apache/sqoop/validation/ValidationException.java +++ b/src/java/org/apache/sqoop/validation/ValidationException.java @@ -24,8 +24,8 @@ */ public class ValidationException extends Exception { - public ValidationException(String s, Throwable throwable) { - super(s, throwable); + public ValidationException(String message) { + super(message); } @Override diff --git a/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java b/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java new file mode 100644 index 00000000..f38164c4 --- /dev/null +++ b/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.validation; + +import com.cloudera.sqoop.SqoopOptions; +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; + +/** + * Tests for AbortOnFailureHandler. + */ +public class AbortOnFailureHandlerTest extends TestCase { + + public void testAbortOnFailureHandlerIsDefaultOption() { + assertEquals(AbortOnFailureHandler.class, + new SqoopOptions(new Configuration()).getValidationFailureHandlerClass()); + } + + /** + * Positive case. + */ + public void testAbortOnFailureHandlerAborting() { + try { + Validator validator = new RowCountValidator(); + validator.validate(new ValidationContext(100, 90)); + fail("AbortOnFailureHandler should have thrown an exception"); + } catch (ValidationException e) { + assertEquals("Validation failed by RowCountValidator. " + + "Reason: The expected counter value was 100 but the actual value " + + "was 90, Row Count at Source: 100, Row Count at Target: 90", + e.getMessage()); + } + } + + /** + * Negative case. + */ + public void testAbortOnFailureHandlerNotAborting() { + try { + Validator validator = new RowCountValidator(); + validator.validate(new ValidationContext(100, 100)); + } catch (ValidationException e) { + fail("AbortOnFailureHandler should NOT have thrown an exception"); + } + } +} diff --git a/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java b/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java new file mode 100644 index 00000000..9ac50744 --- /dev/null +++ b/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.validation; + +import junit.framework.TestCase; + +/** + * Tests for AbsoluteValidationThreshold. + */ +public class AbsoluteValidationThresholdTest extends TestCase { + + /** + * Test the implementation for AbsoluteValidationThreshold. + * Both arguments should be same else fail. + */ + public void testAbsoluteValidationThreshold() { + ValidationThreshold validationThreshold = new AbsoluteValidationThreshold(); + assertTrue(validationThreshold.compare(100, 100)); + assertFalse(validationThreshold.compare(100, 90)); + assertFalse(validationThreshold.compare(90, 100)); + } +} diff --git a/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java b/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java index a53e2816..035d3b10 100644 --- a/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java +++ b/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java @@ -18,8 +18,10 @@ package org.apache.sqoop.validation; +import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.testutil.ImportJobTestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.tool.ImportTool; import java.util.ArrayList; import java.util.Collections; @@ -37,22 +39,11 @@ protected List getExtraArgs(Configuration conf) { } /** - * Test the implementation for AbsoluteValidationThreshold. - * Both arguments should be same else fail. - */ - public void testAbsoluteValidationThreshold() { - ValidationThreshold validationThreshold = new AbsoluteValidationThreshold(); - assertTrue(validationThreshold.compare(100, 100)); - assertFalse(validationThreshold.compare(100, 90)); - assertFalse(validationThreshold.compare(90, 100)); - } - - /** - * Test if teh --validate flag actually made it through the options. + * Test if the --validate flag actually made it through the options. * * @throws Exception */ - public void testValidateOptionIsEnabled() throws Exception { + public void testValidateOptionIsEnabledInCLI() throws Exception { String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"}; String[] insertVals = {"1", "'Bob'", "'sales'"}; @@ -68,6 +59,112 @@ public void testValidateOptionIsEnabled() throws Exception { } } + public void testValidationOptionsParsedCorrectly() throws Exception { + String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"}; + String[] insertVals = {"1", "'Bob'", "'sales'"}; + + try { + createTableWithColTypes(types, insertVals); + + String[] args = getArgv(true, null, getConf()); + ArrayList argsList = new ArrayList(); + argsList.add("--validator"); + argsList.add("org.apache.sqoop.validation.RowCountValidator"); + argsList.add("--validation-threshold"); + argsList.add("org.apache.sqoop.validation.AbsoluteValidationThreshold"); + argsList.add("--validation-failurehandler"); + argsList.add("org.apache.sqoop.validation.AbortOnFailureHandler"); + Collections.addAll(argsList, args); + + assertTrue("Validate option missing.", argsList.contains("--validate")); + assertTrue("Validator option missing.", argsList.contains("--validator")); + + String[] optionArgs = toStringArray(argsList); + + SqoopOptions validationOptions = new ImportTool().parseArguments( + optionArgs, getConf(), getSqoopOptions(getConf()), true); + assertEquals(RowCountValidator.class, + validationOptions.getValidatorClass()); + assertEquals(AbsoluteValidationThreshold.class, + validationOptions.getValidationThresholdClass()); + assertEquals(AbortOnFailureHandler.class, + validationOptions.getValidationFailureHandlerClass()); + } catch (Exception e) { + fail("The validation options are passed correctly: " + e.getMessage()); + } finally { + dropTableIfExists(getTableName()); + } + } + + public void testInvalidValidationOptions() throws Exception { + String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"}; + String[] insertVals = {"1", "'Bob'", "'sales'"}; + + try { + createTableWithColTypes(types, insertVals); + + String[] args = getArgv(true, null, getConf()); + ArrayList argsList = new ArrayList(); + argsList.add("--validator"); + argsList.add("org.apache.sqoop.validation.NullValidator"); + argsList.add("--validation-threshold"); + argsList.add("org.apache.sqoop.validation.NullValidationThreshold"); + argsList.add("--validation-failurehandler"); + argsList.add("org.apache.sqoop.validation.NullFailureHandler"); + Collections.addAll(argsList, args); + + String[] optionArgs = toStringArray(argsList); + + new ImportTool().parseArguments(optionArgs, getConf(), + getSqoopOptions(getConf()), true); + fail("The validation options are incorrect and must throw an exception"); + } catch (Exception e) { + System.out.println("e.getMessage() = " + e.getMessage()); + System.out.println("e.getClass() = " + e.getClass()); + assertEquals( + com.cloudera.sqoop.SqoopOptions.InvalidOptionsException.class, + e.getClass()); + } finally { + dropTableIfExists(getTableName()); + } + } + + private String[] toStringArray(ArrayList argsList) { + String[] optionArgs = new String[argsList.size()]; + for (int i = 0; i < argsList.size(); i++) { + optionArgs[i] = argsList.get(i); + } + return optionArgs; + } + + /** + * Negative case where the row counts do NOT match. + */ + public void testValidatorWithDifferentRowCounts() { + try { + Validator validator = new RowCountValidator(); + validator.validate(new ValidationContext(100, 90)); + fail("FailureHandler should have thrown an exception"); + } catch (ValidationException e) { + assertEquals("Validation failed by RowCountValidator. " + + "Reason: The expected counter value was 100 but the actual value " + + "was 90, Row Count at Source: 100, Row Count at Target: 90", + e.getMessage()); + } + } + + /** + * Positive case where the row counts match. + */ + public void testValidatorWithMatchingRowCounts() { + try { + Validator validator = new RowCountValidator(); + validator.validate(new ValidationContext(100, 100)); + } catch (ValidationException e) { + fail("FailureHandler should NOT have thrown an exception"); + } + } + /** * Test the validation for a sample import, positive case. *