github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-04 04:49:43 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-892:: Validate acceptable number of mappers and reducers

Browse Source 
(Jarcec Cecho via Cheolsoo Park)
This commit is contained in:
Cheolsoo Park 2013-02-23 19:23:27 -08:00
parent cc5ae8f817
commit df3a266c6c
3 changed files with 149 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
core/src/main/java/org/apache/sqoop/framework/FrameworkValidator.java
View File

@ -20,6 +20,9 @@
import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
import org.apache.sqoop.framework.configuration.ExportJobConfiguration;
import org.apache.sqoop.framework.configuration.ImportJobConfiguration;
import org.apache.sqoop.framework.configuration.InputForm;
import org.apache.sqoop.framework.configuration.OutputForm;
import org.apache.sqoop.framework.configuration.ThrottlingForm;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
@ -54,9 +57,8 @@ private Validation validateExportJob(Object jobConfiguration) {
Validation validation = new Validation(ExportJobConfiguration.class);
ExportJobConfiguration configuration = (ExportJobConfiguration)jobConfiguration;
if(configuration.input.inputDirectory == null || configuration.input.inputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");
}
validateInputForm(validation, configuration.input);
validateThrottingForm(validation, configuration.throttling);
return validation;
}
@ -65,10 +67,31 @@ private Validation validateImportJob(Object jobConfiguration) {
Validation validation = new Validation(ImportJobConfiguration.class);
ImportJobConfiguration configuration = (ImportJobConfiguration)jobConfiguration;
if(configuration.output.outputDirectory == null || configuration.output.outputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Input directory is empty");
}
validateOutputForm(validation, configuration.output);
validateThrottingForm(validation, configuration.throttling);
return validation;
}
private void validateInputForm(Validation validation, InputForm input) {
if(input.inputDirectory == null || input.inputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");
}
}
private void validateOutputForm(Validation validation, OutputForm output) {
if(output.outputDirectory == null || output.outputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Input directory is empty");
}
}
private void validateThrottingForm(Validation validation, ThrottlingForm throttling) {
if(throttling.extractors != null && throttling.extractors < 1) {
validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");
}
if(throttling.loaders != null && throttling.loaders < 1) {
validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");
}
}
}

1
core/src/main/java/org/apache/sqoop/framework/configuration/ExportJobConfiguration.java
View File

@ -31,6 +31,7 @@ public class ExportJobConfiguration {
@Form public ThrottlingForm throttling;
public ExportJobConfiguration() {
input = new InputForm();
throttling = new ThrottlingForm();
}
}

119
core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java Normal file
View File

@ -0,0 +1,119 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.framework;
import org.apache.sqoop.framework.configuration.ConnectionConfiguration;
import org.apache.sqoop.framework.configuration.ExportJobConfiguration;
import org.apache.sqoop.framework.configuration.ImportJobConfiguration;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
*
*/
public class TestFrameworkValidator {
FrameworkValidator validator;
@Before
public void setUp() {
validator = new FrameworkValidator();
}
@Test
public void testConnectionValidation() {
ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration();
Validation validation = validator.validateConnection(connectionConfiguration);
assertEquals(Status.FINE, validation.getStatus());
assertEquals(0, validation.getMessages().size());
}
@Test
public void testExportJobValidation() {
ExportJobConfiguration configuration;
Validation validation;
// Empty form is not allowed
configuration = new ExportJobConfiguration();
validation = validator.validateJob(MJob.Type.EXPORT, configuration);
assertEquals(Status.UNACCEPTABLE, validation.getStatus());
assertTrue(validation.getMessages().containsKey(new Validation.FormInput("input.inputDirectory")));
// Explicitly setting extractors and loaders
configuration = new ExportJobConfiguration();
configuration.input.inputDirectory = "/czech/republic";
configuration.throttling.extractors = 3;
configuration.throttling.loaders = 3;
validation = validator.validateJob(MJob.Type.EXPORT, configuration);
assertEquals(Status.FINE, validation.getStatus());
assertEquals(0, validation.getMessages().size());
// Negative and zero values for extractors and loaders
configuration = new ExportJobConfiguration();
configuration.input.inputDirectory = "/czech/republic";
configuration.throttling.extractors = 0;
configuration.throttling.loaders = -1;
validation = validator.validateJob(MJob.Type.EXPORT, configuration);
assertEquals(Status.UNACCEPTABLE, validation.getStatus());
assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors")));
assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders")));
}
@Test
public void testImportJobValidation() {
ImportJobConfiguration configuration;
Validation validation;
// Empty form is not allowed
configuration = new ImportJobConfiguration();
validation = validator.validateJob(MJob.Type.IMPORT, configuration);
assertEquals(Status.UNACCEPTABLE, validation.getStatus());
assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.outputDirectory")));
// Explicitly setting extractors and loaders
configuration = new ImportJobConfiguration();
configuration.output.outputDirectory = "/czech/republic";
configuration.throttling.extractors = 3;
configuration.throttling.loaders = 3;
validation = validator.validateJob(MJob.Type.IMPORT, configuration);
assertEquals(Status.FINE, validation.getStatus());
assertEquals(0, validation.getMessages().size());
// Negative and zero values for extractors and loaders
configuration = new ImportJobConfiguration();
configuration.output.outputDirectory = "/czech/republic";
configuration.throttling.extractors = 0;
configuration.throttling.loaders = -1;
validation = validator.validateJob(MJob.Type.IMPORT, configuration);
assertEquals(Status.UNACCEPTABLE, validation.getStatus());
assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors")));
assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders")));
}
}