github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-16 17:00:53 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1498: Sqoop2: Repository Object refactoring (objects prefixed with M)

Browse Source 
(Veena Basavaraj via Abraham Elmahrek)
This commit is contained in:
Abraham Elmahrek 2014-10-02 23:29:50 -07:00
parent e540668b55
commit 8362c73cc0
177 changed files with 4807 additions and 5247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
client/src/main/java/org/apache/sqoop/client/SqoopClient.java
View File

@ -17,27 +17,28 @@
*/
package org.apache.sqoop.client;
import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
import org.apache.sqoop.json.DriverConfigBean;
import org.apache.sqoop.json.ValidationResultBean;
import org.apache.sqoop.model.FormUtils;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.ValidationResult;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.json.ValidationResultBean;
import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.validation.ConfigValidationResult;
import org.apache.sqoop.validation.Status;
/**
* Sqoop client API.
*
@ -69,9 +70,9 @@ public class SqoopClient {
private Map<Long, ResourceBundle> connectorConfigBundles;
/**
* Cached driverConfig.
* Cached driver.
*/
private MDriverConfig driverConfig;
private MDriver mDriver;
/**
* Cached driverConfig bundle.
*/
@ -120,7 +121,7 @@ public void clearCache() {
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
driverConfigBundle = null;
connectors = new HashMap<Long, MConnector>();
driverConfig = null;
mDriver = null;
isAllConnectors = false;
}
@ -214,11 +215,10 @@ public Collection<MConnector> getConnectors() {
* @param connectorId Connector id.
* @return
*/
public ResourceBundle getConnectorConfigResourceBundle(long connectorId) {
public ResourceBundle getConnectorConfigBundle(long connectorId) {
if(connectorConfigBundles.containsKey(connectorId)) {
return connectorConfigBundles.get(connectorId);
}
retrieveConnector(connectorId);
return connectorConfigBundles.get(connectorId);
}
@ -229,33 +229,46 @@ public ResourceBundle getConnectorConfigResourceBundle(long connectorId) {
* @return
*/
public MDriverConfig getDriverConfig() {
if(driverConfig != null) {
return driverConfig.clone(false);
if (mDriver != null) {
return mDriver.clone(false).getDriverConfig();
}
retrieveAndCacheDriverConfig();
return driverConfig.clone(false);
retrieveAndCacheDriver();
return mDriver.clone(false).getDriverConfig();
}
/**
* Return driver.
*
* @return
*/
public MDriver getDriver() {
if (mDriver != null) {
return mDriver.clone(false);
}
retrieveAndCacheDriver();
return mDriver.clone(false);
}
/**
* Retrieve driverConfig and cache it.
*/
private void retrieveAndCacheDriverConfig() {
DriverConfigBean driverConfigBean = resourceRequests.readDriverConfig();
driverConfig = driverConfigBean.getDriverConfig();
driverConfigBundle = driverConfigBean.getResourceBundle();
private void retrieveAndCacheDriver() {
DriverBean driverBean = resourceRequests.readDriver();
mDriver = driverBean.getDriver();
driverConfigBundle = driverBean.getDriverConfigResourceBundle();
}
/**
* Return driverConfig bundle.
*
*xx
* @return
*/
public ResourceBundle getDriverConfigBundle() {
if(driverConfigBundle != null) {
return driverConfigBundle;
}
retrieveAndCacheDriverConfig();
retrieveAndCacheDriver();
return driverConfigBundle;
}
@ -266,11 +279,7 @@ public ResourceBundle getDriverConfigBundle() {
* @return
*/
public MLink createLink(long connectorId) {
return new MLink(
connectorId,
getConnector(connectorId).getConnectionForms(),
getDriverConfig().getConnectionForms()
);
return new MLink(connectorId, getConnector(connectorId).getLinkConfig());
}
/**
@ -284,7 +293,6 @@ public MLink createLink(String connectorName) {
if (connector == null) {
throw new SqoopException(ClientError.CLIENT_0003, connectorName);
}
return createLink(connector.getPersistenceId());
}
@ -362,9 +370,9 @@ public MJob createJob(long fromLinkId, long toLinkId) {
toLink.getConnectorId(),
fromLink.getPersistenceId(),
toLink.getPersistenceId(),
getConnector(fromLink.getConnectorId()).getJobForms(Direction.FROM),
getConnector(toLink.getConnectorId()).getJobForms(Direction.TO),
getDriverConfig().getJobForms()
getConnector(fromLink.getConnectorId()).getFromConfig(),
getConnector(toLink.getConnectorId()).getToConfig(),
getDriverConfig()
);
}
@ -530,41 +538,36 @@ public List<MSubmission> getSubmissionsForJob(long jobId) {
}
private Status applyLinkValidations(ValidationResultBean bean, MLink link) {
ValidationResult connector = bean.getValidationResults()[0];
ValidationResult driverConfig = bean.getValidationResults()[1];
ConfigValidationResult linkConfig = bean.getValidationResults()[0];
// Apply validation results
FormUtils.applyValidation(link.getConnectorPart().getForms(), connector);
FormUtils.applyValidation(link.getFrameworkPart().getForms(), driverConfig);
ConfigUtils.applyValidation(link.getConnectorLinkConfig().getConfigs(), linkConfig);
Long id = bean.getId();
if(id != null) {
link.setPersistenceId(id);
}
return Status.getWorstStatus(connector.getStatus(), driverConfig.getStatus());
return Status.getWorstStatus(linkConfig.getStatus());
}
private Status applyJobValidations(ValidationResultBean bean, MJob job) {
ValidationResult fromConnector = bean.getValidationResults()[0];
ValidationResult toConnector = bean.getValidationResults()[1];
ValidationResult driverConfig = bean.getValidationResults()[2];
ConfigValidationResult fromConfig = bean.getValidationResults()[0];
ConfigValidationResult toConfig = bean.getValidationResults()[1];
// TODO(VB): fix this as part of SQOOP 1509
//ConfigValidationResult driverConfig = bean.getValidationResults()[2];
// Apply validation results
// @TODO(Abe): From/To validation.
FormUtils.applyValidation(
job.getConnectorPart(Direction.FROM).getForms(),
fromConnector);
FormUtils.applyValidation(job.getFrameworkPart().getForms(), driverConfig);
FormUtils.applyValidation(
job.getConnectorPart(Direction.TO).getForms(),
toConnector);
ConfigUtils.applyValidation(
job.getJobConfig(Direction.FROM).getConfigs(),
fromConfig);
ConfigUtils.applyValidation(
job.getJobConfig(Direction.TO).getConfigs(),
toConfig);
//ConfigUtils.applyValidation(job.getDriverConfig().getSelf().getConfigs(), driverConfig);
Long id = bean.getId();
if(id != null) {
job.setPersistenceId(id);
}
return Status.getWorstStatus(fromConnector.getStatus(), driverConfig.getStatus(), toConnector.getStatus());
return Status.getWorstStatus(fromConfig.getStatus(), toConfig.getStatus());
// driverConfig.getStatus());
}
}

4
client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
View File

@ -22,7 +22,7 @@
import org.json.simple.JSONValue;
/**
* Provide cRud semantics over RESTfull HTTP API for connectors. Only read
* Provide Read semantics over RESTfull HTTP API for connectors. Only read
* is supported as creation, update and delete might be done only directly on
* server side.
*/
@ -38,10 +38,8 @@ public ConnectorBean read(String serverUrl, Long cid) {
response = super.get(serverUrl + RESOURCE + cid);
}
JSONObject jsonObject = (JSONObject)JSONValue.parse(response);
ConnectorBean connectorBean = new ConnectorBean();
connectorBean.restore(jsonObject);
return connectorBean;
}
}

15
client/src/main/java/org/apache/sqoop/client/request/DriverConfigResourceRequest.java
View File

@ -17,26 +17,23 @@
*/
package org.apache.sqoop.client.request;
import org.apache.sqoop.json.DriverConfigBean;
import org.apache.sqoop.json.DriverBean;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
/**
* Provide cRud semantics over RESTfull HTTP API for driverConfig. Only read
* is supported as creation, update and delete is not allowed.
* Provide CRUD semantics over RESTfull HTTP API for driverConfig
*/
public class DriverConfigResourceRequest extends ResourceRequest {
public static final String RESOURCE = "v1/config/driver";
public DriverConfigBean read(String serverUrl) {
public DriverBean read(String serverUrl) {
String response = null;
response = super.get(serverUrl + RESOURCE);
JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
DriverConfigBean driverConfigBean = new DriverConfigBean();
driverConfigBean.restore(jsonObject);
return driverConfigBean;
DriverBean driverBean = new DriverBean();
driverBean.restore(jsonObject);
return driverBean;
}
}

26
client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
View File

@ -34,46 +34,36 @@ public class JobResourceRequest extends ResourceRequest {
private static final String ENABLE = "/enable";
private static final String DISABLE = "/disable";
public JobBean read(String serverUrl, Long xid) {
public JobBean read(String serverUrl, Long linkId) {
String response;
if (xid == null) {
if (linkId == null) {
response = super.get(serverUrl + RESOURCE + "all");
} else {
response = super.get(serverUrl + RESOURCE + xid);
response = super.get(serverUrl + RESOURCE + linkId);
}
JSONObject jsonObject = (JSONObject) JSONValue.parse(response);
JobBean jobBean = new JobBean();
jobBean.restore(jsonObject);
return jobBean;
}
public ValidationResultBean create(String serverUrl, MJob job) {
JobBean jobBean = new JobBean(job);
// Extract all form inputs including sensitive inputs
// Extract all config inputs including sensitive inputs
JSONObject jobJson = jobBean.extract(false);
String response = super.post(serverUrl + RESOURCE, jobJson.toJSONString());
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
return validationBean;
ValidationResultBean validationResultBean = new ValidationResultBean();
validationResultBean.restore((JSONObject) JSONValue.parse(response));
return validationResultBean;
}
public ValidationResultBean update(String serverUrl, MJob job) {
JobBean jobBean = new JobBean(job);
// Extract all form inputs including sensitive inputs
// Extract all config inputs including sensitive inputs
JSONObject jobJson = jobBean.extract(false);
String response = super.put(serverUrl + RESOURCE + job.getPersistenceId(), jobJson.toJSONString());
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
return validationBean;
}

10
client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java
View File

@ -50,28 +50,22 @@ public LinkBean read(String serverUrl, Long xid) {
public ValidationResultBean create(String serverUrl, MLink link) {
LinkBean linkBean = new LinkBean(link);
// Extract all form inputs including sensitive inputs
// Extract all config inputs including sensitive inputs
JSONObject linkJson = linkBean.extract(false);
String response = super.post(serverUrl + RESOURCE, linkJson.toJSONString());
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
return validationBean;
}
public ValidationResultBean update(String serverUrl, MLink link) {
LinkBean linkBean = new LinkBean(link);
// Extract all form inputs including sensitive inputs
// Extract all config inputs including sensitive inputs
JSONObject linkJson = linkBean.extract(false);
String response = super.put(serverUrl + RESOURCE + link.getPersistenceId(), linkJson.toJSONString());
ValidationResultBean validationBean = new ValidationResultBean();
validationBean.restore((JSONObject) JSONValue.parse(response));
return validationBean;
}

4
client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
View File

@ -19,7 +19,7 @@
import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.ConnectorBean;
import org.apache.sqoop.json.DriverConfigBean;
import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.SubmissionBean;
import org.apache.sqoop.json.ValidationResultBean;
@ -83,7 +83,7 @@ public SubmissionResourceRequest getSubmissionResourceRequest() {
return submissionRequest;
}
public DriverConfigBean readDriverConfig() {
public DriverBean readDriver() {
return getDriverConfigResourceRequest().read(serverUrl);
}

73
client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
View File

@ -17,28 +17,36 @@
*/
package org.apache.sqoop.client;
import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
import org.apache.sqoop.json.DriverConfigBean;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.utils.MapResourceBundle;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.*;
import org.apache.sqoop.client.request.SqoopResourceRequests;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.json.ConnectorBean;
import org.apache.sqoop.json.DriverBean;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.utils.MapResourceBundle;
import org.junit.Before;
import org.junit.Test;
public class TestSqoopClient {
@ -62,7 +70,7 @@ public void testGetConnector() {
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
client.getConnectorConfigResourceBundle(1L);
client.getConnectorConfigBundle(1L);
verify(resourceRequests, times(1)).readConnector(1L);
}
@ -74,7 +82,7 @@ public void testGetConnectorByString() {
assertEquals(1, connector.getPersistenceId());
assertEquals("A1", connector.getUniqueName());
client.getConnectorConfigResourceBundle(1L);
client.getConnectorConfigBundle(1L);
verify(resourceRequests, times(0)).readConnector(1L);
verify(resourceRequests, times(1)).readConnector(null);
@ -87,7 +95,7 @@ public void testGetConnectorByString() {
@Test
public void testGetConnectorBundle() {
when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
client.getConnectorConfigResourceBundle(1L);
client.getConnectorConfigBundle(1L);
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
@ -101,12 +109,12 @@ public void testGetConnectorBundle() {
*/
@Test
public void testGetDriverConfig() {
when(resourceRequests.readDriverConfig()).thenReturn(driverConfigBean(driverConfig()));
when(resourceRequests.readDriver()).thenReturn(driverBean(driver()));
client.getDriverConfig();
client.getDriverConfigBundle();
verify(resourceRequests, times(1)).readDriverConfig();
verify(resourceRequests, times(1)).readDriver();
}
/**
@ -115,12 +123,12 @@ public void testGetDriverConfig() {
*/
@Test
public void testGetDriverConfigBundle() {
when(resourceRequests.readDriverConfig()).thenReturn(driverConfigBean(driverConfig()));
when(resourceRequests.readDriver()).thenReturn(driverBean(driver()));
client.getDriverConfigBundle();
client.getDriverConfig();
verify(resourceRequests, times(1)).readDriverConfig();
verify(resourceRequests, times(1)).readDriver();
}
/**
@ -135,12 +143,12 @@ public void testGetConnectors() {
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
client.getConnectorConfigResourceBundle(1);
client.getConnectorConfigBundle(1);
connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
connector = client.getConnector(2);
client.getConnectorConfigResourceBundle(2);
client.getConnectorConfigBundle(2);
assertEquals(2, connector.getPersistenceId());
connectors = client.getConnectors();
@ -173,11 +181,11 @@ public void testGetConnectorOneByOne() {
when(resourceRequests.readConnector(1L)).thenReturn(bean);
when(resourceRequests.readConnector(2L)).thenReturn(bean);
client.getConnectorConfigResourceBundle(1);
client.getConnectorConfigBundle(1);
client.getConnector(1);
client.getConnector(2);
client.getConnectorConfigResourceBundle(2);
client.getConnectorConfigBundle(2);
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
@ -207,21 +215,20 @@ private ConnectorBean connectorBean(MConnector...connectors) {
}
return new ConnectorBean(connectorList, bundles);
}
private DriverConfigBean driverConfigBean(MDriverConfig driverConfig) {
return new DriverConfigBean(driverConfig, new MapResourceBundle(null));
private DriverBean driverBean(MDriver driver) {
return new DriverBean(driver, new MapResourceBundle(null));
}
private MConnector connector(long id) {
MConnector connector = new MConnector("A" + id, "A" + id, "1.0" + id,
new MConnectionForms(null), new MJobForms(null), new MJobForms(null));
new MLinkConfig(null), new MFromConfig(null), new MToConfig(null));
connector.setPersistenceId(id);
return connector;
}
private MDriverConfig driverConfig() {
MDriverConfig driverConfig = new MDriverConfig(new MConnectionForms(null),
new MJobForms(null), "1");
driverConfig.setPersistenceId(1);
return driverConfig;
private MDriver driver() {
MDriver driver = new MDriver(new MDriverConfig(new LinkedList<MConfig>()), "1");
driver.setPersistenceId(1);
return driver;
}
}

121
common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
View File

@ -17,15 +17,15 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.util.FormSerialization.ALL;
import static org.apache.sqoop.json.util.FormSerialization.CLASS;
import static org.apache.sqoop.json.util.FormSerialization.CON_FORMS;
import static org.apache.sqoop.json.util.FormSerialization.ID;
import static org.apache.sqoop.json.util.FormSerialization.JOB_FORMS;
import static org.apache.sqoop.json.util.FormSerialization.NAME;
import static org.apache.sqoop.json.util.FormSerialization.VERSION;
import static org.apache.sqoop.json.util.FormSerialization.extractForms;
import static org.apache.sqoop.json.util.FormSerialization.restoreForms;
import static org.apache.sqoop.json.util.ConfigSerialization.ALL;
import static org.apache.sqoop.json.util.ConfigSerialization.CLASS;
import static org.apache.sqoop.json.util.ConfigSerialization.ID;
import static org.apache.sqoop.json.util.ConfigSerialization.CONNECTOR_JOB_CONFIG;
import static org.apache.sqoop.json.util.ConfigSerialization.CONNECTOR_LINK_CONFIG;
import static org.apache.sqoop.json.util.ConfigSerialization.NAME;
import static org.apache.sqoop.json.util.ConfigSerialization.VERSION;
import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList;
import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
@ -38,24 +38,28 @@
import java.util.Set;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MToConfig;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
/**
* Json representation of the connector object
*
*/
public class ConnectorBean implements JsonBean {
private List<MConnector> connectors;
private Map<Long, ResourceBundle> bundles;
private Map<Long, ResourceBundle> connectorConfigBundles;
// for "extract"
public ConnectorBean(List<MConnector> connectors,
Map<Long, ResourceBundle> bundles) {
public ConnectorBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) {
this.connectors = connectors;
this.bundles = bundles;
this.connectorConfigBundles = bundles;
}
// for "restore"
@ -67,46 +71,46 @@ public List<MConnector> getConnectors() {
}
public Map<Long, ResourceBundle> getResourceBundles() {
return bundles;
return connectorConfigBundles;
}
@SuppressWarnings("unchecked")
@Override
public JSONObject extract(boolean skipSensitive) {
JSONArray array = new JSONArray();
JSONArray connectorArray = new JSONArray();
for (MConnector connector : connectors) {
JSONObject object = new JSONObject();
JSONObject connectorJsonObject = new JSONObject();
object.put(ID, connector.getPersistenceId());
object.put(NAME, connector.getUniqueName());
object.put(CLASS, connector.getClassName());
object.put(VERSION, connector.getVersion());
connectorJsonObject.put(ID, connector.getPersistenceId());
connectorJsonObject.put(NAME, connector.getUniqueName());
connectorJsonObject.put(CLASS, connector.getClassName());
connectorJsonObject.put(VERSION, connector.getVersion());
connectorJsonObject.put(CONNECTOR_LINK_CONFIG,
extractConfigList(connector.getLinkConfig().getConfigs(), skipSensitive));
object.put(CON_FORMS, extractForms(connector.getConnectionForms().getForms(), skipSensitive));
object.put(JOB_FORMS, new JSONObject());
if (connector.getJobForms(Direction.FROM) != null) {
((JSONObject)object.get(JOB_FORMS)).put(
Direction.FROM, extractForms(connector.getJobForms(Direction.FROM).getForms(), skipSensitive));
connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
// add sub fields to the job config for from and to
if (connector.getFromConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(Direction.FROM,
extractConfigList(connector.getFromConfig().getConfigs(), skipSensitive));
}
if (connector.getJobForms(Direction.TO) != null) {
((JSONObject)object.get(JOB_FORMS)).put(
Direction.TO, extractForms(connector.getJobForms(Direction.TO).getForms(), skipSensitive));
if (connector.getToConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(Direction.TO,
extractConfigList(connector.getToConfig().getConfigs(), skipSensitive));
}
array.add(object);
connectorArray.add(connectorJsonObject);
}
JSONObject all = new JSONObject();
all.put(ALL, array);
all.put(ALL, connectorArray);
if(bundles != null && !bundles.isEmpty()) {
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
JSONObject jsonBundles = new JSONObject();
for(Map.Entry<Long, ResourceBundle> entry : bundles.entrySet()) {
jsonBundles.put(entry.getKey().toString(),
extractResourceBundle(entry.getValue()));
for (Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) {
jsonBundles.put(entry.getKey().toString(), extractResourceBundle(entry.getValue()));
}
all.put(CONNECTOR_CONFIGS, jsonBundles);
}
@ -129,34 +133,41 @@ public void restore(JSONObject jsonObject) {
String className = (String) object.get(CLASS);
String version = (String) object.get(VERSION);
MJobForms fromJob = null;
MJobForms toJob = null;
List<MForm> connForms = restoreForms((JSONArray) object.get(CON_FORMS));
JSONObject jobJson = (JSONObject) object.get(JOB_FORMS);
JSONArray fromJobJson = (JSONArray)jobJson.get(Direction.FROM.name());
JSONArray toJobJson = (JSONArray)jobJson.get(Direction.TO.name());
if (fromJobJson != null) {
List<MForm> fromJobForms = restoreForms(fromJobJson);
fromJob = new MJobForms(fromJobForms);
List<MConfig> linkConfigs = restoreConfigList((JSONArray) object.get(CONNECTOR_LINK_CONFIG));
// parent that encapsualtes both the from/to configs
JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (fromJobConfigJson != null) {
List<MConfig> fromJobConfig = restoreConfigList(fromJobConfigJson);
fromConfig = new MFromConfig(fromJobConfig);
}
if (toJobJson != null) {
List<MForm> toJobForms = restoreForms(toJobJson);
toJob = new MJobForms(toJobForms);
if (toJobConfigJson != null) {
List<MConfig> toJobConfig = restoreConfigList(toJobConfigJson);
toConfig = new MToConfig(toJobConfig);
}
MConnectionForms connection = new MConnectionForms(connForms);
MConnector connector = new MConnector(uniqueName, className, version,
connection, fromJob, toJob);
MLinkConfig linkConfig = new MLinkConfig(linkConfigs);
MConnector connector = new MConnector(uniqueName, className, version, linkConfig, fromConfig,
toConfig);
connector.setPersistenceId(connectorId);
connectors.add(connector);
}
if (jsonObject.containsKey(CONNECTOR_CONFIGS)) {
bundles = new HashMap<Long, ResourceBundle>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
JSONObject jsonBundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
Set<Map.Entry<String, JSONObject>> entrySet = jsonBundles.entrySet();
for (Map.Entry<String, JSONObject> entry : entrySet) {
bundles.put(Long.parseLong(entry.getKey()),
connectorConfigBundles.put(Long.parseLong(entry.getKey()),
restoreResourceBundle(entry.getValue()));
}
}

70
common/src/main/java/org/apache/sqoop/json/DriverConfigBean.java → common/src/main/java/org/apache/sqoop/json/DriverBean.java
View File

@ -17,56 +17,63 @@
*/
package org.apache.sqoop.json;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJobForms;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import static org.apache.sqoop.json.util.ConfigSerialization.DRIVER_CONFIG;
import static org.apache.sqoop.json.util.ConfigSerialization.DRIVER_VERSION;
import static org.apache.sqoop.json.util.ConfigSerialization.ID;
import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList;
import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONFIGS;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
import java.util.List;
import java.util.ResourceBundle;
import static org.apache.sqoop.json.util.FormSerialization.*;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
/**
* Json representation of the driver
*
*/
public class DriverBean implements JsonBean {
public class DriverConfigBean implements JsonBean {
public static final String CURRENT_DRIVER_VERSION = "1";
private MDriverConfig driverConfig;
private MDriver driver;
private ResourceBundle bundle;
// for "extract"
public DriverConfigBean(MDriverConfig driverConfig, ResourceBundle bundle) {
this.driverConfig = driverConfig;
public DriverBean(MDriver driver, ResourceBundle bundle) {
this.driver = driver;
this.bundle = bundle;
}
// for "restore"
public DriverConfigBean() {
public DriverBean() {
}
public MDriverConfig getDriverConfig() {
return driverConfig;
public MDriver getDriver() {
return driver;
}
public ResourceBundle getResourceBundle() {
public ResourceBundle getDriverConfigResourceBundle() {
return bundle;
}
@SuppressWarnings("unchecked")
@Override
public JSONObject extract(boolean skipSensitive) {
// TODO(Abe): Add From/To connection forms.
JSONArray conForms =
extractForms(driverConfig.getConnectionForms().getForms(), skipSensitive);
JSONArray jobForms = extractForms(driverConfig.getJobForms().getForms(), skipSensitive);
JSONArray configs =
extractConfigList(driver.getDriverConfig().getConfigs(), skipSensitive);
JSONObject result = new JSONObject();
result.put(ID, driverConfig.getPersistenceId());
result.put(DRIVER_VERSION, driverConfig.getVersion());
result.put(CON_FORMS, conForms);
result.put(JOB_FORMS, jobForms);
result.put(ID, driver.getPersistenceId());
result.put(DRIVER_VERSION, driver.getVersion());
result.put(DRIVER_CONFIG, configs);
result.put(CONFIGS, extractResourceBundle(bundle));
return result;
}
@ -75,18 +82,9 @@ public JSONObject extract(boolean skipSensitive) {
public void restore(JSONObject jsonObject) {
long id = (Long) jsonObject.get(ID);
String driverVersion = (String) jsonObject.get(DRIVER_VERSION);
List<MForm> connForms = restoreForms((JSONArray) jsonObject.get(CON_FORMS));
List<MForm> jobForms = restoreForms((JSONArray) jsonObject.get(JOB_FORMS));
// TODO(Abe): Get From/To connection forms.
driverConfig = new MDriverConfig(
new MConnectionForms(connForms),
new MJobForms(jobForms),
driverVersion);
driverConfig.setPersistenceId(id);
List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_CONFIG));
driver = new MDriver(new MDriverConfig(driverConfig), driverVersion);
driver.setPersistenceId(id);
bundle = restoreResourceBundle((JSONObject) jsonObject.get(CONFIGS));
}
}

73
common/src/main/java/org/apache/sqoop/json/JobBean.java
View File

@ -17,12 +17,17 @@
*/
package org.apache.sqoop.json;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import static org.apache.sqoop.json.util.ConfigSerialization.CREATION_DATE;
import static org.apache.sqoop.json.util.ConfigSerialization.CREATION_USER;
import static org.apache.sqoop.json.util.ConfigSerialization.ENABLED;
import static org.apache.sqoop.json.util.ConfigSerialization.UPDATE_DATE;
import static org.apache.sqoop.json.util.ConfigSerialization.UPDATE_USER;
import static org.apache.sqoop.json.util.ConfigSerialization.extractConfigList;
import static org.apache.sqoop.json.util.ConfigSerialization.restoreConfigList;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.CONNECTOR_CONFIGS;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.DRIVER_CONFIGS;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.extractResourceBundle;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.restoreResourceBundle;
import java.util.ArrayList;
import java.util.Date;
@ -32,11 +37,18 @@
import java.util.ResourceBundle;
import java.util.Set;
import static org.apache.sqoop.json.util.FormSerialization.*;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MToConfig;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
/**
*
* Json representation of the job config
*/
public class JobBean implements JsonBean {
@ -47,9 +59,9 @@ public class JobBean implements JsonBean {
private static final String TO_LINK_ID = "to-link-id";
private static final String FROM_CONNECTOR_ID = "from-connector-id";
private static final String TO_CONNECTOR_ID = "to-connector-id";
private static final String FROM_CONNECTOR_PART = "from-connector";
private static final String TO_CONNECTOR_PART = "to-connector";
private static final String FRAMEWORK_PART = "framework";
private static final String FROM_CONFIG = "from-config";
private static final String TO_CONFIG = "to-config";
private static final String DRIVER_CONFIG = "driver-config";
// Required
private List<MJob> jobs;
@ -114,16 +126,19 @@ public JSONObject extract(boolean skipSensitive) {
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job link associated connectors
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
object.put(FROM_CONNECTOR_PART,
extractForms(job.getConnectorPart(Direction.FROM).getForms(),skipSensitive));
object.put(TO_CONNECTOR_PART,
extractForms(job.getConnectorPart(Direction.TO).getForms(), skipSensitive));
object.put(FRAMEWORK_PART,
extractForms(job.getFrameworkPart().getForms(), skipSensitive));
// job associated links
object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job configs
object.put(FROM_CONFIG, extractConfigList(job
.getJobConfig(Direction.FROM).getConfigs(), skipSensitive));
object.put(TO_CONFIG,
extractConfigList(job.getJobConfig(Direction.TO).getConfigs(), skipSensitive));
object.put(DRIVER_CONFIG,
extractConfigList(job.getDriverConfig().getConfigs(), skipSensitive));
array.add(object);
}
@ -160,22 +175,22 @@ public void restore(JSONObject jsonObject) {
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConnectorPart = (JSONArray) object.get(FROM_CONNECTOR_PART);
JSONArray toConnectorPart = (JSONArray) object.get(TO_CONNECTOR_PART);
JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG);
JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG);
JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG);
List<MForm> fromConnectorParts = restoreForms(fromConnectorPart);
List<MForm> toConnectorParts = restoreForms(toConnectorPart);
List<MForm> frameworkForms = restoreForms(frameworkPart);
List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
List<MConfig> toConfig = restoreConfigList(toConfigJson);
List<MConfig> driverConfig = restoreConfigList(driverConfigJson);
MJob job = new MJob(
fromConnectorId,
toConnectorId,
fromConnectionId,
toConnectionId,
new MJobForms(fromConnectorParts),
new MJobForms(toConnectorParts),
new MJobForms(frameworkForms)
new MFromConfig(fromConfig),
new MToConfig(toConfig),
new MDriverConfig(driverConfig)
);
job.setPersistenceId((Long) object.get(ID));

83
common/src/main/java/org/apache/sqoop/json/JobValidationBean.java
View File

@ -21,41 +21,41 @@
import org.apache.sqoop.common.DirectionError;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.ConfigValidator;
import org.json.simple.JSONObject;
import java.util.HashMap;
import java.util.Map;
/**
* Bean for sending validations across network. This bean will move two
* validation objects at one time - one for connector and second for framework
* part of validated entity. Optionally validation bean can also transfer
* Bean for sending validations across network. This bean will send job validation results
* Optionally validation bean can also transfer
* created persistent id in case that new entity was created.
*/
public class JobValidationBean implements JsonBean {
private static final String ID = "id";
private static final String FRAMEWORK = "framework";
private static final String CONNECTOR = "connector";
private static final String JOB = "job";
private static final String FROM = "from";
private static final String TO = "to";
private static final String DRIVER = "driver";
private static final String STATUS = "status";
private static final String MESSAGE = "message";
private static final String MESSAGES = "messages";
private Long id;
private Validation fromConnectorValidation;
private Validation toConnectorValidation;
private Validation frameworkValidation;
private ConfigValidator fromConfigValidation;
private ConfigValidator toConfigValidation;
private ConfigValidator driverConfigValidation;
// For "extract"
public JobValidationBean(Validation fromConnector, Validation framework, Validation toConnector) {
public JobValidationBean(ConfigValidator fromConnector, ConfigValidator framework, ConfigValidator toConnector) {
this();
this.fromConnectorValidation = fromConnector;
this.toConnectorValidation = toConnector;
this.frameworkValidation = framework;
this.fromConfigValidation = fromConnector;
this.toConfigValidation = toConnector;
this.driverConfigValidation = framework;
}
// For "restore"
@ -63,21 +63,21 @@ public JobValidationBean() {
id = null;
}
public Validation getConnectorValidation(Direction type) {
public ConfigValidator getConnectorValidation(Direction type) {
switch(type) {
case FROM:
return fromConnectorValidation;
return fromConfigValidation;
case TO:
return toConnectorValidation;
return toConfigValidation;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
public Validation getFrameworkValidation() {
return frameworkValidation;
public ConfigValidator getFrameworkValidation() {
return driverConfigValidation;
}
public void setId(Long id) {
@ -91,32 +91,30 @@ public Long getId() {
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONObject object = new JSONObject();
JSONObject connectorObject = new JSONObject();
JSONObject jobObject = new JSONObject();
// Optionally transfer id
if(id != null) {
object.put(ID, id);
}
connectorObject.put(FROM, extractValidation(getConnectorValidation(Direction.FROM)));
connectorObject.put(TO, extractValidation(getConnectorValidation(Direction.TO)));
object.put(FRAMEWORK, extractValidation(frameworkValidation));
object.put(CONNECTOR, connectorObject);
jobObject.put(FROM, extractValidation(getConnectorValidation(Direction.FROM)));
jobObject.put(TO, extractValidation(getConnectorValidation(Direction.TO)));
jobObject.put(DRIVER, extractValidation(driverConfigValidation));
object.put(JOB, jobObject);
return object;
}
@SuppressWarnings("unchecked")
private JSONObject extractValidation(Validation validation) {
private JSONObject extractValidation(ConfigValidator validation) {
JSONObject object = new JSONObject();
object.put(STATUS, validation.getStatus().name());
JSONObject jsonMessages = new JSONObject();
Map<Validation.FormInput, Validation.Message> messages = validation.getMessages();
Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = validation.getMessages();
for(Map.Entry<Validation.FormInput, Validation.Message> entry : messages.entrySet()) {
for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : messages.entrySet()) {
JSONObject jsonEntry = new JSONObject();
jsonEntry.put(STATUS, entry.getValue().getStatus().name());
jsonEntry.put(MESSAGE, entry.getValue().getMessage());
@ -133,20 +131,21 @@ public void restore(JSONObject jsonObject) {
// Optional and accepting NULLs
id = (Long) jsonObject.get(ID);
JSONObject jsonConnectorObject = (JSONObject)jsonObject.get(CONNECTOR);
JSONObject jobJsonObject = (JSONObject)jsonObject.get(JOB);
fromConnectorValidation = restoreValidation(
(JSONObject)jsonConnectorObject.get(FROM));
toConnectorValidation = restoreValidation(
(JSONObject)jsonConnectorObject.get(TO));
frameworkValidation = restoreValidation(
(JSONObject)jsonObject.get(FRAMEWORK));
fromConfigValidation = restoreValidation(
(JSONObject)jobJsonObject.get(FROM));
toConfigValidation = restoreValidation(
(JSONObject)jobJsonObject.get(TO));
driverConfigValidation = restoreValidation(
(JSONObject)jobJsonObject.get(DRIVER));
}
public Validation restoreValidation(JSONObject jsonObject) {
public ConfigValidator restoreValidation(JSONObject jsonObject) {
JSONObject jsonMessages = (JSONObject) jsonObject.get(MESSAGES);
Map<Validation.FormInput, Validation.Message> messages
= new HashMap<Validation.FormInput, Validation.Message>();
Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages
= new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
for(Object key : jsonMessages.keySet()) {
JSONObject jsonMessage = (JSONObject) jsonMessages.get(key);
@ -154,14 +153,14 @@ public Validation restoreValidation(JSONObject jsonObject) {
Status status = Status.valueOf((String) jsonMessage.get(STATUS));
String stringMessage = (String) jsonMessage.get(MESSAGE);
Validation.Message message
= new Validation.Message(status, stringMessage);
ConfigValidator.Message message
= new ConfigValidator.Message(status, stringMessage);
messages.put(new Validation.FormInput((String)key), message);
messages.put(new ConfigValidator.ConfigInput((String)key), message);
}
Status status = Status.valueOf((String) jsonObject.get(STATUS));
return new Validation(status, messages);
return new ConfigValidator(status, messages);
}
}

92
common/src/main/java/org/apache/sqoop/json/LinkBean.java
View File

@ -18,8 +18,8 @@
package org.apache.sqoop.json;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MConfig;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@ -31,7 +31,7 @@
import java.util.ResourceBundle;
import java.util.Set;
import static org.apache.sqoop.json.util.FormSerialization.*;
import static org.apache.sqoop.json.util.ConfigSerialization.*;
import static org.apache.sqoop.json.util.ResourceBundleSerialization.*;
/**
@ -42,16 +42,14 @@
*/
public class LinkBean implements JsonBean {
private static final String CONNECTOR_ID = "connector-id";
private static final String CONNECTOR_PART = "connector";
private static final String FRAMEWORK_PART = "framework";
static final String CONNECTOR_ID = "connector-id";
static final String LINK_CONFIG = "link-config";
// Required
private List<MLink> links;
// Optional
private Map<Long, ResourceBundle> connectorConfigBundles;
private ResourceBundle driverConfigBundle;
private Map<Long, ResourceBundle> linkConfigBundles;
// For "extract"
public LinkBean(MLink link) {
@ -67,72 +65,56 @@ public LinkBean(List<MLink> links) {
// For "restore"
public LinkBean() {
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
}
public void setDriverConfigBundle(ResourceBundle driverConfigBundle) {
this.driverConfigBundle = driverConfigBundle;
linkConfigBundles = new HashMap<Long, ResourceBundle>();
}
public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) {
connectorConfigBundles.put(id, connectorConfigBundle);
linkConfigBundles.put(id, connectorConfigBundle);
}
public boolean hasConnectorBundle(Long id) {
return connectorConfigBundles.containsKey(id);
public boolean hasConnectorConfigBundle(Long id) {
return linkConfigBundles.containsKey(id);
}
public List<MLink> getLinks() {
return links;
}
public ResourceBundle getConnectorBundle(Long id) {
return connectorConfigBundles.get(id);
}
public ResourceBundle getFrameworkBundle() {
return driverConfigBundle;
public ResourceBundle getConnectorConfigBundle(Long id) {
return linkConfigBundles.get(id);
}
@Override
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONArray array = new JSONArray();
JSONArray linkArray = new JSONArray();
for(MLink link : links) {
JSONObject object = new JSONObject();
JSONObject linkJsonObject = new JSONObject();
object.put(ID, link.getPersistenceId());
object.put(NAME, link.getName());
object.put(ENABLED, link.getEnabled());
object.put(CREATION_USER, link.getCreationUser());
object.put(CREATION_DATE, link.getCreationDate().getTime());
object.put(UPDATE_USER, link.getLastUpdateUser());
object.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
object.put(CONNECTOR_ID, link.getConnectorId());
object.put(CONNECTOR_PART,
extractForms(link.getConnectorPart().getForms(), skipSensitive));
object.put(FRAMEWORK_PART,
extractForms(link.getFrameworkPart().getForms(), skipSensitive));
linkJsonObject.put(ID, link.getPersistenceId());
linkJsonObject.put(NAME, link.getName());
linkJsonObject.put(ENABLED, link.getEnabled());
linkJsonObject.put(CREATION_USER, link.getCreationUser());
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(LINK_CONFIG,
extractConfigList(link.getConnectorLinkConfig().getConfigs(), skipSensitive));
array.add(object);
linkArray.add(linkJsonObject);
}
JSONObject all = new JSONObject();
all.put(ALL, array);
if(!connectorConfigBundles.isEmpty()) {
all.put(ALL, linkArray);
if (!linkConfigBundles.isEmpty()) {
JSONObject bundles = new JSONObject();
for(Map.Entry<Long, ResourceBundle> entry : connectorConfigBundles.entrySet()) {
bundles.put(entry.getKey().toString(),
extractResourceBundle(entry.getValue()));
for (Map.Entry<Long, ResourceBundle> entry : linkConfigBundles.entrySet()) {
bundles.put(entry.getKey().toString(), extractResourceBundle(entry.getValue()));
}
all.put(CONNECTOR_CONFIGS, bundles);
}
if(driverConfigBundle != null) {
all.put(DRIVER_CONFIGS,extractResourceBundle(driverConfigBundle));
}
return all;
}
@ -147,15 +129,11 @@ public void restore(JSONObject jsonObject) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(CONNECTOR_ID);
JSONArray connectorPart = (JSONArray) object.get(CONNECTOR_PART);
JSONArray frameworkPart = (JSONArray) object.get(FRAMEWORK_PART);
JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG);
List<MForm> connectorForms = restoreForms(connectorPart);
List<MForm> frameworkForms = restoreForms(frameworkPart);
List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
MLink link = new MLink(connectorId,
new MConnectionForms(connectorForms),
new MConnectionForms(frameworkForms));
MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID));
link.setName((String) object.get(NAME));
@ -172,13 +150,9 @@ public void restore(JSONObject jsonObject) {
JSONObject bundles = (JSONObject) jsonObject.get(CONNECTOR_CONFIGS);
Set<Map.Entry<String, JSONObject>> entrySet = bundles.entrySet();
for (Map.Entry<String, JSONObject> entry : entrySet) {
connectorConfigBundles.put(Long.parseLong(entry.getKey()),
linkConfigBundles.put(Long.parseLong(entry.getKey()),
restoreResourceBundle(entry.getValue()));
}
}
if(jsonObject.containsKey(DRIVER_CONFIGS)) {
driverConfigBundle = restoreResourceBundle(
(JSONObject) jsonObject.get(DRIVER_CONFIGS));
}
}
}

56
common/src/main/java/org/apache/sqoop/json/LinkValidationBean.java
View File

@ -18,37 +18,32 @@
package org.apache.sqoop.json;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.ConfigValidator;
import org.json.simple.JSONObject;
import java.util.HashMap;
import java.util.Map;
/**
* Bean for sending validations across network. This bean will move two
* validation objects at one time - one for connector and second for framework
* part of validated entity. Optionally validation bean can also transfer
* Bean for sending validations across network.This bean will transfer link config
* validation results. Optionally validation bean can also transfer
* created persistent id in case that new entity was created.
*/
public class LinkValidationBean implements JsonBean {
private static final String ID = "id";
private static final String FRAMEWORK = "framework";
private static final String CONNECTOR = "connector";
private static final String STATUS = "status";
private static final String MESSAGE = "message";
private static final String MESSAGES = "messages";
private Long id;
private Validation connectorValidation;
private Validation frameworkValidation;
private ConfigValidator linkConfigValidation;
// For "extract"
public LinkValidationBean(Validation connector, Validation framework) {
public LinkValidationBean(ConfigValidator linkConfigValidator) {
this();
this.connectorValidation = connector;
this.frameworkValidation = framework;
this.linkConfigValidation = linkConfigValidator;
}
// For "restore"
@ -56,12 +51,8 @@ public LinkValidationBean() {
id = null;
}
public Validation getConnectorValidation() {
return connectorValidation;
}
public Validation getFrameworkValidation() {
return frameworkValidation;
public ConfigValidator getLinkConfigValidator() {
return linkConfigValidation;
}
public void setId(Long id) {
@ -80,23 +71,20 @@ public JSONObject extract(boolean skipSensitive) {
if(id != null) {
object.put(ID, id);
}
object.put(CONNECTOR, extractValidation(connectorValidation));
object.put(FRAMEWORK, extractValidation(frameworkValidation));
object.put(LinkBean.LINK_CONFIG, extractValidation(linkConfigValidation));
return object;
}
@SuppressWarnings("unchecked")
private JSONObject extractValidation(Validation validation) {
private JSONObject extractValidation(ConfigValidator validation) {
JSONObject object = new JSONObject();
object.put(STATUS, validation.getStatus().name());
JSONObject jsonMessages = new JSONObject();
Map<Validation.FormInput, Validation.Message> messages = validation.getMessages();
Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = validation.getMessages();
for(Map.Entry<Validation.FormInput, Validation.Message> entry : messages.entrySet()) {
for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : messages.entrySet()) {
JSONObject jsonEntry = new JSONObject();
jsonEntry.put(STATUS, entry.getValue().getStatus().name());
jsonEntry.put(MESSAGE, entry.getValue().getMessage());
@ -113,16 +101,14 @@ public void restore(JSONObject jsonObject) {
// Optional and accepting NULLs
id = (Long) jsonObject.get(ID);
connectorValidation = restoreValidation(
(JSONObject)jsonObject.get(CONNECTOR));
frameworkValidation = restoreValidation(
(JSONObject)jsonObject.get(FRAMEWORK));
linkConfigValidation = restoreValidation(
(JSONObject)jsonObject.get(LinkBean.LINK_CONFIG));
}
public Validation restoreValidation(JSONObject jsonObject) {
public ConfigValidator restoreValidation(JSONObject jsonObject) {
JSONObject jsonMessages = (JSONObject) jsonObject.get(MESSAGES);
Map<Validation.FormInput, Validation.Message> messages
= new HashMap<Validation.FormInput, Validation.Message>();
Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages
= new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
for(Object key : jsonMessages.keySet()) {
JSONObject jsonMessage = (JSONObject) jsonMessages.get(key);
@ -130,14 +116,14 @@ public Validation restoreValidation(JSONObject jsonObject) {
Status status = Status.valueOf((String) jsonMessage.get(STATUS));
String stringMessage = (String) jsonMessage.get(MESSAGE);
Validation.Message message
= new Validation.Message(status, stringMessage);
ConfigValidator.Message message
= new ConfigValidator.Message(status, stringMessage);
messages.put(new Validation.FormInput((String)key), message);
messages.put(new ConfigValidator.ConfigInput((String)key), message);
}
Status status = Status.valueOf((String) jsonObject.get(STATUS));
return new Validation(status, messages);
return new ConfigValidator(status, messages);
}
}

2
common/src/main/java/org/apache/sqoop/json/ThrowableBean.java
View File

@ -26,7 +26,7 @@
import java.util.List;
/**
* Transfer throwable instance.
* Transfer throwable instance as a throwable bean.
*/
public class ThrowableBean implements JsonBean {

18
common/src/main/java/org/apache/sqoop/json/ValidationResultBean.java
View File

@ -19,7 +19,7 @@
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.ValidationResult;
import org.apache.sqoop.validation.ConfigValidationResult;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@ -38,18 +38,18 @@ public class ValidationResultBean implements JsonBean {
private static final String STATUS = "STATUS";
private static final String TEXT = "TEXT";
private ValidationResult[] results;
private ConfigValidationResult[] results;
private Long id;
public ValidationResultBean() {
// Empty, for restore
}
public ValidationResultBean(ValidationResult ... results) {
public ValidationResultBean(ConfigValidationResult ... results) {
this.results = results;
}
public ValidationResult[] getValidationResults() {
public ConfigValidationResult[] getValidationResults() {
return results;
}
@ -65,7 +65,7 @@ public Long getId() {
public JSONObject extract(boolean skipSensitive) {
JSONArray array = new JSONArray();
for(ValidationResult result : results) {
for(ConfigValidationResult result : results) {
JSONObject output = extractValidationResult(result);
array.add(output);
}
@ -78,7 +78,7 @@ public JSONObject extract(boolean skipSensitive) {
return object;
}
private JSONObject extractValidationResult(ValidationResult result) {
private JSONObject extractValidationResult(ConfigValidationResult result) {
JSONObject ret = new JSONObject();
for(Map.Entry<String, List<Message>> entry : result.getMessages().entrySet()) {
@ -110,7 +110,7 @@ private Object extractMessage(Message message) {
@Override
public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(ROOT);
results = new ValidationResult[array.size()];
results = new ConfigValidationResult[array.size()];
int i = 0;
for(Object item : array) {
@ -122,8 +122,8 @@ public void restore(JSONObject jsonObject) {
}
}
private ValidationResult restoreValidationResult(JSONObject item) {
ValidationResult result = new ValidationResult();
private ConfigValidationResult restoreValidationResult(JSONObject item) {
ConfigValidationResult result = new ConfigValidationResult();
Set<Map.Entry<String, JSONArray>> entrySet = item.entrySet();
for(Map.Entry<String, JSONArray> entry : entrySet) {

139
common/src/main/java/org/apache/sqoop/json/util/FormSerialization.java → common/src/main/java/org/apache/sqoop/json/util/ConfigSerialization.java
View File

@ -21,8 +21,8 @@
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.MBooleanInput;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MFormType;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConfigType;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MInputType;
import org.apache.sqoop.model.MIntegerInput;
@ -36,83 +36,86 @@
import java.util.Map;
/**
* Convenient static methods for serializing forms.
* Convenient static methods for serializing config objects.
*/
public final class FormSerialization {
public final class ConfigSerialization {
public static final String ALL = "all";
public static final String ID = "id";
public static final String NAME = "name";
public static final String VERSION = "version";
public static final String DRIVER_VERSION = "driver-version";
public static final String CLASS = "class";
public static final String ENABLED = "enabled";
public static final String CREATION_USER = "creation-user";
public static final String CREATION_DATE = "creation-date";
public static final String UPDATE_USER = "update-user";
public static final String UPDATE_DATE = "update-date";
public static final String CON_FORMS = "con-forms";
public static final String JOB_FORMS = "job-forms";
// TODO(VB): Move these constants to connector bean
public static final String CONNECTOR_LINK_CONFIG = "link-config";
public static final String CONNECTOR_JOB_CONFIG = "job-config";
// TODO:move these configs to driver bean
public static final String DRIVER_VERSION = "driver-version";
public static final String DRIVER_CONFIG = "driver-config";
public static final String FORM_NAME = "name";
public static final String FORM_TYPE = "type";
public static final String FORM_INPUTS = "inputs";
public static final String FORM_INPUT_NAME = "name";
public static final String FORM_INPUT_TYPE = "type";
public static final String FORM_INPUT_SENSITIVE = "sensitive";
public static final String FORM_INPUT_SIZE = "size";
public static final String FORM_INPUT_VALUE = "value";
public static final String FORM_INPUT_VALUES = "values";
public static final String CONFIG_NAME = "name";
public static final String CONFIG_TYPE = "type";
public static final String CONFIG_INPUTS = "inputs";
public static final String CONFIG_INPUT_NAME = "name";
public static final String CONFIG_INPUT_TYPE = "type";
public static final String CONFIG_INPUT_SENSITIVE = "sensitive";
public static final String CONFIG_INPUT_SIZE = "size";
public static final String CONFIG_INPUT_VALUE = "value";
public static final String CONFIG_INPUT_VALUES = "values";
/**
* Transform given list of forms to JSON Array object.
* Transform given list of configs to JSON Array object.
*
* @param mForms List of forms.
* @return JSON object with serialized form of the list.
* @param mConfigs List of configs.
* @return JSON object with serialized config of the list.
*/
@SuppressWarnings("unchecked")
public static JSONArray extractForms(List<MForm> mForms, boolean skipSensitive) {
JSONArray forms = new JSONArray();
public static JSONArray extractConfigList(List<MConfig> mConfigs, boolean skipSensitive) {
JSONArray configs = new JSONArray();
for (MForm mForm : mForms) {
forms.add(extractForm(mForm, skipSensitive));
for (MConfig mConfig : mConfigs) {
configs.add(extractConfig(mConfig, skipSensitive));
}
return forms;
return configs;
}
/**
* Transform given form to JSON Object.
* Transform given config to JSON Object.
*
* @param mForm Given MForm instance
* @param mConfig Given MConfig instance
* @param skipSensitive conditionally add sensitive input values
* @return Serialized JSON object.
*/
@SuppressWarnings("unchecked")
public static JSONObject extractForm(MForm mForm, boolean skipSensitive) {
JSONObject form = new JSONObject();
form.put(ID, mForm.getPersistenceId());
form.put(FORM_NAME, mForm.getName());
form.put(FORM_TYPE, MFormType.CONNECTION.toString());
static JSONObject extractConfig(MConfig mConfig, boolean skipSensitive) {
JSONObject config = new JSONObject();
config.put(ID, mConfig.getPersistenceId());
config.put(CONFIG_NAME, mConfig.getName());
config.put(CONFIG_TYPE, MConfigType.LINK.toString());
JSONArray mInputs = new JSONArray();
form.put(FORM_INPUTS, mInputs);
config.put(CONFIG_INPUTS, mInputs);
for (MInput<?> mInput : mForm.getInputs()) {
for (MInput<?> mInput : mConfig.getInputs()) {
JSONObject input = new JSONObject();
input.put(ID, mInput.getPersistenceId());
input.put(FORM_INPUT_NAME, mInput.getName());
input.put(FORM_INPUT_TYPE, mInput.getType().toString());
input.put(FORM_INPUT_SENSITIVE, mInput.isSensitive());
input.put(CONFIG_INPUT_NAME, mInput.getName());
input.put(CONFIG_INPUT_TYPE, mInput.getType().toString());
input.put(CONFIG_INPUT_SENSITIVE, mInput.isSensitive());
// String specific serialization
if (mInput.getType() == MInputType.STRING) {
input.put(FORM_INPUT_SIZE,
input.put(CONFIG_INPUT_SIZE,
((MStringInput)mInput).getMaxLength());
}
// Enum specific serialization
if(mInput.getType() == MInputType.ENUM) {
input.put(FORM_INPUT_VALUES,
input.put(CONFIG_INPUT_VALUES,
StringUtils.join(((MEnumInput)mInput).getValues(), ","));
}
@ -120,54 +123,54 @@ public static JSONObject extractForm(MForm mForm, boolean skipSensitive) {
// Skip if sensitive
if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) {
if (mInput.getType() == MInputType.MAP) {
input.put(FORM_INPUT_VALUE, mInput.getValue());
input.put(CONFIG_INPUT_VALUE, mInput.getValue());
} else {
input.put(FORM_INPUT_VALUE, mInput.getUrlSafeValueString());
input.put(CONFIG_INPUT_VALUE, mInput.getUrlSafeValueString());
}
}
mInputs.add(input);
}
return form;
return config;
}
/**
* Restore List of MForms from JSON Array.
* Restore List of MConfigs from JSON Array.
*
* @param forms JSON array representing list of MForms
* @return Restored list of MForms
* @param configs JSON array representing list of MConfigs
* @return Restored list of MConfigs
*/
public static List<MForm> restoreForms(JSONArray forms) {
List<MForm> mForms = new ArrayList<MForm>();
public static List<MConfig> restoreConfigList(JSONArray configs) {
List<MConfig> mConfigs = new ArrayList<MConfig>();
for (int i = 0; i < forms.size(); i++) {
mForms.add(restoreForm((JSONObject) forms.get(i)));
for (int i = 0; i < configs.size(); i++) {
mConfigs.add(restoreConfig((JSONObject) configs.get(i)));
}
return mForms;
return mConfigs;
}
/**
* Restore one MForm from JSON Object.
* Restore one MConfig from JSON Object.
*
* @param form JSON representation of the MForm.
* @return Restored MForm.
* @param config JSON representation of the MConfig.
* @return Restored MConfig.
*/
public static MForm restoreForm(JSONObject form) {
JSONArray inputs = (JSONArray) form.get(FORM_INPUTS);
static MConfig restoreConfig(JSONObject config) {
JSONArray inputs = (JSONArray) config.get(CONFIG_INPUTS);
List<MInput<?>> mInputs = new ArrayList<MInput<?>>();
for (int i = 0; i < inputs.size(); i++) {
JSONObject input = (JSONObject) inputs.get(i);
MInputType type =
MInputType.valueOf((String) input.get(FORM_INPUT_TYPE));
String name = (String) input.get(FORM_INPUT_NAME);
Boolean sensitive = (Boolean) input.get(FORM_INPUT_SENSITIVE);
MInputType.valueOf((String) input.get(CONFIG_INPUT_TYPE));
String name = (String) input.get(CONFIG_INPUT_NAME);
Boolean sensitive = (Boolean) input.get(CONFIG_INPUT_SENSITIVE);
MInput mInput = null;
switch (type) {
case STRING: {
long size = (Long) input.get(FORM_INPUT_SIZE);
long size = (Long) input.get(CONFIG_INPUT_SIZE);
mInput = new MStringInput(name, sensitive.booleanValue(), (short) size);
break;
}
@ -184,40 +187,40 @@ public static MForm restoreForm(JSONObject form) {
break;
}
case ENUM: {
String values = (String) input.get(FORM_INPUT_VALUES);
String values = (String) input.get(CONFIG_INPUT_VALUES);
mInput = new MEnumInput(name, sensitive.booleanValue(), values.split(","));
break;
}
}
// Propagate form ID
// Propagate config ID
mInput.setPersistenceId((Long)input.get(ID));
// Propagate form optional value
if(input.containsKey(FORM_INPUT_VALUE)) {
// Propagate config optional value
if(input.containsKey(CONFIG_INPUT_VALUE)) {
switch (type) {
case MAP:
try {
mInput.setValue((Map<String, String>)input.get(FORM_INPUT_VALUE));
mInput.setValue((Map<String, String>)input.get(CONFIG_INPUT_VALUE));
} catch (ClassCastException e) {
throw new SqoopException(SerializationError.SERIALIZATION_001, name + " requires a 'map' value.");
}
break;
default:
mInput.restoreFromUrlSafeValueString(
(String) input.get(FORM_INPUT_VALUE));
(String) input.get(CONFIG_INPUT_VALUE));
break;
}
}
mInputs.add(mInput);
}
MForm mForm = new MForm((String) form.get(FORM_NAME), mInputs);
mForm.setPersistenceId((Long) form.get(ID));
return mForm;
MConfig mConfig = new MConfig((String) config.get(CONFIG_NAME), mInputs);
mConfig.setPersistenceId((Long) config.get(ID));
return mConfig;
}
private FormSerialization() {
private ConfigSerialization() {
// Do not instantiate
}
}

2
common/src/main/java/org/apache/sqoop/json/util/ResourceBundleSerialization.java
View File

@ -43,7 +43,6 @@ public static JSONArray extractResourceBundles(List<ResourceBundle> bundles) {
for (ResourceBundle bundle : bundles) {
array.add(extractResourceBundle(bundle));
}
return array;
}
@ -58,7 +57,6 @@ public static JSONObject extractResourceBundle(ResourceBundle bundle) {
return json;
}
@SuppressWarnings("unchecked")
public static List<ResourceBundle> restoreResourceBundles(JSONArray array) {
List<ResourceBundle> bundles = new LinkedList<ResourceBundle>();
for (Object item : array) {

5
common/src/main/java/org/apache/sqoop/model/Form.java → common/src/main/java/org/apache/sqoop/model/Config.java
View File

@ -21,11 +21,10 @@
import java.lang.annotation.RetentionPolicy;
/**
* Denote form in Configuration class
* Denote config in Configuration class
*/
@Retention(RetentionPolicy.RUNTIME)
public @interface Form {
public @interface Config {
/**
* Optional name for the form object
*

6
common/src/main/java/org/apache/sqoop/model/FormClass.java → common/src/main/java/org/apache/sqoop/model/ConfigClass.java
View File

@ -27,17 +27,17 @@
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface FormClass {
public @interface ConfigClass {
/**
* Default size for Inputs in this form.
* Default size for Inputs in this config.
*
* @return
*/
short defaultSize() default -1;
/**
* List of validators associated with this form.
* List of validators associated with this config.
*
* @return
*/

241
common/src/main/java/org/apache/sqoop/model/FormUtils.java → common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
View File

@ -22,8 +22,8 @@
import org.apache.sqoop.utils.ClassUtils;
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.ValidationResult;
import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.ConfigValidationResult;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
@ -42,81 +42,80 @@
*
* TODO: This class should see some overhaul into more reusable code, especially expose and re-use the methods at the end.
*/
public class FormUtils {
public class ConfigUtils {
/**
* Transform correctly annotated configuration object to corresponding
* list of forms.
* list of configs.
*
* Forms will be order according to the occurrence in the configuration
* Configs will be order according to the occurrence in the configuration
* class. Inputs will be also ordered based on occurrence.
*
* @param configuration Annotated arbitrary configuration object
* @return Corresponding list of forms
* @return Corresponding list of configs
*/
public static List<MForm> toForms(Object configuration) {
return toForms(configuration.getClass(), configuration);
public static List<MConfig> toConfigs(Object configuration) {
return toConfigs(configuration.getClass(), configuration);
}
public static List<MForm> toForms(Class klass) {
return toForms(klass, null);
public static List<MConfig> toConfigs(Class klass) {
return toConfigs(klass, null);
}
@SuppressWarnings("unchecked")
public static List<MForm> toForms(Class klass, Object configuration) {
public static List<MConfig> toConfigs(Class klass, Object configuration) {
Set<String> formNames = new HashSet<String>();
ConfigurationClass global =
ConfigurationClass configurationClass =
(ConfigurationClass)klass.getAnnotation(ConfigurationClass.class);
// Each configuration object must have this class annotation
if(global == null) {
if(configurationClass == null) {
throw new SqoopException(ModelError.MODEL_003,
"Missing annotation ConfigurationClass on class " + klass.getName());
}
List<MForm> forms = new LinkedList<MForm>();
List<MConfig> configs = new LinkedList<MConfig>();
// Iterate over all declared fields
for (Field formField : klass.getDeclaredFields()) {
formField.setAccessible(true);
for (Field field : klass.getDeclaredFields()) {
field.setAccessible(true);
// Each field that should be part of user input should have Input
// annotation.
Form formAnnotation = formField.getAnnotation(Form.class);
Config formAnnotation = field.getAnnotation(Config.class);
if (formAnnotation != null) {
String formName = getFormName(formField, formAnnotation, formNames);
String formName = getFormName(field, formAnnotation, formNames);
Class type = formField.getType();
Class type = field.getType();
Object value = null;
if(configuration != null) {
try {
value = formField.get(configuration);
value = field.get(configuration);
} catch (IllegalAccessException e) {
throw new SqoopException(ModelError.MODEL_005,
"Can't retrieve value from " + formField.getName(), e);
"Can't retrieve value from " + field.getName(), e);
}
}
forms.add(toForm(formName, type, value));
configs.add(toConfig(formName, type, value));
}
}
return forms;
return configs;
}
@SuppressWarnings("unchecked")
private static MForm toForm(String formName, Class<?> klass, Object object) {
FormClass global =
(FormClass)klass.getAnnotation(FormClass.class);
private static MConfig toConfig(String formName, Class klass, Object object) {
ConfigClass global =
(ConfigClass)klass.getAnnotation(ConfigClass.class);
// Each configuration object must have this class annotation
if(global == null) {
throw new SqoopException(ModelError.MODEL_003,
"Missing annotation FormClass on class " + klass.getName());
"Missing annotation ConfigClass on class " + klass.getName());
}
// Intermediate list of inputs
@ -183,7 +182,7 @@ private static MForm toForm(String formName, Class<?> klass, Object object) {
}
}
return new MForm(formName, inputs);
return new MConfig(formName, inputs);
}
private static Field getFieldFromName(Class<?> klass, String name) {
@ -194,7 +193,7 @@ private static Field getFieldFromName(Class<?> klass, String name) {
// reverse lookup form field from custom form name
if (name != null) {
for (Field field : klass.getDeclaredFields()) {
Form formAnnotation = field.getAnnotation(Form.class);
Config formAnnotation = field.getAnnotation(Config.class);
if (formAnnotation == null) {
continue;
}
@ -210,29 +209,35 @@ private static Field getFieldFromName(Class<?> klass, String name) {
}
/**
* Move form values from form list into corresponding configuration object.
* Move config values from config list into corresponding configuration object.
*
* @param forms
* Input form list
* @param configuration
* Output configuration object
* @param configs Input config list
* @param configuration Output configuration object
*/
public static void fromForms(List<MForm> forms, Object configuration) {
Class<?> klass = configuration.getClass();
public static void fromConfigs(List<MConfig> configs, Object configuration) {
Class klass = configuration.getClass();
for (MForm form : forms) {
Field formField = getFieldFromName(klass, form.getName());
for(MConfig config : configs) {
Field configField;
try {
configField = klass.getDeclaredField(config.getName());
} catch (NoSuchFieldException e) {
throw new SqoopException(ModelError.MODEL_006,
"Missing field " + config.getName() + " on config class " + klass.getCanonicalName(), e);
}
Field formField = getFieldFromName(klass, config.getName());
// We need to access this field even if it would be declared as private
formField.setAccessible(true);
Class<?> formClass = formField.getType();
Object newValue = ClassUtils.instantiate(formClass);
configField.setAccessible(true);
Class configClass = configField.getType();
Object newValue = ClassUtils.instantiate(configClass);
if (newValue == null) {
throw new SqoopException(ModelError.MODEL_006,
"Can't instantiate new form " + formClass);
"Can't instantiate new config " + configClass);
}
for (MInput input : form.getInputs()) {
for(MInput input : config.getInputs()) {
String[] splitNames = input.getName().split("\\.");
if (splitNames.length != 2) {
throw new SqoopException(ModelError.MODEL_009, "Invalid name: "
@ -243,7 +248,7 @@ public static void fromForms(List<MForm> forms, Object configuration) {
// TODO(jarcec): Names structures fix, handle error cases
Field inputField;
try {
inputField = formClass.getDeclaredField(inputName);
inputField = configClass.getDeclaredField(inputName);
} catch (NoSuchFieldException e) {
throw new SqoopException(ModelError.MODEL_006, "Missing field "
+ input.getName(), e);
@ -271,30 +276,27 @@ public static void fromForms(List<MForm> forms, Object configuration) {
}
try {
formField.set(configuration, newValue);
configField.set(configuration, newValue);
} catch (IllegalAccessException e) {
throw new SqoopException(ModelError.MODEL_005, "Issue with field "
+ formField.getName(), e);
throw new SqoopException(ModelError.MODEL_005,
"Issue with field " + configField.getName(), e);
}
}
}
/**
* Apply validations on the forms.
* Apply validations on the configs.
*
* @param forms
* Forms that should be updated
* @param validation
* Validation that we should apply
* @param configs Configs that should be updated
* @param validation Validation that we should apply
*/
public static void applyValidation(List<MForm> forms, Validation validation) {
Map<Validation.FormInput, Validation.Message> messages = validation
.getMessages();
public static void applyValidation(List<MConfig> configs, ConfigValidator validation) {
Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = validation.getMessages();
for (MForm form : forms) {
applyValidation(form, messages);
for(MConfig config : configs) {
applyValidation(config, messages);
for (MInput input : form.getInputs()) {
for(MInput input : config.getInputs()) {
applyValidation(input, messages);
}
}
@ -308,12 +310,11 @@ public static void applyValidation(List<MForm> forms, Validation validation) {
* @param messages
* Map of all validation messages
*/
public static void applyValidation(MValidatedElement element,
Map<Validation.FormInput, Validation.Message> messages) {
Validation.FormInput name = new Validation.FormInput(element.getName());
public static void applyValidation(MValidatedElement element, Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages) {
ConfigValidator.ConfigInput name = new ConfigValidator.ConfigInput(element.getName());
if(messages.containsKey(name)) {
Validation.Message message = messages.get(name);
ConfigValidator.Message message = messages.get(name);
element.addValidationMessage(new Message(message.getStatus(), message.getMessage()));
} else {
element.addValidationMessage(new Message(Status.getDefault(), null));
@ -322,16 +323,16 @@ public static void applyValidation(MValidatedElement element,
/**
* Apply given validations on list of forms.
* Apply given validations on list of configs.
*
* @param forms
* @param configs
* @param result
*/
public static void applyValidation(List<MForm> forms, ValidationResult result) {
for(MForm form : forms) {
applyValidation(form, result);
public static void applyValidation(List<MConfig> configs, ConfigValidationResult result) {
for(MConfig config : configs) {
applyValidation(config, result);
for(MInput input : form.getInputs()) {
for(MInput input : config.getInputs()) {
applyValidation(input, result);
}
}
@ -345,7 +346,7 @@ public static void applyValidation(List<MForm> forms, ValidationResult result) {
* @param element
* @param result
*/
public static void applyValidation(MValidatedElement element, ValidationResult result) {
public static void applyValidation(MValidatedElement element, ConfigValidationResult result) {
List<Message> messages = result.getMessages().get(element.getName());
if(messages != null) {
@ -366,13 +367,13 @@ public static void applyValidation(MValidatedElement element, ValidationResult r
public static String toJson(Object configuration) {
Class klass = configuration.getClass();
Set<String> formNames = new HashSet<String>();
ConfigurationClass global = (ConfigurationClass) klass
.getAnnotation(ConfigurationClass.class);
ConfigurationClass configurationClass =
(ConfigurationClass)klass.getAnnotation(ConfigurationClass.class);
// Each configuration object must have this class annotation
if (global == null) {
if(configurationClass == null) {
throw new SqoopException(ModelError.MODEL_003,
"Missing annotation Configuration on class " + klass.getName());
"Missing annotation ConfigurationGroup on class " + klass.getName());
}
JSONObject jsonOutput = new JSONObject();
@ -381,8 +382,8 @@ public static String toJson(Object configuration) {
for (Field formField : klass.getDeclaredFields()) {
formField.setAccessible(true);
// We're processing only form validations
Form formAnnotation = formField.getAnnotation(Form.class);
// We're processing only config validations
Config formAnnotation = formField.getAnnotation(Config.class);
if(formAnnotation == null) {
continue;
}
@ -396,9 +397,9 @@ public static String toJson(Object configuration) {
+ formName, e);
}
JSONObject jsonForm = new JSONObject();
JSONObject jsonConfig = new JSONObject();
// Now process each input on the form
// Now process each input on the config
for(Field inputField : formField.getType().getDeclaredFields()) {
inputField.setAccessible(true);
String inputName = inputField.getName();
@ -425,26 +426,27 @@ public static String toJson(Object configuration) {
}
if(type == String.class) {
jsonForm.put(inputName, value);
jsonConfig.put(inputName, value);
} else if (type.isAssignableFrom(Map.class)) {
JSONObject map = new JSONObject();
for (Object key : ((Map) value).keySet()) {
map.put(key, ((Map) value).get(key));
}
jsonForm.put(inputName, map);
jsonConfig.put(inputName, map);
} else if(type == Integer.class) {
jsonForm.put(inputName, value);
jsonConfig.put(inputName, value);
} else if(type.isEnum()) {
jsonForm.put(inputName, value.toString());
jsonConfig.put(inputName, value.toString());
} else if(type == Boolean.class) {
jsonForm.put(inputName, value);
jsonConfig.put(inputName, value);
}else {
throw new SqoopException(ModelError.MODEL_004, "Unsupported type "
+ type.getName() + " for input " + formName + "." + inputName);
throw new SqoopException(ModelError.MODEL_004,
"Unsupported type " + type.getName() + " for input " + formName + "." + inputName);
}
}
}
jsonOutput.put(formName, jsonForm);
jsonOutput.put(formName, jsonConfig);
}
return jsonOutput.toJSONString();
}
@ -454,45 +456,46 @@ public static String toJson(Object configuration) {
* object.
*
* @param json JSON representation of the configuration object
* @param configuration Configuration object to be filled
* @param configuration ConfigurationGroup object to be filled
*/
public static void fillValues(String json, Object configuration) {
Class klass = configuration.getClass();
JSONObject jsonForms = (JSONObject) JSONValue.parse(json);
Set<String> formNames = new HashSet<String>();
JSONObject jsonConfigs = (JSONObject) JSONValue.parse(json);
for(Field formField : klass.getDeclaredFields()) {
formField.setAccessible(true);
for(Field configField : klass.getDeclaredFields()) {
configField.setAccessible(true);
String configName = configField.getName();
// We're processing only form validations
Form formAnnotation = formField.getAnnotation(Form.class);
// We're processing only config validations
Config formAnnotation = configField.getAnnotation(Config.class);
if(formAnnotation == null) {
continue;
}
String formName = getFormName(formField, formAnnotation, formNames);
String formName = getFormName(configField, formAnnotation, formNames);
try {
formField.set(configuration, formField.getType().newInstance());
configField.set(configuration, configField.getType().newInstance());
} catch (Exception e) {
throw new SqoopException(ModelError.MODEL_005,
"Issue with field " + formName, e);
"Issue with field " + configName, e);
}
JSONObject jsonInputs = (JSONObject) jsonForms.get(formName);
JSONObject jsonInputs = (JSONObject) jsonConfigs.get(configField.getName());
if(jsonInputs == null) {
continue;
}
Object formValue;
Object configValue;
try {
formValue = formField.get(configuration);
configValue = configField.get(configuration);
} catch (IllegalAccessException e) {
throw new SqoopException(ModelError.MODEL_005,
"Issue with field " + formName, e);
"Issue with field " + configName, e);
}
for(Field inputField : formField.getType().getDeclaredFields()) {
for(Field inputField : configField.getType().getDeclaredFields()) {
inputField.setAccessible(true);
String inputName = inputField.getName();
@ -500,10 +503,10 @@ public static void fillValues(String json, Object configuration) {
if(inputAnnotation == null || jsonInputs.get(inputName) == null) {
try {
inputField.set(formValue, null);
inputField.set(configValue, null);
} catch (IllegalAccessException e) {
throw new SqoopException(ModelError.MODEL_005,
"Issue with field " + formName + "." + inputName, e);
"Issue with field " + configName + "." + inputName, e);
}
continue;
}
@ -512,33 +515,33 @@ public static void fillValues(String json, Object configuration) {
try {
if(type == String.class) {
inputField.set(formValue, jsonInputs.get(inputName));
inputField.set(configValue, jsonInputs.get(inputName));
} else if (type.isAssignableFrom(Map.class)) {
Map<String, String> map = new HashMap<String, String>();
JSONObject jsonObject = (JSONObject) jsonInputs.get(inputName);
for(Object key : jsonObject.keySet()) {
map.put((String)key, (String)jsonObject.get(key));
}
inputField.set(formValue, map);
inputField.set(configValue, map);
} else if(type == Integer.class) {
inputField.set(formValue, ((Long)jsonInputs.get(inputName)).intValue());
inputField.set(configValue, ((Long)jsonInputs.get(inputName)).intValue());
} else if(type.isEnum()) {
inputField.set(formValue, Enum.valueOf((Class<? extends Enum>) inputField.getType(), (String) jsonInputs.get(inputName)));
inputField.set(configValue, Enum.valueOf((Class<? extends Enum>) inputField.getType(), (String) jsonInputs.get(inputName)));
} else if(type == Boolean.class) {
inputField.set(formValue, (Boolean) jsonInputs.get(inputName));
inputField.set(configValue, (Boolean) jsonInputs.get(inputName));
}else {
throw new SqoopException(ModelError.MODEL_004,
"Unsupported type " + type.getName() + " for input " + formName + "." + inputName);
"Unsupported type " + type.getName() + " for input " + configName + "." + inputName);
}
} catch (IllegalAccessException e) {
throw new SqoopException(ModelError.MODEL_005,
"Issue with field " + formName + "." + inputName, e);
"Issue with field " + configName + "." + inputName, e);
}
}
}
}
private static String getFormName(Field member, Form annotation, Set<String> existingFormNames) {
private static String getFormName(Field member, Config annotation, Set<String> existingFormNames) {
if (StringUtils.isEmpty(annotation.name())) {
return member.getName();
} else {
@ -578,35 +581,35 @@ public static String getName(Field input, Input annotation) {
return input.getName();
}
public static String getName(Field form, Form annotation) {
return form.getName();
public static String getName(Field config, Config annotation) {
return config.getName();
}
public static ConfigurationClass getConfigurationClassAnnotation(Object object, boolean strict) {
ConfigurationClass annotation = object.getClass().getAnnotation(ConfigurationClass.class);
if(strict && annotation == null) {
throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationClass on class " + object.getClass().getName());
throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationGroupClass on class " + object.getClass().getName());
}
return annotation;
}
public static FormClass getFormClassAnnotation(Object object, boolean strict) {
FormClass annotation = object.getClass().getAnnotation(FormClass.class);
public static ConfigClass getConfigClassAnnotation(Object object, boolean strict) {
ConfigClass annotation = object.getClass().getAnnotation(ConfigClass.class);
if(strict && annotation == null) {
throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationClass on class " + object.getClass().getName());
throw new SqoopException(ModelError.MODEL_003, "Missing annotation ConfigurationGroupClass on class " + object.getClass().getName());
}
return annotation;
}
public static Form getFormAnnotation(Field field, boolean strict) {
Form annotation = field.getAnnotation(Form.class);
public static Config getConfigAnnotation(Field field, boolean strict) {
Config annotation = field.getAnnotation(Config.class);
if(strict && annotation == null) {
throw new SqoopException(ModelError.MODEL_003, "Missing annotation Form on Field " + field.getName() + " on class " + field.getDeclaringClass().getName());
throw new SqoopException(ModelError.MODEL_003, "Missing annotation Config on Field " + field.getName() + " on class " + field.getDeclaringClass().getName());
}
return annotation;

7
common/src/main/java/org/apache/sqoop/model/ConfigurationClass.java
View File

@ -23,15 +23,16 @@
import java.lang.annotation.Target;
/**
* Class annotation. Each class that is used a configuration object where user
* is expected to provide input need to have this annotation.
* Class annotation to represent configuration for the connectors
* Each class that is used a configuration group object, the connector developer
* is expected to provide the inputs needed for this annotation
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface ConfigurationClass {
/**
* List of validators associated with this Configuration class.
* List of validators associated with this Configuration group class.
*
* @return
*/

16
common/src/main/java/org/apache/sqoop/model/MForm.java → common/src/main/java/org/apache/sqoop/model/MConfig.java
View File

@ -27,11 +27,11 @@
* input gathering process to be broken down into multiple steps that can be
* then paged through by the user interface.
*/
public final class MForm extends MValidatedElement implements MClonable {
public final class MConfig extends MValidatedElement implements MClonable {
private final List<MInput<?>> inputs;
public MForm(String name, List<MInput<?>> inputs) {
public MConfig(String name, List<MInput<?>> inputs) {
super(name);
this.inputs = inputs;
@ -73,7 +73,7 @@ public MMapInput getMapInput(String inputName) {
@Override
public String toString() {
StringBuilder sb = new StringBuilder("form-").append(getName());
StringBuilder sb = new StringBuilder("config-").append(getName());
sb.append(":").append(getPersistenceId()).append(":").append(inputs);
return sb.toString();
@ -85,11 +85,11 @@ public boolean equals(Object other) {
return true;
}
if (!(other instanceof MForm)) {
if (!(other instanceof MConfig)) {
return false;
}
MForm mf = (MForm) other;
MConfig mf = (MConfig) other;
return getName().equals(mf.getName())
&& inputs.equals(mf.inputs);
}
@ -106,12 +106,12 @@ public int hashCode() {
}
@Override
public MForm clone(boolean cloneWithValue) {
public MConfig clone(boolean cloneWithValue) {
List<MInput<?>> copyInputs = new ArrayList<MInput<?>>();
for(MInput<?> itr : this.getInputs()) {
copyInputs.add((MInput<?>)itr.clone(cloneWithValue));
}
MForm copyForm = new MForm(this.getName(), copyInputs);
return copyForm;
MConfig copyConfig = new MConfig(this.getName(), copyInputs);
return copyConfig;
}
}

62
common/src/main/java/org/apache/sqoop/model/MFormList.java → common/src/main/java/org/apache/sqoop/model/MConfigList.java
View File

@ -23,28 +23,28 @@
import java.util.List;
/**
* Arbitrary list of forms.
* Arbitrary list of config objects.
*/
public class MFormList implements MClonable {
public class MConfigList implements MClonable {
private final List<MForm> forms;
private final List<MConfig> configObjects;
public MFormList(List<MForm> forms) {
this.forms = forms;
public MConfigList(List<MConfig> configObjects) {
this.configObjects = configObjects;
}
public List<MForm> getForms() {
return forms;
public List<MConfig> getConfigs() {
return configObjects;
}
public MForm getForm(String formName) {
for(MForm form: forms) {
if(formName.equals(form.getName())) {
return form;
public MConfig getConfig(String configName) {
for(MConfig config: configObjects) {
if(configName.equals(config.getName())) {
return config;
}
}
throw new SqoopException(ModelError.MODEL_010, "Form name: " + formName);
throw new SqoopException(ModelError.MODEL_010, "config name: " + configName);
}
public MInput getInput(String name) {
@ -53,7 +53,7 @@ public MInput getInput(String name) {
throw new SqoopException(ModelError.MODEL_009, name);
}
return getForm(parts[0]).getInput(name);
return getConfig(parts[0]).getInput(name);
}
public MStringInput getStringInput(String name) {
@ -79,11 +79,11 @@ public MBooleanInput getBooleanInput(String name) {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof MFormList)) return false;
if (!(o instanceof MConfigList)) return false;
MFormList mFormList = (MFormList) o;
MConfigList mConfigList = (MConfigList) o;
if (!forms.equals(mFormList.forms)) return false;
if (!configObjects.equals(mConfigList.configObjects)) return false;
return true;
}
@ -91,8 +91,8 @@ public boolean equals(Object o) {
@Override
public int hashCode() {
int result = super.hashCode();
for(MForm form : forms) {
result = 31 * result + form.hashCode();
for(MConfig config : configObjects) {
result = 31 * result + config.hashCode();
}
return result;
@ -100,25 +100,25 @@ public int hashCode() {
@Override
public String toString() {
StringBuilder sb = new StringBuilder("Forms: ");
for(MForm form : forms) {
sb.append(form.toString());
StringBuilder sb = new StringBuilder("Configs: ");
for(MConfig config : configObjects) {
sb.append(config.toString());
}
return sb.toString();
}
@Override
public MFormList clone(boolean cloneWithValue) {
List<MForm> copyForms = null;
if(this.getForms() != null) {
copyForms = new ArrayList<MForm>();
for(MForm itr : this.getForms()) {
MForm newForm = itr.clone(cloneWithValue);
newForm.setPersistenceId(itr.getPersistenceId());
copyForms.add(newForm);
public MConfigList clone(boolean cloneWithValue) {
List<MConfig> copyConfigs = null;
if(this.getConfigs() != null) {
copyConfigs = new ArrayList<MConfig>();
for(MConfig itr : this.getConfigs()) {
MConfig newConfig = itr.clone(cloneWithValue);
newConfig.setPersistenceId(itr.getPersistenceId());
copyConfigs.add(newConfig);
}
}
MFormList copyFormList = new MFormList(copyForms);
return copyFormList;
MConfigList copyConfigList = new MConfigList(copyConfigs);
return copyConfigList;
}
}

12
common/src/main/java/org/apache/sqoop/model/MFormType.java → common/src/main/java/org/apache/sqoop/model/MConfigType.java
View File

@ -18,17 +18,17 @@
package org.apache.sqoop.model;
/**
* Represents the various form types supported by the system.
* Represents the various config types supported by the system.
*/
public enum MFormType {
public enum MConfigType {
/** Unknown form type */
/** Unknown config type */
OTHER,
/** Connection form type */
CONNECTION,
/** link config type */
LINK,
/** Job form type */
/** Job config type */
JOB;
}

110
common/src/main/java/org/apache/sqoop/model/MConnector.java
View File

@ -23,28 +23,27 @@
import org.apache.sqoop.common.SupportedDirections;
/**
* Connector metadata.
*
* Includes unique id that identifies connector in metadata store, unique human
* readable name, corresponding name and all forms for all supported job types.
* Connector entity supports the FROM/TO {@link Transferable} Includes unique id
* that identifies connector in the repository, unique human readable name,
* corresponding name and all configs to support the from and to data sources
*/
public final class MConnector extends MPersistableEntity implements MClonable {
private final String uniqueName;
private final String className;
private final MConnectionForms connectionForms;
private final MJobForms fromJobForms;
private final MJobForms toJobForms;
String version;
private final String version;
private final MLinkConfig linkConfig;
private final MFromConfig fromConfig;
private final MToConfig toConfig;
public MConnector(String uniqueName, String className,
String version, MConnectionForms connectionForms,
MJobForms fromJobForms, MJobForms toJobForms) {
public MConnector(String uniqueName, String className, String version, MLinkConfig linkConfig,
MFromConfig fromConfig, MToConfig toConfig) {
this.version = version;
this.connectionForms = connectionForms;
this.fromJobForms = fromJobForms;
this.toJobForms = toJobForms;
this.linkConfig = linkConfig;
this.fromConfig = fromConfig;
this.toConfig = toConfig;
// Why are we abusing NPE?
if (uniqueName == null || className == null) {
throw new NullPointerException();
}
@ -63,17 +62,15 @@ public String getClassName() {
@Override
public String toString() {
MJobForms fromJobForms = this.getJobForms(Direction.FROM);
MJobForms toJobForms = this.getJobForms(Direction.TO);
StringBuilder sb = new StringBuilder("connector-");
sb.append(uniqueName).append(":").append(getPersistenceId()).append(":");
sb.append(className);
sb.append(", ").append(getConnectionForms().toString());
if (fromJobForms != null) {
sb.append(", ").append(fromJobForms.toString());
sb.append(", ").append(getLinkConfig().toString());
if (getConfig(Direction.FROM) != null) {
sb.append(", ").append(getConfig(Direction.FROM).toString());
}
if (toJobForms != null) {
sb.append(", ").append(toJobForms.toString());
if (getConfig(Direction.TO) != null) {
sb.append(", ").append(getConfig(Direction.TO).toString());
}
return sb.toString();
}
@ -94,41 +91,39 @@ public boolean equals(Object other) {
if (supportedDirections.isDirectionSupported(Direction.FROM)
&& mcSupportedDirections.isDirectionSupported(Direction.FROM)
&& !getJobForms(Direction.FROM).equals(mc.getJobForms(Direction.FROM))) {
&& !getFromConfig().equals(mc.getFromConfig())) {
return false;
}
if (supportedDirections.isDirectionSupported(Direction.FROM)
!= mcSupportedDirections.isDirectionSupported(Direction.FROM)) {
if (supportedDirections.isDirectionSupported(Direction.FROM) != mcSupportedDirections
.isDirectionSupported(Direction.FROM)) {
return false;
}
if (supportedDirections.isDirectionSupported(Direction.TO)
&& mcSupportedDirections.isDirectionSupported(Direction.TO)
&& !getJobForms(Direction.TO).equals(mc.getJobForms(Direction.TO))) {
&& !getToConfig().equals(mc.getToConfig())) {
return false;
}
if (supportedDirections.isDirectionSupported(Direction.TO)
!= mcSupportedDirections.isDirectionSupported(Direction.TO)) {
if (supportedDirections.isDirectionSupported(Direction.TO) != mcSupportedDirections
.isDirectionSupported(Direction.TO)) {
return false;
}
return uniqueName.equals(mc.uniqueName)
&& className.equals(mc.className)
&& version.equals(mc.version)
&& connectionForms.equals(mc.getConnectionForms());
return uniqueName.equals(mc.uniqueName) && className.equals(mc.className)
&& version.equals(mc.version) && linkConfig.equals((mc.getLinkConfig()));
}
@Override
public int hashCode() {
SupportedDirections supportedDirections = getSupportedDirections();
int result = getConnectionForms().hashCode();
int result = getLinkConfig().hashCode();
if (supportedDirections.isDirectionSupported(Direction.FROM)) {
result = 31 * result + getJobForms(Direction.FROM).hashCode();
result = 31 * result + getFromConfig().hashCode();
}
if (supportedDirections.isDirectionSupported(Direction.TO)) {
result = 31 * result + getJobForms(Direction.TO).hashCode();
result = 31 * result + getToConfig().hashCode();
}
result = 31 * result + version.hashCode();
result = 31 * result + uniqueName.hashCode();
@ -140,55 +135,54 @@ public MConnector clone(boolean cloneWithValue) {
// Connector never have any values filled
cloneWithValue = false;
MJobForms fromJobForms = this.getJobForms(Direction.FROM);
MJobForms toJobForms = this.getJobForms(Direction.TO);
MFromConfig fromConfig = this.getFromConfig();
MToConfig toConfig = this.getToConfig();
if (fromJobForms != null) {
fromJobForms = fromJobForms.clone(cloneWithValue);
if (fromConfig != null) {
fromConfig = fromConfig.clone(cloneWithValue);
}
if (toJobForms != null) {
toJobForms = toJobForms.clone(cloneWithValue);
if (toConfig != null) {
toConfig = toConfig.clone(cloneWithValue);
}
MConnector copy = new MConnector(
this.getUniqueName(),
this.getClassName(),
this.getVersion(),
this.getConnectionForms().clone(cloneWithValue),
fromJobForms,
toJobForms);
MConnector copy = new MConnector(this.getUniqueName(), this.getClassName(), this.getVersion(),
this.getLinkConfig().clone(cloneWithValue), fromConfig, toConfig);
copy.setPersistenceId(this.getPersistenceId());
return copy;
}
public MConnectionForms getConnectionForms() {
return connectionForms;
public MLinkConfig getLinkConfig() {
return linkConfig;
}
public MJobForms getJobForms(Direction type) {
public MConfigList getConfig(Direction type) {
switch (type) {
case FROM:
return fromJobForms;
return fromConfig;
case TO:
return toJobForms;
return toConfig;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
public MFromConfig getFromConfig() {
return fromConfig;
}
public MToConfig getToConfig() {
return toConfig;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public SupportedDirections getSupportedDirections() {
return new SupportedDirections(this.getJobForms(Direction.FROM) != null,
this.getJobForms(Direction.TO) != null);
return new SupportedDirections(this.getConfig(Direction.FROM) != null,
this.getConfig(Direction.TO) != null);
}
}

82
common/src/main/java/org/apache/sqoop/model/MDriver.java Normal file
View File

@ -0,0 +1,82 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.model;
import java.sql.Driver;
/**
* Describes the configs associated with the {@link Driver} for executing sqoop jobs.
*/
public class MDriver extends MPersistableEntity implements MClonable {
private final MDriverConfig driverConfig;
private final String version;
public MDriver(MDriverConfig driverConfig, String version) {
this.driverConfig = driverConfig;
this.version = version;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("driver-");
sb.append(getPersistenceId()).append(":");
sb.append("version = " + version);
sb.append(", ").append(driverConfig.toString());
return sb.toString();
}
@Override
public boolean equals(Object other) {
if (other == this) {
return true;
}
if (!(other instanceof MDriver)) {
return false;
}
MDriver driver = (MDriver) other;
return version.equals(driver.getVersion()) &&
driverConfig.equals(driver.driverConfig);
}
@Override
public int hashCode() {
int result = driverConfig.hashCode();
result = 31 * result + version.hashCode();
return result;
}
public MDriverConfig getDriverConfig() {
return driverConfig;
}
@Override
public MDriver clone(boolean cloneWithValue) {
cloneWithValue = false;
MDriver copy = new MDriver(this.driverConfig.clone(cloneWithValue), this.version);
copy.setPersistenceId(this.getPersistenceId());
return copy;
}
public String getVersion() {
return version;
}
}

63
common/src/main/java/org/apache/sqoop/model/MDriverConfig.java
View File

@ -17,29 +17,23 @@
*/
package org.apache.sqoop.model;
import java.util.List;
/**
* Describes the configs associated with the {@link Driver} for executing sqoop jobs.
* Config describing all required information for the driver
* NOTE: It extends a config list since {@link MToConfig} could consist of a related config groups
* In future this could be simplified to hold a single list of all configs for the driver
*/
public class MDriverConfig extends MPersistableEntity implements MClonable {
private final MConnectionForms connectionForms;
private final MJobForms jobForms;
String version;
public MDriverConfig(MConnectionForms connectionForms, MJobForms jobForms, String version) {
this.connectionForms = connectionForms;
this.jobForms = jobForms;
this.version = version;
public class MDriverConfig extends MConfigList {
public MDriverConfig(List<MConfig> configs) {
super(configs);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("driver-");
sb.append(getPersistenceId()).append(":");
sb.append("version = " + version);
sb.append(", ").append(connectionForms.toString());
sb.append(jobForms.toString());
StringBuilder sb = new StringBuilder("Driver:");
sb.append(super.toString());
return sb.toString();
}
@ -53,45 +47,18 @@ public boolean equals(Object other) {
return false;
}
MDriverConfig mo = (MDriverConfig) other;
return version.equals(mo.getVersion()) &&
connectionForms.equals(mo.connectionForms) &&
jobForms.equals(mo.jobForms);
MDriverConfig mDriver = (MDriverConfig) other;
return super.equals(mDriver);
}
@Override
public int hashCode() {
int result = connectionForms.hashCode();
result = 31 * result + jobForms.hashCode();
result = 31 * result + version.hashCode();
return result;
}
public MConnectionForms getConnectionForms() {
return connectionForms;
}
public MJobForms getJobForms() {
return jobForms;
return super.hashCode();
}
@Override
public MDriverConfig clone(boolean cloneWithValue) {
//Framework never have any values filled
cloneWithValue = false;
MDriverConfig copy = new MDriverConfig(this.getConnectionForms().clone(cloneWithValue),
this.getJobForms().clone(cloneWithValue), this.version);
copy.setPersistenceId(this.getPersistenceId());
MDriverConfig copy = new MDriverConfig(super.clone(cloneWithValue).getConfigs());
return copy;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
}

64
common/src/main/java/org/apache/sqoop/model/MFromConfig.java Normal file
View File

@ -0,0 +1,64 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.model;
import java.util.List;
/**
* Config describing all required information to build the FROM part of the job
* NOTE: It extends a config list since {@link MFromConfig} could consist of a related config groups
* In future this could be simplified to hold a single list of all configs for the FROM object
*/
public class MFromConfig extends MConfigList {
public MFromConfig(List<MConfig> configs) {
super(configs);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("From: ");
sb.append(super.toString());
return sb.toString();
}
@Override
public boolean equals(Object other) {
if (other == this) {
return true;
}
if (!(other instanceof MFromConfig)) {
return false;
}
MFromConfig mj = (MFromConfig) other;
return super.equals(mj);
}
@Override
public int hashCode() {
return super.hashCode();
}
@Override
public MFromConfig clone(boolean cloneWithValue) {
MFromConfig copy = new MFromConfig(super.clone(cloneWithValue).getConfigs());
return copy;
}
}

102
common/src/main/java/org/apache/sqoop/model/MJob.java
View File

@ -22,29 +22,23 @@
import org.apache.sqoop.common.SqoopException;
/**
* Model describing entire job object including both connector and
* framework part.
* Model describing entire job object including the from/to and driver config information
* to execute the job
*/
public class MJob extends MAccountableEntity implements MClonable {
/**
* Connector reference.
*
* Job object do not immediately depend on connector as there is indirect
* NOTE : Job object do not immediately depend on connector as there is indirect
* dependency through link object, but having this dependency explicitly
* carried along helps a lot.
* carried along helps with not having to make the DB call everytime
*/
private final long fromConnectorId;
private final long toConnectorId;
/**
* Corresponding link objects for connector.
*/
private final long fromLinkId;
private final long toLinkId;
private final MJobForms fromConnectorPart;
private final MJobForms toConnectorPart;
private final MJobForms frameworkPart;
private final MFromConfig fromConfig;
private final MToConfig toConfig;
private final MDriverConfig driverConfig;
/**
* Default constructor to build new MJob model.
@ -53,24 +47,24 @@ public class MJob extends MAccountableEntity implements MClonable {
* @param toConnectorId TO Connector id
* @param fromLinkId FROM Link id
* @param toLinkId TO Link id
* @param fromPart FROM Connector forms
* @param toPart TO Connector forms
* @param frameworkPart Framework forms
* @param fromConfig FROM job config
* @param toConfig TO job config
* @param driverConfig driver config
*/
public MJob(long fromConnectorId,
long toConnectorId,
long fromConnectionId,
long toConnectionId,
MJobForms fromPart,
MJobForms toPart,
MJobForms frameworkPart) {
long fromLinkId,
long toLinkId,
MFromConfig fromConfig,
MToConfig toConfig,
MDriverConfig driverConfig) {
this.fromConnectorId = fromConnectorId;
this.toConnectorId = toConnectorId;
this.fromLinkId = fromConnectionId;
this.toLinkId = toConnectionId;
this.fromConnectorPart = fromPart;
this.toConnectorPart = toPart;
this.frameworkPart = frameworkPart;
this.fromLinkId = fromLinkId;
this.toLinkId = toLinkId;
this.fromConfig = fromConfig;
this.toConfig = toConfig;
this.driverConfig = driverConfig;
}
/**
@ -80,9 +74,9 @@ public MJob(long fromConnectorId,
*/
public MJob(MJob other) {
this(other,
other.getConnectorPart(Direction.FROM).clone(true),
other.getConnectorPart(Direction.TO).clone(true),
other.frameworkPart.clone(true));
other.getFromJobConfig().clone(true),
other.getToJobConfig().clone(true),
other.driverConfig.clone(true));
}
/**
@ -92,29 +86,29 @@ public MJob(MJob other) {
* used otherwise.
*
* @param other MJob model to copy
* @param fromPart FROM Connector forms
* @param toPart TO Connector forms
* @param frameworkPart Framework forms
* @param fromConfig FROM Job config
* @param toConfig TO Job config
* @param driverConfig driverConfig
*/
public MJob(MJob other, MJobForms fromPart, MJobForms toPart, MJobForms frameworkPart) {
public MJob(MJob other, MFromConfig fromConfig, MToConfig toConfig, MDriverConfig driverConfig) {
super(other);
this.fromConnectorId = other.getConnectorId(Direction.FROM);
this.toConnectorId = other.getConnectorId(Direction.TO);
this.fromLinkId = other.getLinkId(Direction.FROM);
this.toLinkId = other.getLinkId(Direction.TO);
this.fromConnectorPart = fromPart;
this.toConnectorPart = toPart;
this.frameworkPart = frameworkPart;
this.fromConfig = fromConfig;
this.toConfig = toConfig;
this.driverConfig = driverConfig;
this.setPersistenceId(other.getPersistenceId());
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("job");
sb.append(" connector-from-part: ").append(getConnectorPart(Direction.FROM));
sb.append(", connector-to-part: ").append(getConnectorPart(Direction.TO));
sb.append(", framework-part: ").append(frameworkPart);
sb.append("From job config: ").append(getJobConfig(Direction.FROM));
sb.append(", To job config: ").append(getJobConfig(Direction.TO));
sb.append(", Driver config: ").append(driverConfig);
return sb.toString();
}
@ -145,21 +139,29 @@ public long getConnectorId(Direction type) {
}
}
public MJobForms getConnectorPart(Direction type) {
public MConfigList getJobConfig(Direction type) {
switch(type) {
case FROM:
return fromConnectorPart;
return fromConfig;
case TO:
return toConnectorPart;
return toConfig;
default:
throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
}
}
public MJobForms getFrameworkPart() {
return frameworkPart;
public MFromConfig getFromJobConfig() {
return fromConfig;
}
public MToConfig getToJobConfig() {
return toConfig;
}
public MDriverConfig getDriverConfig() {
return driverConfig;
}
@Override
@ -172,9 +174,9 @@ public MJob clone(boolean cloneWithValue) {
getConnectorId(Direction.TO),
getLinkId(Direction.FROM),
getLinkId(Direction.TO),
getConnectorPart(Direction.FROM).clone(false),
getConnectorPart(Direction.TO).clone(false),
frameworkPart.clone(false));
getFromJobConfig().clone(false),
getToJobConfig().clone(false),
getDriverConfig().clone(false));
}
}
@ -194,8 +196,8 @@ public boolean equals(Object object) {
&& (job.getLinkId(Direction.FROM) == this.getLinkId(Direction.FROM))
&& (job.getLinkId(Direction.TO) == this.getLinkId(Direction.TO))
&& (job.getPersistenceId() == this.getPersistenceId())
&& (job.getConnectorPart(Direction.FROM).equals(this.getConnectorPart(Direction.FROM)))
&& (job.getConnectorPart(Direction.TO).equals(this.getConnectorPart(Direction.TO)))
&& (job.frameworkPart.equals(this.frameworkPart));
&& (job.getFromJobConfig().equals(this.getJobConfig(Direction.FROM)))
&& (job.getToJobConfig().equals(this.getJobConfig(Direction.TO)))
&& (job.getDriverConfig().equals(this.driverConfig));
}
}

70
common/src/main/java/org/apache/sqoop/model/MLink.java
View File

@ -22,57 +22,49 @@
*/
public class MLink extends MAccountableEntity implements MClonable {
private long connectorId;
private final MConnectionForms connectorPart;
private final MConnectionForms frameworkPart;
// NOTE: we hold this in the model for easy access to the link config object, it might as well be retrieved on the fly using the connectorId
private final MLinkConfig connectorLinkConfig;
/**
* Default constructor to build new MConnection model.
* Default constructor to build new MLink model.
*
* @param connectorId Connector id
* @param connectorPart Connector forms
* @param frameworkPart Framework forms
* @param linkConfig Connector forms
*/
public MLink(long connectorId,
MConnectionForms connectorPart,
MConnectionForms frameworkPart) {
public MLink(long connectorId, MLinkConfig linkConfig) {
this.connectorId = connectorId;
this.connectorPart = connectorPart;
this.frameworkPart = frameworkPart;
this.connectorLinkConfig = linkConfig;
}
/**
* Constructor to create deep copy of another MConnection model.
* Constructor to create deep copy of another MLink model.
*
* @param other MConnection model to copy
* @param other MLink model to copy
*/
public MLink(MLink other) {
this(other, other.connectorPart.clone(true), other.frameworkPart.clone(true));
this(other, other.connectorLinkConfig.clone(true));
}
/**
* Construct new MConnection model as a copy of another with replaced forms.
* Construct new MLink model as a copy of another with replaced forms.
*
* This method is suitable only for metadata upgrade path and should not be
* This method is suitable only for link upgrade path and should not be
* used otherwise.
*
* @param other MConnection model to copy
* @param connectorPart Connector forms
* @param frameworkPart Framework forms
* @param other MLink model to copy
* @param linkConfig link config
*/
public MLink(MLink other, MConnectionForms connectorPart, MConnectionForms frameworkPart) {
public MLink(MLink other, MLinkConfig linkConfig) {
super(other);
this.connectorId = other.connectorId;
this.connectorPart = connectorPart;
this.frameworkPart = frameworkPart;
this.connectorLinkConfig = linkConfig;
this.setPersistenceId(other.getPersistenceId());
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("connection: ").append(getName());
sb.append(" connector-part: ").append(connectorPart);
sb.append(", framework-part: ").append(frameworkPart);
StringBuilder sb = new StringBuilder("link: ").append(getName());
sb.append(" link-config: ").append(connectorLinkConfig);
return sb.toString();
}
@ -85,20 +77,11 @@ public void setConnectorId(long connectorId) {
this.connectorId = connectorId;
}
public MConnectionForms getConnectorPart() {
return connectorPart;
public MLinkConfig getConnectorLinkConfig() {
return connectorLinkConfig;
}
public MConnectionForms getFrameworkPart() {
return frameworkPart;
}
public MForm getConnectorForm(String formName) {
return connectorPart.getForm(formName);
}
public MForm getFrameworkForm(String formName) {
return frameworkPart.getForm(formName);
public MConfig getConnectorLinkConfig(String formName) {
return connectorLinkConfig.getConfig(formName);
}
@Override
@ -106,7 +89,7 @@ public MLink clone(boolean cloneWithValue) {
if(cloneWithValue) {
return new MLink(this);
} else {
return new MLink(connectorId, connectorPart.clone(false), frameworkPart.clone(false));
return new MLink(connectorId, connectorLinkConfig.clone(false));
}
}
@ -120,10 +103,9 @@ public boolean equals(Object object) {
return false;
}
MLink mc = (MLink)object;
return (mc.connectorId == this.connectorId)
&& (mc.getPersistenceId() == this.getPersistenceId())
&& (mc.connectorPart.equals(this.connectorPart))
&& (mc.frameworkPart.equals(this.frameworkPart));
MLink mLink = (MLink)object;
return (mLink.connectorId == this.connectorId)
&& (mLink.getPersistenceId() == this.getPersistenceId())
&& (mLink.connectorLinkConfig.equals(this.connectorLinkConfig));
}
}

18
common/src/main/java/org/apache/sqoop/model/MConnectionForms.java → common/src/main/java/org/apache/sqoop/model/MLinkConfig.java
View File

@ -20,19 +20,19 @@
import java.util.List;
/**
* Metadata describing all required information to build up an connection
* object for one part. Both connector and framework need to supply this object
* to build up entire connection.
* Config describing all required information to build up an link object
* NOTE: It extends a config list since {@link MLink} could consist of a related config groups
* In future this could be simplified to hold a single list of all configs for the link object
*/
public class MConnectionForms extends MFormList {
public class MLinkConfig extends MConfigList {
public MConnectionForms(List<MForm> forms) {
super(forms);
public MLinkConfig(List<MConfig> configs) {
super(configs);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("Connection: ");
StringBuilder sb = new StringBuilder("Link: ");
sb.append(super.toString());
return sb.toString();
}
@ -47,8 +47,8 @@ public boolean equals(Object other) {
}
@Override
public MConnectionForms clone(boolean cloneWithValue) {
MConnectionForms copy = new MConnectionForms(super.clone(cloneWithValue).getForms());
public MLinkConfig clone(boolean cloneWithValue) {
MLinkConfig copy = new MLinkConfig(super.clone(cloneWithValue).getConfigs());
return copy;
}
}

27
common/src/main/java/org/apache/sqoop/model/MJobForms.java → common/src/main/java/org/apache/sqoop/model/MToConfig.java
View File

@ -20,12 +20,21 @@
import java.util.List;
/**
* Metadata describing all required information to build a job
* object with two connectors and a framework.
* Config describing all required information to build the TO part of the job
* NOTE: It extends a config list since {@link MToConfig} could consist of a related config groups
* In future this could be simplified to hold a single list of all configs for the TO object
*/
public class MJobForms extends MFormList {
public MJobForms(List<MForm> forms) {
super(forms);
public class MToConfig extends MConfigList {
public MToConfig(List<MConfig> configs) {
super(configs);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("To: ");
sb.append(super.toString());
return sb.toString();
}
@Override
@ -34,11 +43,11 @@ public boolean equals(Object other) {
return true;
}
if (!(other instanceof MJobForms)) {
if (!(other instanceof MToConfig)) {
return false;
}
MJobForms mj = (MJobForms) other;
MToConfig mj = (MToConfig) other;
return super.equals(mj);
}
@ -48,8 +57,8 @@ public int hashCode() {
}
@Override
public MJobForms clone(boolean cloneWithValue) {
MJobForms copy = new MJobForms(super.clone(cloneWithValue).getForms());
public MToConfig clone(boolean cloneWithValue) {
MToConfig copy = new MToConfig(super.clone(cloneWithValue).getConfigs());
return copy;
}
}

6
common/src/main/java/org/apache/sqoop/model/ModelError.java
View File

@ -24,7 +24,7 @@
*/
public enum ModelError implements ErrorCode {
MODEL_001("Attempt to pass two different set of MForms for single job type."),
MODEL_001("Attempt to pass two different set of MConfigs for single job type."),
MODEL_002("Creating MJob of different job types"),
@ -34,7 +34,7 @@ public enum ModelError implements ErrorCode {
MODEL_005("Can't get field value"),
MODEL_006("Incompatible form list and configuration object"),
MODEL_006("Incompatible config list and configuration object"),
MODEL_007("Primitive types in configuration objects are not allowed"),
@ -42,7 +42,7 @@ public enum ModelError implements ErrorCode {
MODEL_009("Invalid input name"),
MODEL_010("Form do not exist"),
MODEL_010("Config do not exist"),
MODEL_011("Input do not exist"),

6
common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
View File

@ -17,14 +17,12 @@
*/
package org.apache.sqoop.utils;
import org.apache.log4j.Logger;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.log4j.Logger;
public final class ClassUtils {

6
common/src/main/java/org/apache/sqoop/validation/ValidationError.java → common/src/main/java/org/apache/sqoop/validation/ConfigValidationError.java
View File

@ -22,7 +22,7 @@
/**
*
*/
public enum ValidationError implements ErrorCode {
public enum ConfigValidationError implements ErrorCode {
VALIDATION_0000("Unknown error"),
@ -30,7 +30,7 @@ public enum ValidationError implements ErrorCode {
VALIDATION_0002("Usage of missing field"),
VALIDATION_0003("Invalid representation of form and input field"),
VALIDATION_0003("Invalid representation of config and input field"),
VALIDATION_0004("Can't find validator class"),
@ -38,7 +38,7 @@ public enum ValidationError implements ErrorCode {
private final String message;
private ValidationError(String message) {
private ConfigValidationError(String message) {
this.message = message;
}

8
common/src/main/java/org/apache/sqoop/validation/ValidationResult.java → common/src/main/java/org/apache/sqoop/validation/ConfigValidationResult.java
View File

@ -26,7 +26,7 @@
/**
* Result of validation execution.
*/
public class ValidationResult {
public class ConfigValidationResult {
/**
* All messages for each named item.
@ -38,7 +38,7 @@ public class ValidationResult {
*/
Status status;
public ValidationResult() {
public ConfigValidationResult() {
messages = new HashMap<String, List<Message>>();
status = Status.getDefault();
}
@ -49,7 +49,7 @@ public ValidationResult() {
* @param name Full name of the validated object
* @param validator Executed validator
*/
public void addValidator(String name, AbstractValidator validator) {
public void addValidatorResult(String name, AbstractValidator<String> validator) {
if(validator.getStatus() == Status.getDefault()) {
return;
}
@ -67,7 +67,7 @@ public void addValidator(String name, AbstractValidator validator) {
*
* @param result Other validation result
*/
public void merge(ValidationResult result) {
public void mergeValidatorResult(ConfigValidationResult result) {
messages.putAll(result.messages);
status = Status.getWorstStatus(status, result.status);
}

83
common/src/main/java/org/apache/sqoop/validation/ValidationRunner.java → common/src/main/java/org/apache/sqoop/validation/ConfigValidationRunner.java
View File

@ -19,9 +19,9 @@
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.FormUtils;
import org.apache.sqoop.model.Config;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.ConfigUtils;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.utils.ClassUtils;
@ -33,17 +33,18 @@
/**
* Validation runner that will run validators associated with given configuration
* class or form object.
* class or config object.
*
* Execution follows following rules:
* * Run children first (Inputs -> Form -> Class)
* * Run children first (Inputs -> Config -> Class)
* * If any children is not suitable (canProceed = false), skip running parent
*
* Which means that form validator don't have to repeat it's input validators as it will
* Which means that config validator don't have to repeat it's input validators as it will
* be never called if the input's are not valid. Similarly Class validators won't be called
* unless all forms will pass validators.
* unless all configs will pass validators.
*
*/
public class ValidationRunner {
public class ConfigValidationRunner {
/**
* Private cache of instantiated validators.
@ -54,7 +55,7 @@ public class ValidationRunner {
*/
private Map<Class<? extends AbstractValidator>, AbstractValidator> cache;
public ValidationRunner() {
public ConfigValidationRunner() {
cache = new HashMap<Class<? extends AbstractValidator>, AbstractValidator>();
}
@ -64,80 +65,80 @@ public ValidationRunner() {
* @param config Configuration instance
* @return
*/
public ValidationResult validate(Object config) {
ValidationResult result = new ValidationResult();
ConfigurationClass globalAnnotation = FormUtils.getConfigurationClassAnnotation(config, true);
public ConfigValidationResult validate(Object config) {
ConfigValidationResult result = new ConfigValidationResult();
ConfigurationClass globalAnnotation = ConfigUtils.getConfigurationClassAnnotation(config, true);
// Iterate over all declared form and call their validators
// Iterate over all declared config and call their validators
for (Field field : config.getClass().getDeclaredFields()) {
field.setAccessible(true);
Form formAnnotation = FormUtils.getFormAnnotation(field, false);
if(formAnnotation == null) {
Config configAnnotation = ConfigUtils.getConfigAnnotation(field, false);
if(configAnnotation == null) {
continue;
}
String formName = FormUtils.getName(field, formAnnotation);
ValidationResult r = validateForm(formName, FormUtils.getFieldValue(field, config));
result.merge(r);
String configName = ConfigUtils.getName(field, configAnnotation);
ConfigValidationResult r = validateConfig(configName, ConfigUtils.getFieldValue(field, config));
result.mergeValidatorResult(r);
}
// Call class validator only as long as we are in suitable state
if(result.getStatus().canProceed()) {
ValidationResult r = validateArray("", config, globalAnnotation.validators());
result.merge(r);
ConfigValidationResult r = validateArray("", config, globalAnnotation.validators());
result.mergeValidatorResult(r);
}
return result;
}
/**
* Validate given form instance.
* Validate given config instance.
*
* @param formName Form's name to build full name for all inputs.
* @param form Form instance
* @param configName Config's name to build full name for all inputs.
* @param config Config instance
* @return
*/
public ValidationResult validateForm(String formName, Object form) {
ValidationResult result = new ValidationResult();
FormClass formAnnotation = FormUtils.getFormClassAnnotation(form, true);
public ConfigValidationResult validateConfig(String configName, Object config) {
ConfigValidationResult result = new ConfigValidationResult();
ConfigClass configAnnotation = ConfigUtils.getConfigClassAnnotation(config, true);
// Iterate over all declared inputs and call their validators
for (Field field : form.getClass().getDeclaredFields()) {
Input inputAnnotation = FormUtils.getInputAnnotation(field, false);
for (Field field : config.getClass().getDeclaredFields()) {
Input inputAnnotation = ConfigUtils.getInputAnnotation(field, false);
if(inputAnnotation == null) {
continue;
}
String name = formName + "." + FormUtils.getName(field, inputAnnotation);
String name = configName + "." + ConfigUtils.getName(field, inputAnnotation);
ValidationResult r = validateArray(name, FormUtils.getFieldValue(field, form), inputAnnotation.validators());
result.merge(r);
ConfigValidationResult r = validateArray(name, ConfigUtils.getFieldValue(field, config), inputAnnotation.validators());
result.mergeValidatorResult(r);
}
// Call form validator only as long as we are in suitable state
// Call config validator only as long as we are in suitable state
if(result.getStatus().canProceed()) {
ValidationResult r = validateArray(formName, form, formAnnotation.validators());
result.merge(r);
ConfigValidationResult r = validateArray(configName, config, configAnnotation.validators());
result.mergeValidatorResult(r);
}
return result;
}
/**
* Execute array of validators on given object (can be input/form/class).
* Execute array of validators on given object (can be input/config/class).
*
* @param name Full name of the object
* @param object Input, Form or Class instance
* @param object Input, Config or Class instance
* @param validators Validators array
* @return
*/
private ValidationResult validateArray(String name, Object object, Validator[] validators) {
ValidationResult result = new ValidationResult();
private ConfigValidationResult validateArray(String name, Object object, Validator[] validators) {
ConfigValidationResult result = new ConfigValidationResult();
for (Validator validator : validators) {
AbstractValidator v = executeValidator(object, validator);
result.addValidator(name, v);
result.addValidatorResult(name, v);
}
return result;
@ -146,7 +147,7 @@ private ValidationResult validateArray(String name, Object object, Validator[] v
/**
* Execute single validator.
*
* @param object Input, Form or Class instance
* @param object Input, Config or Class instance
* @param validator Validator annotation
* @return
*/
@ -159,7 +160,7 @@ private AbstractValidator executeValidator(Object object, Validator validator) {
// This could happen if we would be missing some connector's jars on our classpath
if(instance == null) {
throw new SqoopException(ValidationError.VALIDATION_0004, validator.value().getName());
throw new SqoopException(ConfigValidationError.VALIDATION_0004, validator.value().getName());
}
cache.put(validator.value(), instance);

104
common/src/main/java/org/apache/sqoop/validation/Validation.java → common/src/main/java/org/apache/sqoop/validation/ConfigValidator.java
View File

@ -24,11 +24,11 @@
import java.util.Map;
/**
* Validation class.
* Config validators.
*
* This class represents validations to given configuration object.
* This class represents validations for the sqoop objects
*/
public class Validation {
public class ConfigValidator {
// Configuration class that belongs to this validation
Class klass;
@ -37,15 +37,15 @@ public class Validation {
Status status;
// Status messages for various fields
Map<FormInput, Message> messages;
Map<ConfigInput, Message> messages;
public Validation(Class klass) {
public ConfigValidator(Class klass) {
this.klass = klass;
status = Status.getDefault();
messages = new HashMap<FormInput, Message>();
messages = new HashMap<ConfigInput, Message>();
}
public Validation(Status status, Map<FormInput, Message> messages) {
public ConfigValidator(Status status, Map<ConfigInput, Message> messages) {
this.status = status;
this.messages = messages;
}
@ -54,68 +54,68 @@ public Status getStatus() {
return status;
}
public Map<FormInput, Message> getMessages() {
public Map<ConfigInput, Message> getMessages() {
return messages;
}
/**
* Add message to form.
* Add message to config.
*
* @param status Severity of the message
* @param form Form name, must be defined in the class
* @param config Config name, must be defined in the class
* @param message Validation message
*/
public void addMessage(Status status, String form, String message) {
addMessage(status, form, null, message);
public void addMessage(Status status, String config, String message) {
addMessage(status, config, null, message);
}
/**
* Add message to input in one of the forms.
* Add message to input in one of the configs.
*
* @param status Severity of the message
* @param form Form name, must be defined in the class
* @param input Field name, must be defined in the form class
* @param config Config name, must be defined in the class
* @param input Field name, must be defined in the config class
* @param message Validation message
*/
public void addMessage(Status status, String form, String input, String message ) {
public void addMessage(Status status, String config, String input, String message ) {
if( klass == null) {
throw new SqoopException(ValidationError.VALIDATION_0001);
throw new SqoopException(ConfigValidationError.VALIDATION_0001);
}
assert form != null;
assert config != null;
assert message != null;
// Field for specified form
Field formField;
// Field for specified config
Field configField;
// Load the form field and verify that it exists
// Load the config field and verify that it exists
try {
formField = klass.getDeclaredField(form);
configField = klass.getDeclaredField(config);
} catch (NoSuchFieldException e) {
throw new SqoopException(ValidationError.VALIDATION_0002,
"Can't get form " + form + " from " + klass.getName(), e);
throw new SqoopException(ConfigValidationError.VALIDATION_0002,
"Can't get config " + config + " from " + klass.getName(), e);
}
// If this is form message, just save the message and continue
// If this is config message, just save the message and continue
if(input == null) {
setMessage(status, form, input, message);
setMessage(status, config, input, message);
return;
}
// Verify that specified input exists on the form
// Verify that specified input exists on the config
try {
formField.getType().getDeclaredField(input);
configField.getType().getDeclaredField(input);
} catch (NoSuchFieldException e) {
throw new SqoopException(ValidationError.VALIDATION_0002,
"Can't get input " + input + " from form" + formField.getType().getName(), e);
throw new SqoopException(ConfigValidationError.VALIDATION_0002,
"Can't get input " + input + " from config" + configField.getType().getName(), e);
}
setMessage(status, form, input, message);
setMessage(status, config, input, message);
}
private void setMessage(Status status, String form, String input, String message) {
private void setMessage(Status status, String config, String input, String message) {
this.status = Status.getWorstStatus(this.status, status);
messages.put(new FormInput(form, input), new Message(status, message));
messages.put(new ConfigInput(config, input), new Message(status, message));
}
public static class Message {
@ -162,32 +162,32 @@ public String toString() {
}
}
public static class FormInput {
private String form;
public static class ConfigInput{
private String config;
private String input;
public FormInput(String form, String input) {
this.form = form;
public ConfigInput(String config, String input) {
this.config = config;
this.input = input;
}
public FormInput(String formInput) {
assert formInput != null;
String []parts = formInput.split("\\.");
public ConfigInput(String configInput) {
assert configInput != null;
String []parts = configInput.split("\\.");
if(formInput.isEmpty() || (parts.length != 1 && parts.length != 2)) {
throw new SqoopException(ValidationError.VALIDATION_0003,
"Specification " + formInput + " is not in valid format form.input");
if(configInput.isEmpty() || (parts.length != 1 && parts.length != 2)) {
throw new SqoopException(ConfigValidationError.VALIDATION_0003,
"Specification " + configInput + " is not in valid configat config.input");
}
this.form = parts[0];
this.config = parts[0];
if(parts.length == 2) {
this.input = parts[1];
}
}
public String getForm() {
return form;
public String getConfig() {
return config;
}
public String getInput() {
@ -199,11 +199,11 @@ public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
FormInput formInput = (FormInput) o;
ConfigInput configInput = (ConfigInput) o;
if (form != null ? !form.equals(formInput.form) : formInput.form != null)
if (config != null ? !config.equals(configInput.config) : configInput.config != null)
return false;
if (input != null ? !input.equals(formInput.input) : formInput.input != null)
if (input != null ? !input.equals(configInput.input) : configInput.input != null)
return false;
return true;
@ -211,7 +211,7 @@ public boolean equals(Object o) {
@Override
public int hashCode() {
int result = form != null ? form.hashCode() : 0;
int result = config != null ? config.hashCode() : 0;
result = 31 * result + (input != null ? input.hashCode() : 0);
return result;
}
@ -219,10 +219,10 @@ public int hashCode() {
@Override
public String toString() {
if(input == null) {
return form;
return config;
}
return form + "." + input;
return config + "." + input;
}
}
}

2
common/src/main/java/org/apache/sqoop/validation/Message.java
View File

@ -21,7 +21,7 @@
* Validation message.
*
* Validation message have always two parts - severity and textual information about what
* is wrong. It can be associated with Input, Form or Configuration class.
* is wrong. It can be associated with Input, Config or ConfigurationGroup class.
*/
public class Message {
private Status status;

148
common/src/test/java/org/apache/sqoop/json/TestUtil.java → common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java
View File

@ -17,64 +17,83 @@
*/
package org.apache.sqoop.json;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MJobForms;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.utils.MapResourceBundle;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriver;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.utils.MapResourceBundle;
/**
*
*/
public class TestUtil {
public class ConfigTestUtil {
public static MConnector getConnector(String name) {
return getConnector(name, true, true);
}
public static MConnector getConnector(String name, boolean from, boolean to) {
MJobForms fromJobForms = null;
MJobForms toJobForms = null;
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (from) {
fromJobForms = getJobForms();
fromConfig = getFromConfig();
}
if (to) {
toJobForms = getJobForms();
toConfig = getToConfig();
}
return new MConnector(name, name + ".class", "1.0-test",
getConnectionForms(), fromJobForms, toJobForms);
getLinkConfig(), fromConfig, toConfig);
}
public static MDriverConfig getDriverConfig() {
return new MDriverConfig(getConnectionForms(), getJobForms(), "1");
}
public static MLink getLink(String name) {
return new MLink(1, getConnector(name).getConnectionForms(), getDriverConfig()
.getConnectionForms());
return new MLink(1, getConnector(name).getLinkConfig());
}
public static MJob getJob(String name) {
return new MJob(1, 2, 1, 2, getConnector(name).getJobForms(Direction.FROM), getConnector(name)
.getJobForms(Direction.TO), getDriverConfig().getJobForms());
return new MJob(1, 2, 1, 2, getConnector(name).getFromConfig(), getConnector(name)
.getToConfig(), getDriverConfig());
}
public static MConnectionForms getConnectionForms() {
public static MDriverConfig getDriverConfig() {
List<MInput<?>> inputs;
MIntegerInput input;
MConfig config;
List<MConfig> driverConfigs = new ArrayList<MConfig>();
inputs = new ArrayList<MInput<?>>();
input = new MIntegerInput("numExtractors", false);
input.setPersistenceId(1);
inputs.add(input);
input = new MIntegerInput("numLoaders", false);
input.setPersistenceId(2);
inputs.add(input);
config = new MConfig("driver", inputs);
config.setPersistenceId(10);
driverConfigs.add(config);
return new MDriverConfig(driverConfigs);
}
public static MLinkConfig getLinkConfig() {
List<MInput<?>> inputs;
MStringInput input;
MForm form;
List<MForm> connectionForms = new ArrayList<MForm>();
MConfig config;
List<MConfig> linkConfig = new ArrayList<MConfig>();
inputs = new ArrayList<MInput<?>>();
input = new MStringInput("url", false, (short) 10);
@ -91,18 +110,18 @@ public static MConnectionForms getConnectionForms() {
input.setValue("test");
inputs.add(input);
form = new MForm("connection", inputs);
form.setPersistenceId(10);
connectionForms.add(form);
config = new MConfig("connection", inputs);
config.setPersistenceId(10);
linkConfig.add(config);
return new MConnectionForms(connectionForms);
return new MLinkConfig(linkConfig);
}
public static MJobForms getJobForms() {
static MFromConfig getFromConfig() {
List<MInput<?>> inputs;
MStringInput input;
MForm form;
List<MForm> jobForms = new ArrayList<MForm>();
MConfig config;
List<MConfig> jobConfigs = new ArrayList<MConfig>();
inputs = new ArrayList<MInput<?>>();
@ -118,9 +137,9 @@ public static MJobForms getJobForms() {
input.setPersistenceId(6);
inputs.add(input);
form = new MForm("Z", inputs);
form.setPersistenceId(11);
jobForms.add(form);
config = new MConfig("Z", inputs);
config.setPersistenceId(11);
jobConfigs.add(config);
inputs = new ArrayList<MInput<?>>();
@ -136,11 +155,56 @@ public static MJobForms getJobForms() {
input.setPersistenceId(9);
inputs.add(input);
form = new MForm("connection", inputs);
form.setPersistenceId(12);
jobForms.add(form);
config = new MConfig("from-table", inputs);
config.setPersistenceId(12);
jobConfigs.add(config);
return new MJobForms(jobForms);
return new MFromConfig(jobConfigs);
}
static MToConfig getToConfig() {
List<MInput<?>> inputs;
MStringInput input;
MConfig config;
List<MConfig> jobConfigs = new ArrayList<MConfig>();
inputs = new ArrayList<MInput<?>>();
input = new MStringInput("A", false, (short) 10);
input.setPersistenceId(4);
inputs.add(input);
input = new MStringInput("B", false, (short) 10);
input.setPersistenceId(5);
inputs.add(input);
input = new MStringInput("C", false, (short) 10);
input.setPersistenceId(6);
inputs.add(input);
config = new MConfig("Z", inputs);
config.setPersistenceId(11);
jobConfigs.add(config);
inputs = new ArrayList<MInput<?>>();
input = new MStringInput("D", false, (short) 10);
input.setPersistenceId(7);
inputs.add(input);
input = new MStringInput("E", false, (short) 10);
input.setPersistenceId(8);
inputs.add(input);
input = new MStringInput("F", false, (short) 10);
input.setPersistenceId(9);
inputs.add(input);
config = new MConfig("to-table", inputs);
config.setPersistenceId(12);
jobConfigs.add(config);
return new MToConfig(jobConfigs);
}
public static ResourceBundle getResourceBundle() {

37
common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
View File

@ -17,10 +17,9 @@
*/
package org.apache.sqoop.json;
import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
import static org.apache.sqoop.json.ConfigTestUtil.getConnector;
import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
import static org.junit.Assert.*;
import java.util.HashMap;
import java.util.LinkedList;
@ -28,8 +27,10 @@
import java.util.Map;
import java.util.ResourceBundle;
import static org.apache.sqoop.json.TestUtil.*;
import static org.junit.Assert.*;
import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
/**
*
@ -48,26 +49,26 @@ public void testSerialization() {
connectors.add(getConnector("mysql"));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, getResourceBundle());
bundles.put(2L, getResourceBundle());
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, getResourceBundle());
configBundles.put(2L, getResourceBundle());
// Serialize it to JSON object
ConnectorBean bean = new ConnectorBean(connectors, bundles);
JSONObject json = bean.extract(false);
ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles);
JSONObject connectorJSON = connectorBean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
String connectorJSONString = connectorJSON.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorBean retrievedBean = new ConnectorBean();
retrievedBean.restore(retrievedJson);
JSONObject parsedConnector = (JSONObject) JSONValue.parse(connectorJSONString);
ConnectorBean parsedConnectorBean = new ConnectorBean();
parsedConnectorBean.restore(parsedConnector);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0));
ResourceBundle retrievedBundle = retrievedBean.getResourceBundles().get(1L);
ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L);
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));

26
common/src/test/java/org/apache/sqoop/json/TestDriverConfigBean.java → common/src/test/java/org/apache/sqoop/json/TestDriverBean.java
View File

@ -17,31 +17,31 @@
*/
package org.apache.sqoop.json;
import org.apache.sqoop.model.MDriverConfig;
import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
import static org.junit.Assert.assertEquals;
import java.util.ResourceBundle;
import org.apache.sqoop.model.MDriver;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
import java.util.ResourceBundle;
import static org.apache.sqoop.json.TestUtil.*;
import static org.junit.Assert.*;
/**
*
*/
public class TestDriverConfigBean {
public class TestDriverBean {
/**
* Test that by JSON serialization followed by deserialization we will get
* equal framework object.
* equal drive config object.
*/
@Test
public void testSerialization() {
MDriverConfig driverConfig = getDriverConfig();
MDriver driver = new MDriver(ConfigTestUtil.getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION);
// Serialize it to JSON object
DriverConfigBean bean = new DriverConfigBean(driverConfig, getResourceBundle());
DriverBean bean = new DriverBean(driver, getResourceBundle());
JSONObject json = bean.extract(false);
// "Move" it across network in text form
@ -49,12 +49,12 @@ public void testSerialization() {
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
DriverConfigBean retrievedBean = new DriverConfigBean();
DriverBean retrievedBean = new DriverBean();
retrievedBean.restore(retrievedJson);
assertEquals(driverConfig, retrievedBean.getDriverConfig());
assertEquals(driver, retrievedBean.getDriver());
ResourceBundle retrievedBundle = retrievedBean.getResourceBundle();
ResourceBundle retrievedBundle = retrievedBean.getDriverConfigResourceBundle();
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
}

40
common/src/test/java/org/apache/sqoop/json/TestJobBean.java
View File

@ -17,6 +17,11 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getJob;
import static org.junit.Assert.assertEquals;
import java.util.Date;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput;
@ -25,11 +30,6 @@
import org.json.simple.parser.ParseException;
import org.junit.Test;
import java.util.Date;
import static org.apache.sqoop.json.TestUtil.getJob;
import static org.junit.Assert.assertEquals;
/**
*
*/
@ -46,25 +46,25 @@ public void testSerialization() throws ParseException {
job.setEnabled(false);
// Fill some data at the beginning
MStringInput input = (MStringInput) job.getConnectorPart(Direction.FROM)
.getForms().get(0).getInputs().get(0);
MStringInput input = (MStringInput) job.getJobConfig(Direction.FROM)
.getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there!");
input = (MStringInput) job.getConnectorPart(Direction.TO)
.getForms().get(0).getInputs().get(0);
input = (MStringInput) job.getJobConfig(Direction.TO)
.getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
JobBean bean = new JobBean(job);
JSONObject json = bean.extract(false);
JobBean jobBean = new JobBean(job);
JSONObject jobJson = jobBean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
String jobJsonString = jobJson.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject)JSONValue.parseWithException(string);
JobBean retrievedBean = new JobBean();
retrievedBean.restore(retrievedJson);
MJob target = retrievedBean.getJobs().get(0);
JSONObject parsedJobJson = (JSONObject)JSONValue.parseWithException(jobJsonString);
JobBean parsedJobBean = new JobBean();
parsedJobBean.restore(parsedJobJson);
MJob target = parsedJobBean.getJobs().get(0);
// Check id and name
assertEquals(666, target.getPersistenceId());
@ -78,11 +78,11 @@ public void testSerialization() throws ParseException {
assertEquals(false, target.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) target.getConnectorPart(Direction.FROM)
.getForms().get(0).getInputs().get(0);
MStringInput targetInput = (MStringInput) target.getJobConfig(Direction.FROM)
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) target.getConnectorPart(Direction.TO)
.getForms().get(0).getInputs().get(0);
targetInput = (MStringInput) target.getJobConfig(Direction.TO)
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}

56
common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
View File

@ -17,19 +17,21 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getLink;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.Date;
import org.apache.sqoop.json.util.ConfigSerialization;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONArray;
import org.json.simple.JSONValue;
import org.junit.Test;
import java.util.Date;
import static org.junit.Assert.*;
import static org.apache.sqoop.json.TestUtil.*;
/**
*
*/
@ -48,31 +50,31 @@ public void testSerialization() {
link.setEnabled(false);
// Fill some data at the beginning
MStringInput input = (MStringInput) link.getConnectorPart().getForms()
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
input.setValue("Hi there!");
// Serialize it to JSON object
LinkBean bean = new LinkBean(link);
JSONObject json = bean.extract(false);
LinkBean linkBean = new LinkBean(link);
JSONObject json = linkBean.extract(false);
// Check for sensitivity
JSONArray all = (JSONArray)json.get("all");
JSONArray all = (JSONArray)json.get(ConfigSerialization.ALL);
JSONObject allItem = (JSONObject)all.get(0);
JSONArray connectors = (JSONArray)allItem.get("connector");
JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG);
JSONObject connector = (JSONObject)connectors.get(0);
JSONArray inputs = (JSONArray)connector.get("inputs");
JSONArray inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS);
for (Object input1 : inputs) {
assertTrue(((JSONObject)input1).containsKey("sensitive"));
assertTrue(((JSONObject)input1).containsKey(ConfigSerialization.CONFIG_INPUT_SENSITIVE));
}
// "Move" it across network in text form
String string = json.toJSONString();
String linkJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString);
LinkBean retrievedBean = new LinkBean();
retrievedBean.restore(retrievedJson);
retrievedBean.restore(parsedLinkJson);
MLink target = retrievedBean.getLinks().get(0);
// Check id and name
@ -85,8 +87,8 @@ public void testSerialization() {
assertEquals(false, target.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) target.getConnectorPart()
.getForms().get(0).getInputs().get(0);
MStringInput targetInput = (MStringInput) target.getConnectorLinkConfig()
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
}
@ -104,7 +106,7 @@ public void testSensitivityFilter() {
link.setEnabled(true);
// Fill some data at the beginning
MStringInput input = (MStringInput) link.getConnectorPart().getForms()
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
input.setValue("Hi there!");
@ -114,25 +116,25 @@ public void testSensitivityFilter() {
JSONObject jsonFiltered = bean.extract(true);
// Sensitive values should exist
JSONArray all = (JSONArray)json.get("all");
JSONArray all = (JSONArray)json.get(ConfigSerialization.ALL);
JSONObject allItem = (JSONObject)all.get(0);
JSONArray connectors = (JSONArray)allItem.get("connector");
JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG);
JSONObject connector = (JSONObject)connectors.get(0);
JSONArray inputs = (JSONArray)connector.get("inputs");
JSONArray inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS);
assertEquals(3, inputs.size());
// Inputs are ordered when creating link
JSONObject password = (JSONObject)inputs.get(2);
assertTrue(password.containsKey("value"));
assertTrue(password.containsKey(ConfigSerialization.CONFIG_INPUT_VALUE));
// Sensitive values should not exist
all = (JSONArray)jsonFiltered.get("all");
all = (JSONArray)jsonFiltered.get(ConfigSerialization.ALL);
allItem = (JSONObject)all.get(0);
connectors = (JSONArray)allItem.get("connector");
connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG);
connector = (JSONObject)connectors.get(0);
inputs = (JSONArray)connector.get("inputs");
inputs = (JSONArray)connector.get(ConfigSerialization.CONFIG_INPUTS);
assertEquals(3, inputs.size());
// Inputs are ordered when creating link
password = (JSONObject)inputs.get(2);
assertFalse(password.containsKey("value"));
assertFalse(password.containsKey(ConfigSerialization.CONFIG_INPUT_VALUE));
}
}

85
common/src/test/java/org/apache/sqoop/json/TestValidationBean.java
View File

@ -17,17 +17,19 @@
*/
package org.apache.sqoop.json;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.*;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Status;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
/**
*
@ -44,7 +46,7 @@ public void testJobValidationBeanSerialization() {
);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
// "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@ -54,43 +56,43 @@ public void testJobValidationBeanSerialization() {
assertNull(retrievedBean.getId());
Validation.FormInput fa = new Validation.FormInput("f", "i");
Validation.FormInput fb = new Validation.FormInput("f2", "i2");
ConfigValidator.ConfigInput fa = new ConfigValidator.ConfigInput("c", "i");
ConfigValidator.ConfigInput fb = new ConfigValidator.ConfigInput("c2", "i2");
Validation fromConnector = retrievedBean.getConnectorValidation(Direction.FROM);
ConfigValidator fromConnector = retrievedBean.getConnectorValidation(Direction.FROM);
assertEquals(Status.FINE, fromConnector.getStatus());
assertEquals(2, fromConnector.getMessages().size());
assertTrue(fromConnector.getMessages().containsKey(fa));
assertEquals(new Validation.Message(Status.FINE, "d"),
assertEquals(new ConfigValidator.Message(Status.FINE, "d"),
fromConnector.getMessages().get(fa));
Validation toConnector = retrievedBean.getConnectorValidation(Direction.TO);
ConfigValidator toConnector = retrievedBean.getConnectorValidation(Direction.TO);
assertEquals(Status.FINE, toConnector.getStatus());
assertEquals(2, toConnector.getMessages().size());
assertTrue(toConnector.getMessages().containsKey(fa));
assertEquals(new Validation.Message(Status.FINE, "d"),
assertEquals(new ConfigValidator.Message(Status.FINE, "d"),
toConnector.getMessages().get(fa));
Validation framework = retrievedBean.getFrameworkValidation();
ConfigValidator framework = retrievedBean.getFrameworkValidation();
assertEquals(Status.UNACCEPTABLE, framework.getStatus());
assertEquals(2, framework.getMessages().size());
assertTrue(framework.getMessages().containsKey(fb));
assertEquals(new Validation.Message(Status.UNACCEPTABLE, "c"),
assertEquals(new ConfigValidator.Message(Status.UNACCEPTABLE, "c"),
framework.getMessages().get(fb));
}
@Test
public void testJobValidationBeanId() {
// Serialize it to JSON object
JobValidationBean bean = new JobValidationBean(
JobValidationBean jobValidatioBean = new JobValidationBean(
getValidation(Status.FINE),
getValidation(Status.FINE),
getValidation(Status.FINE)
);
bean.setId((long) 10);
JSONObject json = bean.extract(false);
jobValidatioBean.setId((long) 10);
JSONObject json = jobValidatioBean.extract(false);
// "Move" it across network in text form
// "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@ -105,12 +107,10 @@ public void testJobValidationBeanId() {
public void testLinkValidationBeanSerialization() {
// Serialize it to JSON object
LinkValidationBean bean = new LinkValidationBean(
getValidation(Status.FINE),
getValidation(Status.UNACCEPTABLE)
);
getValidation(Status.FINE));
JSONObject json = bean.extract(false);
// "Move" it across network in text form
// "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@ -120,35 +120,25 @@ public void testLinkValidationBeanSerialization() {
assertNull(retrievedBean.getId());
Validation.FormInput fa = new Validation.FormInput("f", "i");
Validation.FormInput fb = new Validation.FormInput("f2", "i2");
Validation connector = retrievedBean.getConnectorValidation();
ConfigValidator.ConfigInput ca = new ConfigValidator.ConfigInput("c", "i");
ConfigValidator connector = retrievedBean.getLinkConfigValidator();
assertEquals(Status.FINE, connector.getStatus());
assertEquals(2, connector.getMessages().size());
assertTrue(connector.getMessages().containsKey(fa));
assertEquals(new Validation.Message(Status.FINE, "d"),
connector.getMessages().get(fa));
Validation framework = retrievedBean.getFrameworkValidation();
assertEquals(Status.UNACCEPTABLE, framework.getStatus());
assertEquals(2, framework.getMessages().size());
assertTrue(framework.getMessages().containsKey(fb));
assertEquals(new Validation.Message(Status.UNACCEPTABLE, "c"),
framework.getMessages().get(fb));
assertTrue(connector.getMessages().containsKey(ca));
assertEquals(new ConfigValidator.Message(Status.FINE, "d"),
connector.getMessages().get(ca));
}
@Test
public void testLinkValidationBeanId() {
// Serialize it to JSON object
LinkValidationBean bean = new LinkValidationBean(
getValidation(Status.FINE),
getValidation(Status.FINE)
);
bean.setId((long) 10);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
// "Move" it across network in text config
String string = json.toJSONString();
// Retrieved transferred object
@ -159,17 +149,12 @@ public void testLinkValidationBeanId() {
assertEquals((Long)(long) 10, retrievedBean.getId());
}
public Validation getValidation(Status status) {
Map<Validation.FormInput, Validation.Message> messages =
new HashMap<Validation.FormInput, Validation.Message>();
public ConfigValidator getValidation(Status status) {
Map<ConfigValidator.ConfigInput, ConfigValidator.Message> messages = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
messages.put(
new Validation.FormInput("f", "i"),
new Validation.Message(status, "d"));
messages.put(
new Validation.FormInput("f2", "i2"),
new Validation.Message(status, "c"));
messages.put(new ConfigValidator.ConfigInput("c", "i"), new ConfigValidator.Message(status, "d"));
messages.put(new ConfigValidator.ConfigInput("c2", "i2"), new ConfigValidator.Message(status, "c"));
return new Validation(status, messages);
return new ConfigValidator(status, messages);
}
}

24
common/src/test/java/org/apache/sqoop/json/TestValidationResultBean.java
View File

@ -19,7 +19,7 @@
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.ValidationResult;
import org.apache.sqoop.validation.ConfigValidationResult;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
@ -39,31 +39,31 @@ public class TestValidationResultBean {
@Test
public void testEmptyTransfer() {
ValidationResult []empty = new ValidationResult[0];
ConfigValidationResult []empty = new ConfigValidationResult[0];
ValidationResult []retrieved = transfer(empty);
ConfigValidationResult []retrieved = transfer(empty);
assertEquals(0, retrieved.length);
}
@Test
public void testOneMessage() {
ValidationResult []empty = new ValidationResult[] {
ConfigValidationResult []empty = new ConfigValidationResult[] {
getResultA()
};
ValidationResult []retrieved = transfer(empty);
ConfigValidationResult []retrieved = transfer(empty);
assertEquals(1, retrieved.length);
verifyResultA(retrieved[0]);
}
@Test
public void testTwoMessages() {
ValidationResult []empty = new ValidationResult[] {
ConfigValidationResult []empty = new ConfigValidationResult[] {
getResultA(),
getResultA()
};
ValidationResult []retrieved = transfer(empty);
ConfigValidationResult []retrieved = transfer(empty);
assertEquals(2, retrieved.length);
verifyResultA(retrieved[0]);
@ -79,7 +79,7 @@ public void testId() {
assertNull(idNull);
}
public void verifyResultA(ValidationResult result) {
public void verifyResultA(ConfigValidationResult result) {
assertNotNull(result);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
@ -98,8 +98,8 @@ public void verifyResultA(ValidationResult result) {
assertEquals("B", messagesA.get(1).getMessage());
}
public ValidationResult getResultA() {
ValidationResult result = new ValidationResult();
public ConfigValidationResult getResultA() {
ConfigValidationResult result = new ConfigValidationResult();
List<Message> messages = new LinkedList<Message>();
messages.add(new Message(Status.ACCEPTABLE, "A"));
messages.add(new Message(Status.UNACCEPTABLE, "B"));
@ -109,7 +109,7 @@ public ValidationResult getResultA() {
private Long transfer(Long id) {
ValidationResultBean bean = new ValidationResultBean(new ValidationResult[0]);
ValidationResultBean bean = new ValidationResultBean(new ConfigValidationResult[0]);
bean.setId(id);
JSONObject json = bean.extract(false);
@ -122,7 +122,7 @@ private Long transfer(Long id) {
return retrievedBean.getId();
}
private ValidationResult[] transfer(ValidationResult [] results) {
private ConfigValidationResult[] transfer(ConfigValidationResult [] results) {
ValidationResultBean bean = new ValidationResultBean(results);
JSONObject json = bean.extract(false);

48
common/src/test/java/org/apache/sqoop/json/util/TestFormSerialization.java → common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
View File

@ -20,7 +20,7 @@
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.MBooleanInput;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MMapInput;
@ -40,7 +40,7 @@
/**
*
*/
public class TestFormSerialization {
public class TestConfigSerialization {
@Test
public void testAllDataTypes() {
@ -48,16 +48,16 @@ public void testAllDataTypes() {
Map<String, String> map = new HashMap<String, String>();
map.put("A", "B");
// Fill form with all values
MForm form = getForm();
form.getStringInput("String").setValue("A");
form.getMapInput("Map").setValue(map);
form.getIntegerInput("Integer").setValue(1);
form.getBooleanInput("Boolean").setValue(true);
form.getEnumInput("Enum").setValue("YES");
// Fill config with all values
MConfig config = getConfig();
config.getStringInput("String").setValue("A");
config.getMapInput("Map").setValue(map);
config.getIntegerInput("Integer").setValue(1);
config.getBooleanInput("Boolean").setValue(true);
config.getEnumInput("Enum").setValue("YES");
// Serialize that into JSON
JSONObject jsonObject = FormSerialization.extractForm(form, false);
JSONObject jsonObject = ConfigSerialization.extractConfig(config, false);
assertNotNull(jsonObject);
// Exchange the data on string level
@ -65,7 +65,7 @@ public void testAllDataTypes() {
JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson);
// And retrieve back from JSON representation
MForm retrieved = FormSerialization.restoreForm(retrievedJson);
MConfig retrieved = ConfigSerialization.restoreConfig(retrievedJson);
// Verify all expected values
assertEquals("A", retrieved.getStringInput("String").getValue());
@ -77,44 +77,44 @@ public void testAllDataTypes() {
@Test
public void testMapDataType() {
MForm form = getMapForm();
MConfig config = getMapConfig();
// Inserted values
Map<String, String> map = new HashMap<String, String>();
map.put("A", "B");
form.getMapInput("Map").setValue(map);
config.getMapInput("Map").setValue(map);
// Serialize
JSONObject jsonObject = FormSerialization.extractForm(form, false);
JSONObject jsonObject = ConfigSerialization.extractConfig(config, false);
String serializedJson = jsonObject.toJSONString();
// Deserialize
JSONObject retrievedJson = (JSONObject) JSONValue.parse(serializedJson);
MForm retrieved = FormSerialization.restoreForm(retrievedJson);
MConfig retrieved = ConfigSerialization.restoreConfig(retrievedJson);
assertEquals(map, retrieved.getMapInput("Map").getValue());
}
@Test(expected=SqoopException.class)
public void testMapDataTypeException() {
MForm form = getMapForm();
MConfig config = getMapConfig();
// Inserted values
Map<String, String> map = new HashMap<String, String>();
map.put("A", "B");
form.getMapInput("Map").setValue(map);
config.getMapInput("Map").setValue(map);
// Serialize
JSONObject jsonObject = FormSerialization.extractForm(form, false);
JSONObject jsonObject = ConfigSerialization.extractConfig(config, false);
String serializedJson = jsonObject.toJSONString();
// Replace map value with a fake string to force exception
String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", "\"nonsensical string\"");
System.out.println(badSerializedJson);
JSONObject retrievedJson = (JSONObject) JSONValue.parse(badSerializedJson);
FormSerialization.restoreForm(retrievedJson);
ConfigSerialization.restoreConfig(retrievedJson);
}
protected MForm getMapForm() {
protected MConfig getMapConfig() {
List<MInput<?>> inputs;
MInput input;
@ -123,15 +123,15 @@ protected MForm getMapForm() {
input = new MMapInput("Map", false);
inputs.add(input);
return new MForm("f", inputs);
return new MConfig("c", inputs);
}
/**
* Return form with all data types.
* Return config with all data types.
*
* @return
*/
protected MForm getForm() {
protected MConfig getConfig() {
List<MInput<?>> inputs;
MInput input;
@ -152,6 +152,6 @@ protected MForm getForm() {
input = new MEnumInput("Enum", false, new String[] {"YES", "NO"});
inputs.add(input);
return new MForm("f", inputs);
return new MConfig("c", inputs);
}
}

290
common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java Normal file
View File

@ -0,0 +1,290 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.model;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import junit.framework.TestCase;
import org.apache.sqoop.common.SqoopException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
* Test config utils
*/
public class TestConfigUtils extends TestCase {
public void testConfigs() {
TestConfiguration config = new TestConfiguration();
config.aConfig.a1 = "value";
List<MConfig> configsByInstance = ConfigUtils.toConfigs(config);
assertEquals(getConfigs(), configsByInstance);
assertEquals("value", configsByInstance.get(0).getInputs().get(0).getValue());
List<MConfig> configsByClass = ConfigUtils.toConfigs(TestConfiguration.class);
assertEquals(getConfigs(), configsByClass);
List<MConfig> configsByBoth = ConfigUtils.toConfigs(TestConfiguration.class, config);
assertEquals(getConfigs(), configsByBoth);
assertEquals("value", configsByBoth.get(0).getInputs().get(0).getValue());
}
public void testConfigsMissingAnnotation() {
try {
ConfigUtils.toConfigs(ConfigWithout.class);
} catch(SqoopException ex) {
assertEquals(ModelError.MODEL_003, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testNonUniqueFormNameAttributes() {
try {
ConfigUtils.toConfigs(ConfigurationWithNonUniqueFormNameAttribute.class);
} catch (SqoopException ex) {
assertEquals(ModelError.MODEL_012, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testInvalidFormNameAttribute() {
try {
ConfigUtils.toConfigs(ConfigurationWithInvalidFormNameAttribute.class);
} catch (SqoopException ex) {
assertEquals(ModelError.MODEL_013, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testInvalidFormNameAttributeLength() {
try {
ConfigUtils.toConfigs(ConfigurationWithInvalidFormNameAttributeLength.class);
} catch (SqoopException ex) {
assertEquals(ModelError.MODEL_014, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testFailureOnPrimitiveType() {
PrimitiveConfig config = new PrimitiveConfig();
try {
ConfigUtils.toConfigs(config);
fail("We were expecting exception for unsupported type.");
} catch(SqoopException ex) {
assertEquals(ModelError.MODEL_007, ex.getErrorCode());
}
}
public void testFillValues() {
List<MConfig> configs = getConfigs();
((MStringInput)configs.get(0).getInputs().get(0)).setValue("value");
TestConfiguration config = new TestConfiguration();
ConfigUtils.fromConfigs(configs, config);
assertEquals("value", config.aConfig.a1);
}
public void testFillValuesObjectReuse() {
List<MConfig> configs = getConfigs();
((MStringInput)configs.get(0).getInputs().get(0)).setValue("value");
TestConfiguration config = new TestConfiguration();
config.aConfig.a2 = "x";
config.bConfig.b1 = "y";
ConfigUtils.fromConfigs(configs, config);
assertEquals("value", config.aConfig.a1);
assertNull(config.aConfig.a2);
assertNull(config.bConfig.b2);
assertNull(config.bConfig.b2);
}
public void testJson() {
TestConfiguration config = new TestConfiguration();
config.aConfig.a1 = "A";
config.bConfig.b2 = "B";
config.cConfig.intValue = 4;
config.cConfig.map.put("C", "D");
config.cConfig.enumeration = Enumeration.X;
String json = ConfigUtils.toJson(config);
TestConfiguration targetConfig = new TestConfiguration();
// Old values from should be always removed
targetConfig.aConfig.a2 = "X";
targetConfig.bConfig.b1 = "Y";
// Nulls in configs shouldn't be an issue either
targetConfig.cConfig = null;
ConfigUtils.fillValues(json, targetConfig);
assertEquals("A", targetConfig.aConfig.a1);
assertNull(targetConfig.aConfig.a2);
assertNull(targetConfig.bConfig.b1);
assertEquals("B", targetConfig.bConfig.b2);
assertEquals((Integer)4, targetConfig.cConfig.intValue);
assertEquals(1, targetConfig.cConfig.map.size());
assertTrue(targetConfig.cConfig.map.containsKey("C"));
assertEquals("D", targetConfig.cConfig.map.get("C"));
assertEquals(Enumeration.X, targetConfig.cConfig.enumeration);
}
/**
* Config structure that corresponds to Config class declared below
* @return Config structure
*/
protected List<MConfig> getConfigs() {
List<MConfig> ret = new LinkedList<MConfig>();
List<MInput<?>> inputs;
// Config A
inputs = new LinkedList<MInput<?>>();
inputs.add(new MStringInput("aConfig.a1", false, (short)30));
inputs.add(new MStringInput("aConfig.a2", true, (short)-1));
ret.add(new MConfig("aConfig", inputs));
// Config B
inputs = new LinkedList<MInput<?>>();
inputs.add(new MStringInput("bConfig.b1", false, (short)2));
inputs.add(new MStringInput("bConfig.b2", false, (short)3));
ret.add(new MConfig("bConfig", inputs));
// Config C
inputs = new LinkedList<MInput<?>>();
inputs.add(new MIntegerInput("cConfig.intValue", false));
inputs.add(new MMapInput("cConfig.map", false));
inputs.add(new MEnumInput("cConfig.enumeration", false, new String[]{"X", "Y"}));
ret.add(new MConfig("cConfig", inputs));
return ret;
}
@ConfigurationClass
public static class ConfigurationWithNonUniqueFormNameAttribute {
public ConfigurationWithNonUniqueFormNameAttribute() {
aForm = new InvalidConfig();
bForm = new InvalidConfig();
}
@Config(name = "sameName")
InvalidConfig aForm;
@Config(name = "sameName")
InvalidConfig bForm;
}
@ConfigurationClass
public static class ConfigurationWithInvalidFormNameAttribute {
public ConfigurationWithInvalidFormNameAttribute() {
invalidForm = new InvalidConfig();
}
@Config(name = "#_form")
InvalidConfig invalidForm;
}
@ConfigurationClass
public static class ConfigurationWithInvalidFormNameAttributeLength {
public ConfigurationWithInvalidFormNameAttributeLength() {
invalidLengthForm = new InvalidConfig();
}
@Config(name = "longest_form_more_than_30_characers")
InvalidConfig invalidLengthForm;
}
@ConfigurationClass
public static class TestConfiguration {
public TestConfiguration() {
aConfig = new AConfig();
bConfig = new BConfig();
cConfig = new CConfig();
}
@Config AConfig aConfig;
@Config BConfig bConfig;
@Config CConfig cConfig;
}
@ConfigurationClass
public static class PrimitiveConfig {
@Config DConfig dConfig;
}
@ConfigClass
public static class AConfig {
@Input(size = 30) String a1;
@Input(sensitive = true) String a2;
}
@ConfigClass
public static class BConfig {
@Input(size = 2) String b1;
@Input(size = 3) String b2;
}
@ConfigClass
public static class CConfig {
@Input Integer intValue;
@Input Map<String, String> map;
@Input Enumeration enumeration;
public CConfig() {
map = new HashMap<String, String>();
}
}
@ConfigClass
public static class InvalidConfig {
}
@ConfigClass
public static class DConfig {
@Input int value;
}
public static class ConfigWithout {
}
enum Enumeration {
X,
Y,
}
}

295
common/src/test/java/org/apache/sqoop/model/TestFormUtils.java
View File

@ -1,295 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.model;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.junit.Test;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.*;
/**
* Test form utils
*/
public class TestFormUtils {
@Test
public void testToForms() {
Config config = new Config();
config.aForm.a1 = "value";
List<MForm> formsByInstance = FormUtils.toForms(config);
assertEquals(getForms(), formsByInstance);
assertEquals("value", formsByInstance.get(0).getInputs().get(0).getValue());
List<MForm> formsByClass = FormUtils.toForms(Config.class);
assertEquals(getForms(), formsByClass);
List<MForm> formsByBoth = FormUtils.toForms(Config.class, config);
assertEquals(getForms(), formsByBoth);
assertEquals("value", formsByBoth.get(0).getInputs().get(0).getValue());
}
@Test
public void testToFormsMissingAnnotation() {
try {
FormUtils.toForms(ConfigWithout.class);
} catch(SqoopException ex) {
assertEquals(ModelError.MODEL_003, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testNonUniqueFormNameAttributes() {
try {
FormUtils.toForms(ConfigurationWithNonUniqueFormNameAttribute.class);
} catch (SqoopException ex) {
assertEquals(ModelError.MODEL_012, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testInvalidFormNameAttribute() {
try {
FormUtils.toForms(ConfigurationWithInvalidFormNameAttribute.class);
} catch (SqoopException ex) {
assertEquals(ModelError.MODEL_013, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
public void testInvalidFormNameAttributeLength() {
try {
FormUtils.toForms(ConfigurationWithInvalidFormNameAttributeLength.class);
} catch (SqoopException ex) {
assertEquals(ModelError.MODEL_014, ex.getErrorCode());
return;
}
fail("Correct exception wasn't thrown");
}
@Test
public void testFailureOnPrimitiveType() {
PrimitiveConfig config = new PrimitiveConfig();
try {
FormUtils.toForms(config);
fail("We were expecting exception for unsupported type.");
} catch(SqoopException ex) {
assertEquals(ModelError.MODEL_007, ex.getErrorCode());
}
}
@Test
public void testFillValues() {
List<MForm> forms = getForms();
((MStringInput)forms.get(0).getInputs().get(0)).setValue("value");
Config config = new Config();
FormUtils.fromForms(forms, config);
assertEquals("value", config.aForm.a1);
}
@Test
public void testFillValuesObjectReuse() {
List<MForm> forms = getForms();
((MStringInput)forms.get(0).getInputs().get(0)).setValue("value");
Config config = new Config();
config.aForm.a2 = "x";
config.bForm.b1 = "y";
FormUtils.fromForms(forms, config);
assertEquals("value", config.aForm.a1);
assertNull(config.aForm.a2);
assertNull(config.bForm.b2);
assertNull(config.bForm.b2);
}
@Test
public void testJson() {
Config config = new Config();
config.aForm.a1 = "A";
config.bForm.b2 = "B";
config.cForm.intValue = 4;
config.cForm.map.put("C", "D");
config.cForm.enumeration = Enumeration.X;
String json = FormUtils.toJson(config);
Config targetConfig = new Config();
// Old values from should be always removed
targetConfig.aForm.a2 = "X";
targetConfig.bForm.b1 = "Y";
// Nulls in forms shouldn't be an issue either
targetConfig.cForm = null;
FormUtils.fillValues(json, targetConfig);
assertEquals("A", targetConfig.aForm.a1);
assertNull(targetConfig.aForm.a2);
assertNull(targetConfig.bForm.b1);
assertEquals("B", targetConfig.bForm.b2);
assertEquals((Integer)4, targetConfig.cForm.intValue);
assertEquals(1, targetConfig.cForm.map.size());
assertTrue(targetConfig.cForm.map.containsKey("C"));
assertEquals("D", targetConfig.cForm.map.get("C"));
assertEquals(Enumeration.X, targetConfig.cForm.enumeration);
}
/**
* Form structure that corresponds to Config class declared below
* @return Form structure
*/
protected List<MForm> getForms() {
List<MForm> ret = new LinkedList<MForm>();
List<MInput<?>> inputs;
// Form A
inputs = new LinkedList<MInput<?>>();
inputs.add(new MStringInput("aForm.a1", false, (short)30));
inputs.add(new MStringInput("aForm.a2", true, (short)-1));
ret.add(new MForm("aForm", inputs));
// Form B
inputs = new LinkedList<MInput<?>>();
inputs.add(new MStringInput("bForm.b1", false, (short)2));
inputs.add(new MStringInput("bForm.b2", false, (short)3));
ret.add(new MForm("bForm", inputs));
// Form C
inputs = new LinkedList<MInput<?>>();
inputs.add(new MIntegerInput("cForm.intValue", false));
inputs.add(new MMapInput("cForm.map", false));
inputs.add(new MEnumInput("cForm.enumeration", false, new String[]{"X", "Y"}));
ret.add(new MForm("cForm", inputs));
return ret;
}
@ConfigurationClass
public static class ConfigurationWithNonUniqueFormNameAttribute {
public ConfigurationWithNonUniqueFormNameAttribute() {
aForm = new InvalidForm();
bForm = new InvalidForm();
}
@Form(name = "sameName")
InvalidForm aForm;
@Form(name = "sameName")
InvalidForm bForm;
}
@ConfigurationClass
public static class ConfigurationWithInvalidFormNameAttribute {
public ConfigurationWithInvalidFormNameAttribute() {
invalidForm = new InvalidForm();
}
@Form(name = "#_form")
InvalidForm invalidForm;
}
@ConfigurationClass
public static class ConfigurationWithInvalidFormNameAttributeLength {
public ConfigurationWithInvalidFormNameAttributeLength() {
invalidLengthForm = new InvalidForm();
}
@Form(name = "longest_form_more_than_30_characers")
InvalidForm invalidLengthForm;
}
@ConfigurationClass
public static class Config {
public Config() {
aForm = new AForm();
bForm = new BForm();
cForm = new CForm();
}
@Form AForm aForm;
@Form BForm bForm;
@Form CForm cForm;
}
@ConfigurationClass
public static class PrimitiveConfig {
@Form DForm dForm;
}
@FormClass
public static class AForm {
@Input(size = 30) String a1;
@Input(sensitive = true) String a2;
}
@FormClass
public static class BForm {
@Input(size = 2) String b1;
@Input(size = 3) String b2;
}
@FormClass
public static class CForm {
@Input Integer intValue;
@Input Map<String, String> map;
@Input Enumeration enumeration;
public CForm() {
map = new HashMap<String, String>();
}
}
@FormClass
public static class InvalidForm {
}
@FormClass
public static class DForm {
@Input int value;
}
public static class ConfigWithout {
}
enum Enumeration {
X,
Y,
}
}

9
common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
View File

@ -35,14 +35,13 @@ public class TestMAccountableEntity {
*/
@Test
public void testInitialization() {
List<MForm> forms = new ArrayList<MForm>();
List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
MForm form = new MForm("FORMNAME", list);
forms.add(form);
MAccountableEntity link = new MLink(123l, new MConnectionForms(
forms), new MConnectionForms(forms));
MConfig config = new MConfig("CONFIGNAME", list);
configs.add(config);
MAccountableEntity link = new MLink(123l, new MLinkConfig(configs));
// Initially creation date and last update date is same
assertEquals(link.getCreationDate(), link.getLastUpdateDate());
Date testCreationDate = new Date();

33
common/src/test/java/org/apache/sqoop/model/TestMForm.java → common/src/test/java/org/apache/sqoop/model/TestMConfig.java
View File

@ -24,10 +24,7 @@
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.model.MForm
*/
public class TestMForm {
public class TestMConfig {
/**
* Test for initialization
@ -40,10 +37,10 @@ public void testInitialization() {
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input1);
list.add(input2);
MForm mform = new MForm("form", list);
MConfig mConfig = new MConfig("config", list);
assertEquals("form", mform.getName());
assertEquals(2, mform.getInputs().size());
assertEquals("config", mConfig.getName());
assertEquals(2, mConfig.getInputs().size());
}
/**
@ -56,23 +53,23 @@ public void testEquals() {
List<MInput<?>> list1 = new ArrayList<MInput<?>>();
list1.add(input1);
list1.add(input2);
MForm mform1 = new MForm("form", list1);
MConfig mform1 = new MConfig("config", list1);
MInput<Integer> input3 = new MIntegerInput("sqoopsqoop1", false);
MInput<Integer> input4 = new MIntegerInput("sqoopsqoop2", false);
List<MInput<?>> list2 = new ArrayList<MInput<?>>();
list2.add(input3);
list2.add(input4);
MForm mform2 = new MForm("form", list2);
MConfig mform2 = new MConfig("config", list2);
assertEquals(mform2, mform1);
}
@Test
public void testGetInputs() {
MIntegerInput intInput = new MIntegerInput("Form.A", false);
MMapInput mapInput = new MMapInput("Form.B", false);
MStringInput stringInput = new MStringInput("Form.C", false, (short)3);
MEnumInput enumInput = new MEnumInput("Form.D", false, new String[] {"I", "V"});
MIntegerInput intInput = new MIntegerInput("Config.A", false);
MMapInput mapInput = new MMapInput("Config.B", false);
MStringInput stringInput = new MStringInput("Config.C", false, (short)3);
MEnumInput enumInput = new MEnumInput("Config.D", false, new String[] {"I", "V"});
List<MInput<?>> inputs = new ArrayList<MInput<?>>();
inputs.add(intInput);
@ -80,10 +77,10 @@ public void testGetInputs() {
inputs.add(stringInput);
inputs.add(enumInput);
MForm form = new MForm("Form", inputs);
assertEquals(intInput, form.getIntegerInput("Form.A"));
assertEquals(mapInput, form.getMapInput("Form.B"));
assertEquals(stringInput, form.getStringInput("Form.C"));
assertEquals(enumInput, form.getEnumInput("Form.D"));
MConfig config = new MConfig("Config", inputs);
assertEquals(intInput, config.getIntegerInput("Config.A"));
assertEquals(mapInput, config.getMapInput("Config.B"));
assertEquals(stringInput, config.getStringInput("Config.C"));
assertEquals(enumInput, config.getEnumInput("Config.D"));
}
}

29
common/src/test/java/org/apache/sqoop/model/TestMFormList.java → common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
View File

@ -25,34 +25,31 @@
import static org.junit.Assert.assertEquals;
/**
*
*/
public class TestMFormList {
public class TestMConfigList {
@Test
public void testGetInputs() {
List<MForm> forms = new LinkedList<MForm>();
List<MConfig> configs = new LinkedList<MConfig>();
MIntegerInput intInput = new MIntegerInput("Form1.A", false);
MMapInput mapInput = new MMapInput("Form1.B", false);
MIntegerInput intInput = new MIntegerInput("Config1.A", false);
MMapInput mapInput = new MMapInput("Config1.B", false);
List<MInput<?>> inputs = new ArrayList<MInput<?>>();
inputs.add(intInput);
inputs.add(mapInput);
forms.add(new MForm("Form1", inputs));
configs.add(new MConfig("Config1", inputs));
MStringInput stringInput = new MStringInput("Form2.C", false, (short)3);
MEnumInput enumInput = new MEnumInput("Form2.D", false, new String[] {"I", "V"});
MStringInput stringInput = new MStringInput("Config2.C", false, (short)3);
MEnumInput enumInput = new MEnumInput("Config2.D", false, new String[] {"I", "V"});
inputs = new ArrayList<MInput<?>>();
inputs.add(stringInput);
inputs.add(enumInput);
forms.add(new MForm("Form2", inputs));
configs.add(new MConfig("Config2", inputs));
MFormList form = new MFormList(forms);
assertEquals(intInput, form.getIntegerInput("Form1.A"));
assertEquals(mapInput, form.getMapInput("Form1.B"));
assertEquals(stringInput, form.getStringInput("Form2.C"));
assertEquals(enumInput, form.getEnumInput("Form2.D"));
MConfigList config = new MConfigList(configs);
assertEquals(intInput, config.getIntegerInput("Config1.A"));
assertEquals(mapInput, config.getMapInput("Config1.B"));
assertEquals(stringInput, config.getStringInput("Config2.C"));
assertEquals(enumInput, config.getEnumInput("Config2.D"));
}
}

140
common/src/test/java/org/apache/sqoop/model/TestMConnector.java
View File

@ -17,6 +17,13 @@
*/
package org.apache.sqoop.model;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -24,39 +31,34 @@
import org.apache.sqoop.common.Direction;
import org.junit.Test;
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.model.TestMConnector
*/
public class TestMConnector {
private MConnector createConnector(List<Direction> supportedDirections) {
List<MForm> forms = new ArrayList<MForm>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
input.setValue(100);
List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput inputs = new MIntegerInput("INTEGER-INPUT", false);
inputs.setValue(100);
MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
strInput.setValue("TEST-VALUE");
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
list.add(inputs);
list.add(strInput);
MForm form = new MForm("FORMNAME", list);
forms.add(form);
MConfig config = new MConfig("CONFIGNAME", list);
configs.add(config);
MConnectionForms connectionForms1 = new MConnectionForms(forms);
MJobForms fromForm = null;
MJobForms toForm = null;
MLinkConfig linkConfig = new MLinkConfig(configs);
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (supportedDirections.contains(Direction.FROM)) {
fromForm = new MJobForms(forms);
fromConfig = new MFromConfig(configs);
}
if (supportedDirections.contains(Direction.TO)) {
toForm = new MJobForms(forms);
toConfig = new MToConfig(configs);
}
return new MConnector("NAME", "CLASSNAME", "1.0",
connectionForms1, fromForm, toForm);
linkConfig, fromConfig, toConfig);
}
/**
@ -64,32 +66,32 @@ private MConnector createConnector(List<Direction> supportedDirections) {
*/
@Test
public void testInitialization() {
List<MForm> fromJobForms = new ArrayList<MForm>();
List<MForm> toJobForms = new ArrayList<MForm>();
MConnectionForms connectionForms1 = new MConnectionForms(fromJobForms);
MJobForms fromJobForm1 = new MJobForms(fromJobForms);
MJobForms toJobForm1 = new MJobForms(toJobForms);
List<MConfig> fromJobConfig = new ArrayList<MConfig>();
List<MConfig> toJobConfig = new ArrayList<MConfig>();
MLinkConfig linkConfig = new MLinkConfig(fromJobConfig);
MFromConfig fromConfig1 = new MFromConfig(fromJobConfig);
MToConfig toConfig1 = new MToConfig(toJobConfig);
MConnector connector1 = new MConnector("NAME", "CLASSNAME", "1.0",
connectionForms1, fromJobForm1, toJobForm1);
linkConfig, fromConfig1, toConfig1);
assertEquals("NAME", connector1.getUniqueName());
assertEquals("CLASSNAME", connector1.getClassName());
assertEquals("1.0", connector1.getVersion());
MConnector connector2 = new MConnector("NAME", "CLASSNAME", "1.0",
connectionForms1, fromJobForm1, toJobForm1);
linkConfig, fromConfig1, toConfig1);
assertEquals(connector2, connector1);
MConnector connector3 = new MConnector("NAME1", "CLASSNAME", "2.0",
connectionForms1, fromJobForm1, toJobForm1);
linkConfig, fromConfig1, toConfig1);
assertFalse(connector1.equals(connector3));
try {
connector1 = new MConnector(null, "CLASSNAME", "1.0", connectionForms1,
fromJobForm1, toJobForm1); // Expecting null pointer exception
connector1 = new MConnector(null, "CLASSNAME", "1.0", linkConfig,
fromConfig1, toConfig1); // Expecting null pointer exception
} catch (NullPointerException e) {
assertTrue(true);
}
try {
connector1 = new MConnector("NAME", null, "1.0", connectionForms1,
fromJobForm1, toJobForm1); // Expecting null pointer exception
connector1 = new MConnector("NAME", null, "1.0", linkConfig,
fromConfig1, toConfig1); // Expecting null pointer exception
} catch (NullPointerException e) {
assertTrue(true);
}
@ -97,48 +99,48 @@ public void testInitialization() {
@Test
public void testClone() {
MConnector connector1 = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
assertEquals("NAME", connector1.getUniqueName());
assertEquals("CLASSNAME", connector1.getClassName());
assertEquals("1.0", connector1.getVersion());
//Clone with values. Checking values copying after the cloning. But form values will be null
MConnector clone1 = connector1.clone(true);
assertEquals("NAME", clone1.getUniqueName());
assertEquals("CLASSNAME", clone1.getClassName());
assertEquals("1.0", clone1.getVersion());
MForm clonedForm1 = clone1.getConnectionForms().getForms().get(0);
assertNull(clonedForm1.getInputs().get(0).getValue());
assertNull(clonedForm1.getInputs().get(1).getValue());
MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
assertEquals("NAME", connector.getUniqueName());
assertEquals("CLASSNAME", connector.getClassName());
assertEquals("1.0", connector.getVersion());
//Clone with values. Checking values copying after the cloning. But config values will be null
MConnector cloneConnector1 = connector.clone(true);
assertEquals("NAME", cloneConnector1.getUniqueName());
assertEquals("CLASSNAME", cloneConnector1.getClassName());
assertEquals("1.0", cloneConnector1.getVersion());
MConfig clonedLinkConfig = cloneConnector1.getLinkConfig().getConfigs().get(0);
assertNull(clonedLinkConfig.getInputs().get(0).getValue());
assertNull(clonedLinkConfig.getInputs().get(1).getValue());
MForm clonedForm2 = clone1.getJobForms(Direction.FROM).getForms().get(0);
assertNull(clonedForm2.getInputs().get(0).getValue());
assertNull(clonedForm2.getInputs().get(1).getValue());
MConfig clonedFromConfig = cloneConnector1.getConfig(Direction.FROM).getConfigs().get(0);
assertNull(clonedFromConfig.getInputs().get(0).getValue());
assertNull(clonedFromConfig.getInputs().get(1).getValue());
MForm clonedForm3 = clone1.getJobForms(Direction.TO).getForms().get(0);
assertNull(clonedForm3.getInputs().get(0).getValue());
assertNull(clonedForm3.getInputs().get(1).getValue());
MConfig clonedToConfig = cloneConnector1.getConfig(Direction.TO).getConfigs().get(0);
assertNull(clonedToConfig.getInputs().get(0).getValue());
assertNull(clonedToConfig.getInputs().get(1).getValue());
//Clone without values. Inputs value will be null after cloning.
MConnector clone2 = connector1.clone(false);
clonedForm1 = clone2.getConnectionForms().getForms().get(0);
assertNull(clonedForm1.getInputs().get(0).getValue());
assertNull(clonedForm1.getInputs().get(1).getValue());
clonedForm2 = clone2.getJobForms(Direction.FROM).getForms().get(0);
assertNull(clonedForm2.getInputs().get(0).getValue());
assertNull(clonedForm2.getInputs().get(1).getValue());
clonedForm3 = clone2.getJobForms(Direction.TO).getForms().get(0);
assertNull(clonedForm3.getInputs().get(0).getValue());
assertNull(clonedForm3.getInputs().get(1).getValue());
MConnector clonedConnector2 = connector.clone(false);
clonedLinkConfig = clonedConnector2.getLinkConfig().getConfigs().get(0);
assertNull(clonedLinkConfig.getInputs().get(0).getValue());
assertNull(clonedLinkConfig.getInputs().get(1).getValue());
clonedFromConfig = clonedConnector2.getConfig(Direction.FROM).getConfigs().get(0);
assertNull(clonedFromConfig.getInputs().get(0).getValue());
assertNull(clonedFromConfig.getInputs().get(1).getValue());
clonedToConfig = clonedConnector2.getConfig(Direction.TO).getConfigs().get(0);
assertNull(clonedToConfig.getInputs().get(0).getValue());
assertNull(clonedToConfig.getInputs().get(1).getValue());
}
@Test
public void testFromDirection() {
MConnector connector = createConnector(Arrays.asList(Direction.FROM));
// Clone should clone only one job form.
// Clone should clone only one job config.
MConnector clone = connector.clone(true);
assertNotNull(clone.getJobForms(Direction.FROM));
assertNull(clone.getJobForms(Direction.TO));
assertNotNull(clone.getFromConfig());
assertNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
@ -148,10 +150,10 @@ public void testFromDirection() {
public void testToDirection() {
MConnector connector = createConnector(Arrays.asList(Direction.TO));
// Clone should clone only one job form.
// Clone should clone only one job config.
MConnector clone = connector.clone(true);
assertNull(clone.getJobForms(Direction.FROM));
assertNotNull(clone.getJobForms(Direction.TO));
assertNull(clone.getFromConfig());
assertNotNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
@ -161,10 +163,10 @@ public void testToDirection() {
public void testNoDirection() {
MConnector connector = createConnector(Arrays.asList(new Direction[0]));
// Clone should clone only one job form.
// Clone should clone only one job config.
MConnector clone = connector.clone(true);
assertNull(clone.getJobForms(Direction.FROM));
assertNull(clone.getJobForms(Direction.TO));
assertNull(clone.getFromConfig());
assertNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());
@ -174,10 +176,10 @@ public void testNoDirection() {
public void testBothDirections() {
MConnector connector = createConnector(Arrays.asList(Direction.FROM, Direction.TO));
// Clone should clone only one job form.
// Clone should clone only one job config.
MConnector clone = connector.clone(true);
assertNotNull(clone.getJobForms(Direction.FROM));
assertNotNull(clone.getJobForms(Direction.TO));
assertNotNull(clone.getFromConfig());
assertNotNull(clone.getToConfig());
assertEquals(connector, clone);
assertEquals(connector.toString(), clone.toString());
assertNotEquals(connector.hashCode(), clone.hashCode());

40
common/src/test/java/org/apache/sqoop/model/TestMDriver.java Normal file
View File

@ -0,0 +1,40 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.model;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.List;
import org.apache.sqoop.json.DriverBean;
import org.junit.Test;
public class TestMDriver {
@Test
public void testDriver() {
List<MConfig> driverConfig = new ArrayList<MConfig>();
driverConfig.add(new MConfig("driver-test", new ArrayList<MInput<?>>()));
MDriverConfig mDriverConfig = new MDriverConfig(driverConfig);
MDriver driver = new MDriver(mDriverConfig, DriverBean.CURRENT_DRIVER_VERSION);
assertEquals(1, driver.getDriverConfig().getConfigs().size());
assertEquals("driver-test", driver.getDriverConfig().getConfigs().get(0).getName());
}
}

47
common/src/test/java/org/apache/sqoop/model/TestMDriverConfig.java
View File

@ -1,47 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.model;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
/**
*
*/
public class TestMDriverConfig {
@Test
public void testForms() {
List<MForm> connectionFormList = new ArrayList<MForm>();
List<MForm> jobFormList = new ArrayList<MForm>();
connectionFormList.add(new MForm("connection-test", new ArrayList<MInput<?>>()));
jobFormList.add(new MForm("job-test", new ArrayList<MInput<?>>()));
MConnectionForms connectionForms = new MConnectionForms(connectionFormList);
MJobForms jobForms = new MJobForms(jobFormList);
MDriverConfig driver = new MDriverConfig(connectionForms, jobForms, "1");
assertEquals(1, driver.getJobForms().getForms().size());
assertEquals("job-test", driver.getJobForms().getForms().get(0).getName());
assertEquals(1, driver.getConnectionForms().getForms().size());
assertEquals("connection-test", driver.getConnectionForms().getForms().get(0).getName());
}
}

110
common/src/test/java/org/apache/sqoop/model/TestMJob.java
View File

@ -17,17 +17,15 @@
*/
package org.apache.sqoop.model;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.util.ArrayList;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.junit.Test;
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.model.MJob
*/
public class TestMJob {
/**
* Test class for initialization
@ -40,9 +38,9 @@ public void testInitialization() {
assertEquals(456l, job.getConnectorId(Direction.TO));
assertEquals("Buffy", job.getCreationUser());
assertEquals("Vampire", job.getName());
assertEquals(fromForms(), job.getConnectorPart(Direction.FROM));
assertEquals(toForms(), job.getConnectorPart(Direction.TO));
assertEquals(frameworkForms(), job.getFrameworkPart());
assertEquals(fromConfig(), job.getJobConfig(Direction.FROM));
assertEquals(toConfig(), job.getJobConfig(Direction.TO));
assertEquals(driverConfig(), job.getDriverConfig());
// Test copy constructor
MJob copy = new MJob(job);
@ -50,19 +48,19 @@ public void testInitialization() {
assertEquals(456l, copy.getConnectorId(Direction.TO));
assertEquals("Buffy", copy.getCreationUser());
assertEquals("Vampire", copy.getName());
assertEquals(fromForms(), copy.getConnectorPart(Direction.FROM));
assertEquals(toForms(), copy.getConnectorPart(Direction.TO));
assertEquals(frameworkForms(), copy.getFrameworkPart());
assertEquals(fromConfig(), copy.getJobConfig(Direction.FROM));
assertEquals(toConfig(), copy.getJobConfig(Direction.TO));
assertEquals(driverConfig(), copy.getDriverConfig());
// Test constructor for metadata upgrade (the order of forms is different)
MJob upgradeCopy = new MJob(job, fromForms(), toForms(), frameworkForms());
// Test constructor for metadata upgrade (the order of configs is different)
MJob upgradeCopy = new MJob(job, fromConfig(), toConfig(), driverConfig());
assertEquals(123l, upgradeCopy.getConnectorId(Direction.FROM));
assertEquals(456l, upgradeCopy.getConnectorId(Direction.TO));
assertEquals("Buffy", upgradeCopy.getCreationUser());
assertEquals("Vampire", upgradeCopy.getName());
assertEquals(fromForms(), upgradeCopy.getConnectorPart(Direction.FROM));
assertEquals(toForms(), upgradeCopy.getConnectorPart(Direction.TO));
assertEquals(frameworkForms(), upgradeCopy.getFrameworkPart());
assertEquals(fromConfig(), upgradeCopy.getJobConfig(Direction.FROM));
assertEquals(toConfig(), upgradeCopy.getJobConfig(Direction.TO));
assertEquals(driverConfig(), upgradeCopy.getDriverConfig());
}
@Test
@ -70,42 +68,42 @@ public void testClone() {
MJob job = job();
// Clone without value
MJob withoutValue = job.clone(false);
assertEquals(job, withoutValue);
assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutValue.getPersistenceId());
assertNull(withoutValue.getName());
assertNull(withoutValue.getCreationUser());
assertEquals(fromForms(), withoutValue.getConnectorPart(Direction.FROM));
assertEquals(toForms(), withoutValue.getConnectorPart(Direction.TO));
assertEquals(frameworkForms(), withoutValue.getFrameworkPart());
assertNull(withoutValue.getConnectorPart(Direction.FROM)
.getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
assertNull(withoutValue.getConnectorPart(Direction.FROM)
.getForm("FORMNAME").getInput("STRING-INPUT").getValue());
MJob withoutJobValue = job.clone(false);
assertEquals(job, withoutJobValue);
assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutJobValue.getPersistenceId());
assertNull(withoutJobValue.getName());
assertNull(withoutJobValue.getCreationUser());
assertEquals(fromConfig(), withoutJobValue.getJobConfig(Direction.FROM));
assertEquals(toConfig(), withoutJobValue.getJobConfig(Direction.TO));
assertEquals(driverConfig(), withoutJobValue.getDriverConfig());
assertNull(withoutJobValue.getJobConfig(Direction.FROM)
.getConfig("CONFIGFROMNAME").getInput("INTEGER-INPUT").getValue());
assertNull(withoutJobValue.getJobConfig(Direction.FROM)
.getConfig("CONFIGFROMNAME").getInput("STRING-INPUT").getValue());
// Clone with value
MJob withValue = job.clone(true);
assertEquals(job, withValue);
assertEquals(job.getPersistenceId(), withValue.getPersistenceId());
assertEquals(job.getName(), withValue.getName());
assertEquals(job.getCreationUser(), withValue.getCreationUser());
assertEquals(fromForms(), withValue.getConnectorPart(Direction.FROM));
assertEquals(toForms(), withValue.getConnectorPart(Direction.TO));
assertEquals(frameworkForms(), withValue.getFrameworkPart());
assertEquals(100, withValue.getConnectorPart(Direction.FROM)
.getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
assertEquals("TEST-VALUE", withValue.getConnectorPart(Direction.FROM)
.getForm("FORMNAME").getInput("STRING-INPUT").getValue()); }
MJob withJobValue = job.clone(true);
assertEquals(job, withJobValue);
assertEquals(job.getPersistenceId(), withJobValue.getPersistenceId());
assertEquals(job.getName(), withJobValue.getName());
assertEquals(job.getCreationUser(), withJobValue.getCreationUser());
assertEquals(fromConfig(), withJobValue.getJobConfig(Direction.FROM));
assertEquals(toConfig(), withJobValue.getJobConfig(Direction.TO));
assertEquals(driverConfig(), withJobValue.getDriverConfig());
assertEquals(100, withJobValue.getJobConfig(Direction.FROM)
.getConfig("CONFIGFROMNAME").getInput("INTEGER-INPUT").getValue());
assertEquals("TEST-VALUE", withJobValue.getJobConfig(Direction.FROM)
.getConfig("CONFIGFROMNAME").getInput("STRING-INPUT").getValue()); }
private MJob job() {
MJob job = new MJob(123l, 456l, 1L, 2L, fromForms(), toForms(), frameworkForms());
MJob job = new MJob(123l, 456l, 1L, 2L, fromConfig(), toConfig(), driverConfig());
job.setName("Vampire");
job.setCreationUser("Buffy");
return job;
}
private MJobForms fromForms() {
List<MForm> forms = new ArrayList<MForm>();
private MFromConfig fromConfig() {
List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
input.setValue(100);
MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
@ -113,28 +111,28 @@ private MJobForms fromForms() {
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
list.add(strInput);
MForm form = new MForm("FORMNAME", list);
forms.add(form);
return new MJobForms(forms);
MConfig config = new MConfig("CONFIGFROMNAME", list);
configs.add(config);
return new MFromConfig(configs);
}
private MJobForms toForms() {
List<MForm> forms = new ArrayList<MForm>();
private MToConfig toConfig() {
List<MConfig> configs = new ArrayList<MConfig>();
MMapInput input = new MMapInput("MAP-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
MForm form = new MForm("form", list);
forms.add(form);
return new MJobForms(forms);
MConfig config = new MConfig("CONFIGTONAME", list);
configs.add(config);
return new MToConfig(configs);
}
private MJobForms frameworkForms() {
List<MForm> forms = new ArrayList<MForm>();
private MDriverConfig driverConfig() {
List<MConfig> configs = new ArrayList<MConfig>();
MMapInput input = new MMapInput("MAP-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
MForm form = new MForm("form", list);
forms.add(form);
return new MJobForms(forms);
MConfig config = new MConfig("CONFIGDRIVERNAME", list);
configs.add(config);
return new MDriverConfig(configs);
}
}

22
common/src/test/java/org/apache/sqoop/model/TestMJobForms.java → common/src/test/java/org/apache/sqoop/model/TestMJobConfig.java
View File

@ -24,23 +24,19 @@
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.model.MJobForms
*/
public class TestMJobForms {
public class TestMJobConfig {
/**
* Test for class initialization and values
*/
@Test
public void testInitialization() {
List<MForm> forms = new ArrayList<MForm>();
MJobForms jobform1 = new MJobForms(forms);
List<MForm> forms2 = new ArrayList<MForm>();
MJobForms jobform2 = new MJobForms(forms2);
assertEquals(jobform2, jobform1);
// Add a form to list for checking not equals
MForm m = new MForm("test", null);
forms2.add(m);
assertFalse(jobform1.equals(jobform2));
List<MConfig> configs = new ArrayList<MConfig>();
MFromConfig fromJobConfig = new MFromConfig(configs);
List<MConfig> configs2 = new ArrayList<MConfig>();
MFromConfig fromJobConfig2 = new MFromConfig(configs2);
assertEquals(fromJobConfig2, fromJobConfig);
MConfig c = new MConfig("test", null);
configs2.add(c);
assertFalse(fromJobConfig.equals(fromJobConfig2));
}
}

74
common/src/test/java/org/apache/sqoop/model/TestMLink.java
View File

@ -24,9 +24,6 @@
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.model.MConnection
*/
public class TestMLink {
/**
@ -39,8 +36,7 @@ public void testInitialization() {
assertEquals(123l, link.getConnectorId());
assertEquals("Vampire", link.getName());
assertEquals("Buffy", link.getCreationUser());
assertEquals(forms1(), link.getConnectorPart());
assertEquals(forms2(), link.getFrameworkPart());
assertEquals(linkConfig(), link.getConnectorLinkConfig());
// Test copy constructor
MLink copy = new MLink(link);
@ -48,17 +44,7 @@ public void testInitialization() {
assertEquals("Vampire", copy.getName());
assertEquals("Buffy", copy.getCreationUser());
assertEquals(link.getCreationDate(), copy.getCreationDate());
assertEquals(forms1(), copy.getConnectorPart());
assertEquals(forms2(), copy.getFrameworkPart());
// Test constructor for metadata upgrade (the order of forms is different)
MLink upgradeCopy = new MLink(link, forms2(), forms1());
assertEquals(123l, upgradeCopy.getConnectorId());
assertEquals("Vampire", upgradeCopy.getName());
assertEquals("Buffy", upgradeCopy.getCreationUser());
assertEquals(link.getCreationDate(), upgradeCopy.getCreationDate());
assertEquals(forms2(), upgradeCopy.getConnectorPart());
assertEquals(forms1(), upgradeCopy.getFrameworkPart());
assertEquals(linkConfig(), copy.getConnectorLinkConfig());
}
@Test
@ -66,37 +52,35 @@ public void testClone() {
MLink link = link();
// Clone without value
MLink withoutValue = link.clone(false);
assertEquals(link, withoutValue);
assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutValue.getPersistenceId());
assertNull(withoutValue.getName());
assertNull(withoutValue.getCreationUser());
assertEquals(forms1(), withoutValue.getConnectorPart());
assertEquals(forms2(), withoutValue.getFrameworkPart());
assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
assertNull(withoutValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
MLink withoutLinkValue = link.clone(false);
assertEquals(link, withoutLinkValue);
assertEquals(MPersistableEntity.PERSISTANCE_ID_DEFAULT, withoutLinkValue.getPersistenceId());
assertNull(withoutLinkValue.getName());
assertNull(withoutLinkValue.getCreationUser());
assertEquals(linkConfig(), withoutLinkValue.getConnectorLinkConfig());
assertNull(withoutLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("INTEGER-INPUT").getValue());
assertNull(withoutLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("STRING-INPUT").getValue());
// Clone with value
MLink withValue = link.clone(true);
assertEquals(link, withValue);
assertEquals(link.getPersistenceId(), withValue.getPersistenceId());
assertEquals(link.getName(), withValue.getName());
assertEquals(link.getCreationUser(), withValue.getCreationUser());
assertEquals(forms1(), withValue.getConnectorPart());
assertEquals(forms2(), withValue.getFrameworkPart());
assertEquals(100, withValue.getConnectorPart().getForm("FORMNAME").getInput("INTEGER-INPUT").getValue());
assertEquals("TEST-VALUE", withValue.getConnectorPart().getForm("FORMNAME").getInput("STRING-INPUT").getValue());
MLink withLinkValue = link.clone(true);
assertEquals(link, withLinkValue);
assertEquals(link.getPersistenceId(), withLinkValue.getPersistenceId());
assertEquals(link.getName(), withLinkValue.getName());
assertEquals(link.getCreationUser(), withLinkValue.getCreationUser());
assertEquals(linkConfig(), withLinkValue.getConnectorLinkConfig());
assertEquals(100, withLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("INTEGER-INPUT").getValue());
assertEquals("TEST-VALUE", withLinkValue.getConnectorLinkConfig().getConfig("CONFIGNAME").getInput("STRING-INPUT").getValue());
}
private MLink link() {
MLink link = new MLink(123l, forms1(), forms2());
MLink link = new MLink(123l, linkConfig());
link.setName("Vampire");
link.setCreationUser("Buffy");
return link;
}
private MConnectionForms forms1() {
List<MForm> forms = new ArrayList<MForm>();
private MLinkConfig linkConfig() {
List<MConfig> configs = new ArrayList<MConfig>();
MIntegerInput input = new MIntegerInput("INTEGER-INPUT", false);
input.setValue(100);
MStringInput strInput = new MStringInput("STRING-INPUT",false,(short)20);
@ -104,19 +88,9 @@ private MConnectionForms forms1() {
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
list.add(strInput);
MForm form = new MForm("FORMNAME", list);
forms.add(form);
return new MConnectionForms(forms);
}
private MConnectionForms forms2() {
List<MForm> forms = new ArrayList<MForm>();
MMapInput input = new MMapInput("MAP-INPUT", false);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
MForm form = new MForm("form", list);
forms.add(form);
return new MConnectionForms(forms);
MConfig config = new MConfig("CONFIGNAME", list);
configs.add(config);
return new MLinkConfig(configs);
}
}

25
common/src/test/java/org/apache/sqoop/model/TestMConnectionForms.java → common/src/test/java/org/apache/sqoop/model/TestMLinkConfig.java
View File

@ -24,25 +24,22 @@
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.model.MConnectionForms
*/
public class TestMConnectionForms {
public class TestMLinkConfig {
/**
* Test for class initialization and values
*/
@Test
public void testInitialization() {
List<MForm> forms = new ArrayList<MForm>();
MConnectionForms connectionForms1 = new MConnectionForms(forms);
List<MForm> testForms = new ArrayList<MForm>();
assertEquals(testForms, connectionForms1.getForms());
MConnectionForms connectionForms2 = new MConnectionForms(testForms);
assertEquals(connectionForms2, connectionForms1);
// Add a form to list for checking not equals
MForm m = new MForm("test", null);
testForms.add(m);
assertFalse(connectionForms1.equals(connectionForms2));
List<MConfig> configs = new ArrayList<MConfig>();
MLinkConfig linkConfig = new MLinkConfig(configs);
List<MConfig> testConfig = new ArrayList<MConfig>();
assertEquals(testConfig, linkConfig.getConfigs());
MLinkConfig linkConfig2 = new MLinkConfig(testConfig);
assertEquals(linkConfig2, linkConfig);
// Add a config to list for checking not equals
MConfig c = new MConfig("test", null);
testConfig.add(c);
assertFalse(linkConfig.equals(linkConfig2));
}
}

70
common/src/test/java/org/apache/sqoop/validation/TestValidation.java
View File

@ -17,16 +17,20 @@
*/
package org.apache.sqoop.validation;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.HashMap;
import java.util.Map;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.validation.Validation.FormInput;
import org.apache.sqoop.validation.Validation.Message;
import org.apache.sqoop.validation.ConfigValidator.ConfigInput;
import org.apache.sqoop.validation.ConfigValidator.Message;
import org.junit.Test;
import static org.junit.Assert.*;
/**
* Test class for org.apache.sqoop.validation.Validation
*/
@ -38,42 +42,42 @@ public class TestValidation {
@Test
public void testInitialization() {
/* Check initialization with class */
Validation validation = new Validation(Class.class);
ConfigValidator validation = new ConfigValidator(Class.class);
assertNotNull(validation);
assertEquals(Status.FINE, validation.getStatus());
assertEquals(0, validation.getMessages().size());
/* Check initialization with status and message as null */
Validation validationNull = new Validation(null, null);
ConfigValidator validationNull = new ConfigValidator(null, null);
assertNotNull(validationNull);
assertNull(validationNull.getStatus());
assertNull(validationNull.getMessages());
/* Check initialization with status and message with values */
Status s1 = Status.FINE;
Map<FormInput, Message> msg1 = new HashMap<Validation.FormInput, Validation.Message>();
Validation validation1 = new Validation(s1, msg1);
Map<ConfigInput, Message> msg1 = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
ConfigValidator validation1 = new ConfigValidator(s1, msg1);
assertNotNull(validation1);
assertEquals(Status.FINE, validation1.getStatus());
assertEquals(0, validation1.getMessages().size());
/* Check initialization with status and message with values */
Status s2 = Status.ACCEPTABLE;
Map<FormInput, Message> msg2 = new HashMap<Validation.FormInput, Validation.Message>();
Validation validation2 = new Validation(s2, msg2);
Map<ConfigInput, Message> msg2 = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
ConfigValidator validation2 = new ConfigValidator(s2, msg2);
assertNotNull(validation2);
assertEquals(Status.ACCEPTABLE, validation2.getStatus());
assertEquals(0, validation2.getMessages().size());
/* Check initialization with status and message with values */
Status s3 = Status.ACCEPTABLE;
Map<FormInput, Message> msg3 = new HashMap<Validation.FormInput, Validation.Message>();
Validation.FormInput fi = new Validation.FormInput("form\\.input");
Validation.Message message = new Validation.Message(Status.FINE, "sqoop");
Map<ConfigInput, Message> msg3 = new HashMap<ConfigValidator.ConfigInput, ConfigValidator.Message>();
ConfigValidator.ConfigInput fi = new ConfigValidator.ConfigInput("config\\.input");
ConfigValidator.Message message = new ConfigValidator.Message(Status.FINE, "sqoop");
msg3.put(fi, message);
Validation validation3 = new Validation(s3, msg3);
Validation.FormInput fiTest = new Validation.FormInput("form\\.input");
Validation.Message messageTest = new Validation.Message(Status.FINE,
ConfigValidator validation3 = new ConfigValidator(s3, msg3);
ConfigValidator.ConfigInput fiTest = new ConfigValidator.ConfigInput("config\\.input");
ConfigValidator.Message messageTest = new ConfigValidator.Message(Status.FINE,
"sqoop");
assertEquals(messageTest, validation3.getMessages().get(fiTest));
assertEquals(Status.ACCEPTABLE, validation3.getStatus());
@ -82,13 +86,13 @@ public void testInitialization() {
/**
* Test for Validation.ForInput
*/
public void testFormInput() {
Validation.FormInput fi = new Validation.FormInput("test\\.test");
public void testConfigInput() {
ConfigValidator.ConfigInput fi = new ConfigValidator.ConfigInput("test\\.test");
assertNotNull(fi);
/* Passing null */
try {
new Validation.FormInput(null);
new ConfigValidator.ConfigInput(null);
fail("Assert error is expected");
} catch (AssertionError e) {
assertTrue(true);
@ -96,31 +100,31 @@ public void testFormInput() {
/* Passing empty and check exception messages */
try {
new Validation.FormInput("");
new ConfigValidator.ConfigInput("");
fail("SqoopException is expected");
} catch (SqoopException e) {
assertEquals(ValidationError.VALIDATION_0003.getMessage(), e
assertEquals(ConfigValidationError.VALIDATION_0003.getMessage(), e
.getErrorCode().getMessage());
}
/* Passing value and check */
Validation.FormInput fi2 = new Validation.FormInput("form\\.input");
assertEquals("form\\", fi2.getForm());
ConfigValidator.ConfigInput fi2 = new ConfigValidator.ConfigInput("config\\.input");
assertEquals("config\\", fi2.getConfig());
assertEquals("input", fi2.getInput());
/* Check equals */
Validation.FormInput fiOne = new Validation.FormInput("form\\.input");
Validation.FormInput fiTwo = new Validation.FormInput("form\\.input");
ConfigValidator.ConfigInput fiOne = new ConfigValidator.ConfigInput("config\\.input");
ConfigValidator.ConfigInput fiTwo = new ConfigValidator.ConfigInput("config\\.input");
assertEquals(fiOne, fiTwo);
/* toString() method check */
assertEquals("form\\.input", fiOne.toString());
assertEquals("config\\.input", fiOne.toString());
// Checking null as input field (form validation)
Validation.FormInput fi3 = new FormInput("form");
assertEquals("form", fi3.getForm());
// Checking null as input field (config validation)
ConfigValidator.ConfigInput fi3 = new ConfigInput("config");
assertEquals("config", fi3.getConfig());
assertNull(fi3.getInput());
assertEquals("form", fi3.toString());
assertEquals("config", fi3.toString());
}
@ -129,17 +133,17 @@ public void testFormInput() {
*/
public void testMessage() {
/* Passing null */
Validation.Message msg1 = new Validation.Message(null, null);
ConfigValidator.Message msg1 = new ConfigValidator.Message(null, null);
assertNull(msg1.getStatus());
assertNull(msg1.getMessage());
/* Passing values */
Validation.Message msg2 = new Validation.Message(Status.FINE, "sqoop");
ConfigValidator.Message msg2 = new ConfigValidator.Message(Status.FINE, "sqoop");
assertEquals(Status.FINE, msg2.getStatus());
assertEquals("sqoop", msg2.getMessage());
/* Check for equal */
Validation.Message msg3 = new Validation.Message(Status.FINE, "sqoop");
ConfigValidator.Message msg3 = new ConfigValidator.Message(Status.FINE, "sqoop");
assertEquals(msg2, msg3);
}
}

96
common/src/test/java/org/apache/sqoop/validation/TestValidationRunner.java
View File

@ -18,8 +18,8 @@
package org.apache.sqoop.validation;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.Config;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.validators.Contains;
@ -35,18 +35,18 @@
*/
public class TestValidationRunner {
@FormClass(validators = {@Validator(FormA.FormValidator.class)})
public static class FormA {
@ConfigClass(validators = {@Validator(ConfigA.ConfigValidator.class)})
public static class ConfigA {
@Input(validators = {@Validator(NotNull.class)})
String notNull;
public static class FormValidator extends AbstractValidator<FormA> {
public static class ConfigValidator extends AbstractValidator<ConfigA> {
@Override
public void validate(FormA form) {
if(form.notNull == null) {
public void validate(ConfigA config) {
if(config.notNull == null) {
addMessage(Status.UNACCEPTABLE, "null");
}
if("error".equals(form.notNull)) {
if("error".equals(config.notNull)) {
addMessage(Status.UNACCEPTABLE, "error");
}
}
@ -54,83 +54,83 @@ public void validate(FormA form) {
}
@Test
public void testValidateForm() {
FormA form = new FormA();
ValidationRunner runner = new ValidationRunner();
ValidationResult result;
public void testValidateConfig() {
ConfigA config = new ConfigA();
ConfigValidationRunner runner = new ConfigValidationRunner();
ConfigValidationResult result;
// Null string should fail on Input level and should not call form level validators
form.notNull = null;
result = runner.validateForm("formName", form);
// Null string should fail on Input level and should not call config level validators
config.notNull = null;
result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formName.notNull"));
assertTrue(result.getMessages().containsKey("configName.notNull"));
// String "error" should trigger form level error, but not Input level
form.notNull = "error";
result = runner.validateForm("formName", form);
// String "error" should trigger config level error, but not Input level
config.notNull = "error";
result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formName"));
assertTrue(result.getMessages().containsKey("configName"));
// Acceptable state
form.notNull = "This is truly random string";
result = runner.validateForm("formName", form);
config.notNull = "This is truly random string";
result = runner.validateConfig("configName", config);
assertEquals(Status.FINE, result.getStatus());
assertEquals(0, result.getMessages().size());
}
@FormClass
public static class FormB {
@ConfigClass
public static class ConfigB {
@Input(validators = {@Validator(NotNull.class), @Validator(NotEmpty.class)})
String str;
}
@FormClass
public static class FormC {
@ConfigClass
public static class ConfigC {
@Input(validators = {@Validator(value = Contains.class, strArg = "findme")})
String str;
}
@Test
public void testMultipleValidatorsOnSingleInput() {
FormB form = new FormB();
ValidationRunner runner = new ValidationRunner();
ValidationResult result;
ConfigB config = new ConfigB();
ConfigValidationRunner runner = new ConfigValidationRunner();
ConfigValidationResult result;
form.str = null;
result = runner.validateForm("formName", form);
config.str = null;
result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formName.str"));
assertEquals(2, result.getMessages().get("formName.str").size());
assertTrue(result.getMessages().containsKey("configName.str"));
assertEquals(2, result.getMessages().get("configName.str").size());
}
@Test
public void testValidatorWithParameters() {
FormC form = new FormC();
ValidationRunner runner = new ValidationRunner();
ValidationResult result;
ConfigC config = new ConfigC();
ConfigValidationRunner runner = new ConfigValidationRunner();
ConfigValidationResult result;
// Sub string not found
form.str = "Mordor";
result = runner.validateForm("formName", form);
config.str = "Mordor";
result = runner.validateConfig("configName", config);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formName.str"));
assertTrue(result.getMessages().containsKey("configName.str"));
// Sub string found
form.str = "Morfindmedor";
result = runner.validateForm("formName", form);
config.str = "Morfindmedor";
result = runner.validateConfig("configName", config);
assertEquals(Status.FINE, result.getStatus());
assertEquals(0, result.getMessages().size());
}
@ConfigurationClass(validators = {@Validator(ConfigurationA.ClassValidator.class)})
public static class ConfigurationA {
@Form FormA formA;
@Config ConfigA formA;
public ConfigurationA() {
formA = new FormA();
formA = new ConfigA();
}
public static class ClassValidator extends AbstractValidator<ConfigurationA> {
@ -149,24 +149,24 @@ public void validate(ConfigurationA conf) {
@Test
public void testValidate() {
ConfigurationA conf = new ConfigurationA();
ValidationRunner runner = new ValidationRunner();
ValidationResult result;
ConfigValidationRunner runner = new ConfigValidationRunner();
ConfigValidationResult result;
// Null string should fail on Input level and should not call form nor class level validators
// Null string should fail on Input level and should not call config nor class level validators
conf.formA.notNull = null;
result = runner.validate(conf);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formA.notNull"));
// String "error" should trigger form level error, but not Input nor class level
// String "error" should trigger config level error, but not Input nor class level
conf.formA.notNull = "error";
result = runner.validate(conf);
assertEquals(Status.UNACCEPTABLE, result.getStatus());
assertEquals(1, result.getMessages().size());
assertTrue(result.getMessages().containsKey("formA"));
// String "conf-error" should trigger class level error, but not Input nor Form level
// String "conf-error" should trigger class level error, but not Input nor Config level
conf.formA.notNull = "conf-error";
result = runner.validate(conf);
assertEquals(Status.UNACCEPTABLE, result.getStatus());

12
common/src/test/java/org/apache/sqoop/validation/validators/TestClassAvailable.java
View File

@ -17,19 +17,17 @@
*/
package org.apache.sqoop.validation.validators;
import static org.junit.Assert.assertEquals;
import java.util.List;
import org.apache.sqoop.validation.Message;
import org.apache.sqoop.validation.Status;
import org.junit.Test;
import java.util.List;
import static org.junit.Assert.assertEquals;
/**
*/
public class TestClassAvailable {
AbstractValidator validator = new ClassAvailable();
AbstractValidator<String> validator = new ClassAvailable();
@Test
public void test() {

2
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
View File

@ -92,7 +92,7 @@ public To getTo() {
}
@Override
public Validator getValidator() {
public Validator getConfigValidator() {
return genericJdbcValidator;
}

56
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
View File

@ -18,21 +18,20 @@
*/
package org.apache.sqoop.connector.jdbc;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MJobForms;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MLinkConfig;
public class GenericJdbcConnectorUpgrader extends RepositoryUpgrader {
private static final Logger LOG =
Logger.getLogger(GenericJdbcConnectorUpgrader.class);
private static final Logger LOG = Logger.getLogger(GenericJdbcConnectorUpgrader.class);
/*
* For now, there is no real upgrade. So copy all data over,
@ -41,41 +40,40 @@ public class GenericJdbcConnectorUpgrader extends RepositoryUpgrader {
*/
@Override
public void upgrade(MConnectionForms original,
MConnectionForms upgradeTarget) {
doUpgrade(original.getForms(), upgradeTarget.getForms());
public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@Override
public void upgrade(MJobForms original, MJobForms upgradeTarget) {
doUpgrade(original.getForms(), upgradeTarget.getForms());
public void upgrade(MConfigList original, MConfigList upgradeTarget) {
doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@SuppressWarnings("unchecked")
private void doUpgrade(List<MForm> original, List<MForm> target) {
// Easier to find the form in the original forms list if we use a map.
// Since the constructor of MJobForms takes a list,
private void doUpgrade(List<MConfig> original, List<MConfig> target) {
// Easier to find the config in the original list if we use a map.
// Since the constructor takes a list,
// index is not guaranteed to be the same, so we need to look for
// equivalence
Map<String, MForm> formMap = new HashMap<String, MForm>();
for (MForm form : original) {
formMap.put(form.getName(), form);
Map<String, MConfig> configMap = new HashMap<String, MConfig>();
for (MConfig config : original) {
configMap.put(config.getName(), config);
}
for (MForm form : target) {
List<MInput<?>> inputs = form.getInputs();
MForm originalForm = formMap.get(form.getName());
if (originalForm == null) {
LOG.warn("Form: '" + form.getName() + "' not present in old " +
"connector. So it and its inputs will not be transferred by the upgrader.");
for (MConfig config : target) {
List<MInput<?>> inputs = config.getInputs();
MConfig orginalConfig = configMap.get(config.getName());
if (orginalConfig == null) {
LOG.warn("Config: '" + config.getName() + "' not present in old " +
"generic JDBC connector. So it and its inputs will not be transferred by the upgrader.");
continue;
}
for (MInput input : inputs) {
try {
MInput originalInput = originalForm.getInput(input.getName());
MInput originalInput = orginalConfig.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
LOG.warn("Input: '" + input.getName() + "' not present in old " +
"connector. So it will not be transferred by the upgrader.");
"generic JDBC connector. So it will not be transferred by the upgrader.");
}
}
}

12
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExtractor.java
View File

@ -34,12 +34,12 @@ public class GenericJdbcExtractor extends Extractor<LinkConfiguration, FromJobCo
private long rowsRead = 0;
@Override
public void extract(ExtractorContext context, LinkConfiguration linkConf,
FromJobConfiguration fromJobConf, GenericJdbcPartition partition) {
String driver = linkConf.link.jdbcDriver;
String url = linkConf.link.connectionString;
String username = linkConf.link.username;
String password = linkConf.link.password;
public void extract(ExtractorContext context, LinkConfiguration linkConfig,
FromJobConfiguration fromJobConfig, GenericJdbcPartition partition) {
String driver = linkConfig.linkConfig.jdbcDriver;
String url = linkConfig.linkConfig.connectionString;
String username = linkConfig.linkConfig.username;
String password = linkConfig.linkConfig.password;
GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
String query = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_FROM_DATA_SQL);

2
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromDestroyer.java
View File

@ -29,7 +29,7 @@ public class GenericJdbcFromDestroyer extends Destroyer<LinkConfiguration, FromJ
Logger.getLogger(GenericJdbcFromDestroyer.class);
@Override
public void destroy(DestroyerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
public void destroy(DestroyerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
LOG.info("Running generic JDBC connector destroyer");
}

36
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java
View File

@ -45,34 +45,34 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F
private GenericJdbcExecutor executor;
@Override
public void initialize(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
configureJdbcProperties(context.getContext(), linkConf, fromJobConf);
public void initialize(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
configureJdbcProperties(context.getContext(), linkConfig, fromJobConfig);
try {
configurePartitionProperties(context.getContext(), linkConf, fromJobConf);
configureTableProperties(context.getContext(), linkConf, fromJobConf);
configurePartitionProperties(context.getContext(), linkConfig, fromJobConfig);
configureTableProperties(context.getContext(), linkConfig, fromJobConfig);
} finally {
executor.close();
}
}
@Override
public List<String> getJars(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
List<String> jars = new LinkedList<String>();
jars.add(ClassUtils.jarForClass(linkConf.link.jdbcDriver));
jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver));
return jars;
}
@Override
public Schema getSchema(InitializerContext context, LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
configureJdbcProperties(context.getContext(), linkConf, fromJobConf);
public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
configureJdbcProperties(context.getContext(), linkConfig, fromJobConfig);
String schemaName = fromJobConf.fromJobConfig.tableName;
String schemaName = fromJobConfig.fromJobConfig.tableName;
if(schemaName == null) {
schemaName = "Query";
} else if(fromJobConf.fromJobConfig.schemaName != null) {
schemaName = fromJobConf.fromJobConfig.schemaName + "." + schemaName;
} else if(fromJobConfig.fromJobConfig.schemaName != null) {
schemaName = fromJobConfig.fromJobConfig.schemaName + "." + schemaName;
}
Schema schema = new Schema(schemaName);
@ -117,11 +117,11 @@ public Schema getSchema(InitializerContext context, LinkConfiguration linkConf,
}
}
private void configureJdbcProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
String driver = connectionConfig.link.jdbcDriver;
String url = connectionConfig.link.connectionString;
String username = connectionConfig.link.username;
String password = connectionConfig.link.password;
private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
String driver = linkConfig.linkConfig.jdbcDriver;
String url = linkConfig.linkConfig.connectionString;
String username = linkConfig.linkConfig.username;
String password = linkConfig.linkConfig.password;
assert driver != null;
assert url != null;
@ -129,7 +129,7 @@ private void configureJdbcProperties(MutableContext context, LinkConfiguration c
executor = new GenericJdbcExecutor(driver, url, username, password);
}
private void configurePartitionProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
private void configurePartitionProperties(MutableContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
// ----- configure column name -----
String partitionColumnName = fromJobConfig.fromJobConfig.partitionColumn;
@ -234,7 +234,7 @@ private void configurePartitionProperties(MutableContext context, LinkConfigurat
}
}
private void configureTableProperties(MutableContext context, LinkConfiguration connectionConfig, FromJobConfiguration fromJobConfig) {
private void configureTableProperties(MutableContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
String dataSql;
String fieldNames;

10
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcLoader.java
View File

@ -30,11 +30,11 @@ public class GenericJdbcLoader extends Loader<LinkConfiguration, ToJobConfigurat
private int batchesPerTransaction = DEFAULT_BATCHES_PER_TRANSACTION;
@Override
public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) throws Exception{
String driver = linkConf.link.jdbcDriver;
String url = linkConf.link.connectionString;
String username = linkConf.link.username;
String password = linkConf.link.password;
public void load(LoaderContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) throws Exception{
String driver = linkConfig.linkConfig.jdbcDriver;
String url = linkConfig.linkConfig.connectionString;
String username = linkConfig.linkConfig.username;
String password = linkConfig.linkConfig.password;
GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
executor.setAutoCommit(false);

5
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcPartitioner.java
View File

@ -47,7 +47,8 @@ public class GenericJdbcPartitioner extends Partitioner<LinkConfiguration, FromJ
private Boolean partitionColumnNull;
@Override
public List<Partition> getPartitions(PartitionerContext context,LinkConfiguration linkConf, FromJobConfiguration fromJobConf) {
public List<Partition> getPartitions(PartitionerContext context, LinkConfiguration linkConfig,
FromJobConfiguration fromJobConfig) {
List<Partition> partitions = new LinkedList<Partition>();
numberPartitions = context.getMaxPartitions();
@ -56,7 +57,7 @@ public List<Partition> getPartitions(PartitionerContext context,LinkConfiguratio
partitionMinValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE);
partitionMaxValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE);
partitionColumnNull = fromJobConf.fromJobConfig.partitionColumnNull;
partitionColumnNull = fromJobConfig.fromJobConfig.partitionColumnNull;
if (partitionColumnNull == null) {
partitionColumnNull = false;
}

18
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToDestroyer.java
View File

@ -28,26 +28,26 @@ public class GenericJdbcToDestroyer extends Destroyer<LinkConfiguration, ToJobCo
private static final Logger LOG = Logger.getLogger(GenericJdbcToDestroyer.class);
@Override
public void destroy(DestroyerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
public void destroy(DestroyerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
LOG.info("Running generic JDBC connector destroyer");
final String tableName = toJobConf.toJobConfig.tableName;
final String stageTableName = toJobConf.toJobConfig.stageTableName;
final String tableName = toJobConfig.toJobConfig.tableName;
final String stageTableName = toJobConfig.toJobConfig.stageTableName;
final boolean stageEnabled = stageTableName != null &&
stageTableName.length() > 0;
if(stageEnabled) {
moveDataToDestinationTable(linkConf,
moveDataToDestinationTable(linkConfig,
context.isSuccess(), stageTableName, tableName);
}
}
private void moveDataToDestinationTable(LinkConfiguration linkConf,
private void moveDataToDestinationTable(LinkConfiguration linkConfig,
boolean success, String stageTableName, String tableName) {
GenericJdbcExecutor executor =
new GenericJdbcExecutor(linkConf.link.jdbcDriver,
linkConf.link.connectionString,
linkConf.link.username,
linkConf.link.password);
new GenericJdbcExecutor(linkConfig.linkConfig.jdbcDriver,
linkConfig.linkConfig.connectionString,
linkConfig.linkConfig.username,
linkConfig.linkConfig.password);
try {
if(success) {
LOG.info("Job completed, transferring data from stage fromTable to " +

32
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
View File

@ -43,35 +43,35 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ
Logger.getLogger(GenericJdbcToInitializer.class);
@Override
public void initialize(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
configureJdbcProperties(context.getContext(), linkConf, toJobConf);
public void initialize(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
configureJdbcProperties(context.getContext(), linkConfig, toJobConfig);
try {
configureTableProperties(context.getContext(), linkConf, toJobConf);
configureTableProperties(context.getContext(), linkConfig, toJobConfig);
} finally {
executor.close();
}
}
@Override
public List<String> getJars(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
List<String> jars = new LinkedList<String>();
jars.add(ClassUtils.jarForClass(linkConf.link.jdbcDriver));
jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver));
return jars;
}
@Override
public Schema getSchema(InitializerContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
configureJdbcProperties(context.getContext(), linkConf, toJobConf);
public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
configureJdbcProperties(context.getContext(), linkConfig, toJobConfig);
String schemaName = toJobConf.toJobConfig.tableName;
String schemaName = toJobConfig.toJobConfig.tableName;
if (schemaName == null) {
throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0019,
"Table name extraction not supported yet.");
}
if(toJobConf.toJobConfig.schemaName != null) {
schemaName = toJobConf.toJobConfig.schemaName + "." + schemaName;
if(toJobConfig.toJobConfig.schemaName != null) {
schemaName = toJobConfig.toJobConfig.schemaName + "." + schemaName;
}
Schema schema = new Schema(schemaName);
@ -110,11 +110,11 @@ public Schema getSchema(InitializerContext context, LinkConfiguration linkConf,
}
}
private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) {
String driver = linkConf.link.jdbcDriver;
String url = linkConf.link.connectionString;
String username = linkConf.link.username;
String password = linkConf.link.password;
private void configureJdbcProperties(MutableContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
String driver = linkConfig.linkConfig.jdbcDriver;
String url = linkConfig.linkConfig.connectionString;
String username = linkConfig.linkConfig.username;
String password = linkConfig.linkConfig.password;
assert driver != null;
assert url != null;
@ -122,7 +122,7 @@ private void configureJdbcProperties(MutableContext context, LinkConfiguration l
executor = new GenericJdbcExecutor(driver, url, username, password);
}
private void configureTableProperties(MutableContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConfig) {
private void configureTableProperties(MutableContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
String dataSql;
String schemaName = toJobConfig.toJobConfig.schemaName;

30
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcValidator.java
View File

@ -22,7 +22,7 @@
import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Validator;
import java.sql.DriverManager;
@ -34,30 +34,30 @@
public class GenericJdbcValidator extends Validator {
@Override
public Validation validateLink(Object configuration) {
Validation validation = new Validation(LinkConfiguration.class);
LinkConfiguration linkConf = (LinkConfiguration)configuration;
public ConfigValidator validateConfigForLink(Object configuration) {
ConfigValidator validation = new ConfigValidator(LinkConfiguration.class);
LinkConfiguration linkConfig = (LinkConfiguration)configuration;
if(linkConf.link.jdbcDriver == null) {
if(linkConfig.linkConfig.jdbcDriver == null) {
validation.addMessage(Status.UNACCEPTABLE, "link", "jdbcDriver", "Driver can't be empty");
} else {
try {
Class.forName(linkConf.link.jdbcDriver);
Class.forName(linkConfig.linkConfig.jdbcDriver);
} catch (ClassNotFoundException e) {
validation.addMessage(Status.UNACCEPTABLE, "link", "jdbcDriver", "Can't load specified driver");
}
}
if(linkConf.link.connectionString == null) {
if(linkConfig.linkConfig.connectionString == null) {
validation.addMessage(Status.UNACCEPTABLE, "link", "connectionString", "JDBC URL can't be empty");
} else if(!linkConf.link.connectionString.startsWith("jdbc:")) {
} else if(!linkConfig.linkConfig.connectionString.startsWith("jdbc:")) {
validation.addMessage(Status.UNACCEPTABLE, "link", "connectionString", "This do not seem as a valid JDBC URL");
}
// See if we can connect to the database
try {
DriverManager.getConnection(linkConf.link.connectionString,
linkConf.link.username, linkConf.link.password);
DriverManager.getConnection(linkConfig.linkConfig.connectionString,
linkConfig.linkConfig.username, linkConfig.linkConfig.password);
} catch (SQLException e) {
validation.addMessage(Status.ACCEPTABLE, "link", "Can't connect to the database with given credentials: " + e.getMessage());
}
@ -67,7 +67,7 @@ public Validation validateLink(Object configuration) {
}
@Override
public Validation validateJob(Object jobConfiguration) {
public ConfigValidator validateConfigForJob(Object jobConfiguration) {
if (jobConfiguration instanceof FromJobConfiguration) {
return validateFromJobConfiguration((FromJobConfiguration)jobConfiguration);
} else if (jobConfiguration instanceof ToJobConfiguration) {
@ -78,8 +78,8 @@ public Validation validateJob(Object jobConfiguration) {
}
}
private Validation validateToJobConfiguration(ToJobConfiguration configuration) {
Validation validation = new Validation(FromJobConfiguration.class);
private ConfigValidator validateToJobConfiguration(ToJobConfiguration configuration) {
ConfigValidator validation = new ConfigValidator(FromJobConfiguration.class);
if(configuration.toJobConfig.tableName == null && configuration.toJobConfig.sql == null) {
validation.addMessage(Status.UNACCEPTABLE, "toJobConfig", "Either table name or SQL must be specified");
@ -102,8 +102,8 @@ private Validation validateToJobConfiguration(ToJobConfiguration configuration)
return validation;
}
private Validation validateFromJobConfiguration(FromJobConfiguration configuration) {
Validation validation = new Validation(FromJobConfiguration.class);
private ConfigValidator validateFromJobConfiguration(FromJobConfiguration configuration) {
ConfigValidator validation = new ConfigValidator(FromJobConfiguration.class);
if(configuration.fromJobConfig.tableName == null && configuration.fromJobConfig.sql == null) {
validation.addMessage(Status.UNACCEPTABLE, "fromJobConfig", "Either table name or SQL must be specified");

14
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfig.java
View File

@ -18,7 +18,7 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.connector.jdbc.GenericJdbcConnectorConstants;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
@ -28,7 +28,7 @@
/**
*
*/
@FormClass( validators = {@Validator(FromJobConfig.FormValidator.class)})
@ConfigClass( validators = {@Validator(FromJobConfig.ConfigValidator.class)})
public class FromJobConfig {
@Input(size = 50)
public String schemaName;
@ -51,16 +51,16 @@ public class FromJobConfig {
@Input(size = 50)
public String boundaryQuery;
public static class FormValidator extends AbstractValidator<FromJobConfig> {
public static class ConfigValidator extends AbstractValidator<FromJobConfig> {
@Override
public void validate(FromJobConfig form) {
if(form.tableName == null && form.sql == null) {
public void validate(FromJobConfig config) {
if(config.tableName == null && config.sql == null) {
addMessage(Status.UNACCEPTABLE, "Either table name or SQL must be specified");
}
if(form.tableName != null && form.sql != null) {
if(config.tableName != null && config.sql != null) {
addMessage(Status.UNACCEPTABLE, "Both table name and SQL cannot be specified");
}
if(form.schemaName != null && form.sql != null) {
if(config.schemaName != null && config.sql != null) {
addMessage(Status.UNACCEPTABLE, "Both schema name and SQL cannot be specified");
}
}

4
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromJobConfiguration.java
View File

@ -18,14 +18,14 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.Config;
/**
*
*/
@ConfigurationClass
public class FromJobConfiguration {
@Form public FromJobConfig fromJobConfig;
@Config public FromJobConfig fromJobConfig;
public FromJobConfiguration() {
fromJobConfig = new FromJobConfig();

6
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfig.java
View File

@ -17,7 +17,7 @@
*/
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
@ -33,7 +33,7 @@
/**
*
*/
@FormClass(validators = {@Validator(LinkConfig.FormValidator.class)})
@ConfigClass(validators = {@Validator(LinkConfig.ConfigValidator.class)})
public class LinkConfig {
@Input(size = 128, validators = {@Validator(NotEmpty.class), @Validator(ClassAvailable.class)} )
public String jdbcDriver;
@ -50,7 +50,7 @@ public class LinkConfig {
@Input
public Map<String, String> jdbcProperties;
public static class FormValidator extends AbstractValidator<LinkConfig> {
public static class ConfigValidator extends AbstractValidator<LinkConfig> {
@Override
public void validate(LinkConfig linkConfig) {
// See if we can connect to the database

6
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/LinkConfiguration.java
View File

@ -18,7 +18,7 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.Config;
/**
*
@ -26,9 +26,9 @@
@ConfigurationClass
public class LinkConfiguration {
@Form public LinkConfig link;
@Config public LinkConfig linkConfig;
public LinkConfiguration() {
link = new LinkConfig();
linkConfig = new LinkConfig();
}
}

16
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
View File

@ -17,7 +17,7 @@
*/
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
import org.apache.sqoop.model.Validator;
import org.apache.sqoop.validation.Status;
@ -26,7 +26,7 @@
/**
*
*/
@FormClass(validators = {@Validator(ToJobConfig.FormValidator.class)})
@ConfigClass(validators = {@Validator(ToJobConfig.ConfigValidator.class)})
public class ToJobConfig {
@Input(size = 50) public String schemaName;
@Input(size = 2000) public String tableName;
@ -35,19 +35,19 @@ public class ToJobConfig {
@Input(size = 2000) public String stageTableName;
@Input public Boolean clearStageTable;
public static class FormValidator extends AbstractValidator<ToJobConfig> {
public static class ConfigValidator extends AbstractValidator<ToJobConfig> {
@Override
public void validate(ToJobConfig form) {
if(form.tableName == null && form.sql == null) {
public void validate(ToJobConfig config) {
if(config.tableName == null && config.sql == null) {
addMessage(Status.UNACCEPTABLE, "Either table name or SQL must be specified");
}
if(form.tableName != null && form.sql != null) {
if(config.tableName != null && config.sql != null) {
addMessage(Status.UNACCEPTABLE, "Both table name and SQL cannot be specified");
}
if(form.tableName == null && form.stageTableName != null) {
if(config.tableName == null && config.stageTableName != null) {
addMessage(Status.UNACCEPTABLE, "Stage table name cannot be specified without specifying table name");
}
if(form.stageTableName == null && form.clearStageTable != null) {
if(config.stageTableName == null && config.clearStageTable != null) {
addMessage(Status.UNACCEPTABLE, "Clear stage table cannot be specified without specifying name of the stage table.");
}
}

4
connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfiguration.java
View File

@ -18,14 +18,14 @@
package org.apache.sqoop.connector.jdbc.configuration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.Config;
/**
*
*/
@ConfigurationClass
public class ToJobConfiguration {
@Form public ToJobConfig toJobConfig;
@Config public ToJobConfig toJobConfig;
public ToJobConfiguration() {
toJobConfig = new ToJobConfig();

24
connector/connector-generic-jdbc/src/main/resources/generic-jdbc-connector-config.properties
View File

@ -18,33 +18,33 @@
############################
# Link Config
#
link.label = Link configuration
link.help = You must supply the information requested in order to \
linkConfig.label = Link configuration
linkConfig.help = You must supply the information requested in order to \
create a connection object.
# jdbc driver
link.jdbcDriver.label = JDBC Driver Class
link.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
linkConfig.jdbcDriver.label = JDBC Driver Class
linkConfig.jdbcDriver.help = Enter the fully qualified class name of the JDBC \
driver that will be used for establishing this connection.
# connect string
link.connectionString.label = JDBC Connection String
link.connectionString.help = Enter the value of JDBC connection string to be \
linkConfig.connectionString.label = JDBC Connection String
linkConfig.connectionString.help = Enter the value of JDBC connection string to be \
used by this connector for creating connections.
# username string
link.username.label = Username
link.username.help = Enter the username to be used for connecting to the \
linkConfig.username.label = Username
linkConfig.username.help = Enter the username to be used for connecting to the \
database.
# password string
link.password.label = Password
link.password.help = Enter the password to be used for connecting to the \
linkConfig.password.label = Password
linkConfig.password.help = Enter the password to be used for connecting to the \
database.
# jdbc properties
link.jdbcProperties.label = JDBC Connection Properties
link.jdbcProperties.help = Enter any JDBC properties that should be \
linkConfig.jdbcProperties.label = JDBC Connection Properties
linkConfig.jdbcProperties.help = Enter any JDBC properties that should be \
supplied during the creation of connection.
# From Job Config

3
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/GenericJdbcExecutorTest.java
View File

@ -60,7 +60,6 @@ public void setUp() {
}
@Test
@SuppressWarnings("unchecked")
public void testDeleteTableData() throws Exception {
executor.deleteTableData(table);
assertEquals("Table " + table + " is expected to be empty.",
@ -68,7 +67,6 @@ public void testDeleteTableData() throws Exception {
}
@Test
@SuppressWarnings("unchecked")
public void testMigrateData() throws Exception {
assertEquals("Table " + emptyTable + " is expected to be empty.",
0, executor.getTableRowCount(emptyTable));
@ -86,7 +84,6 @@ public void testMigrateData() throws Exception {
}
@Test
@SuppressWarnings("unchecked")
public void testGetTableRowCount() throws Exception {
assertEquals("Table " + table + " is expected to be empty.",
NUMBER_OF_ROWS, executor.getTableRowCount(table));

24
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java
View File

@ -74,10 +74,10 @@ public void tearDown() {
public void testQuery() throws Exception {
MutableContext context = new MutableMapContext();
LinkConfiguration connectionConfig = new LinkConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
@ -92,25 +92,25 @@ public void testQuery() throws Exception {
partition = new GenericJdbcPartition();
partition.setConditions("-50.0 <= DCOL AND DCOL < -16.6666666666666665");
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("-16.6666666666666665 <= DCOL AND DCOL < 16.666666666666667");
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("16.666666666666667 <= DCOL AND DCOL <= 50.0");
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
@Test
public void testSubquery() throws Exception {
MutableContext context = new MutableMapContext();
LinkConfiguration connectionConfig = new LinkConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
FromJobConfiguration jobConfig = new FromJobConfiguration();
@ -127,15 +127,15 @@ public void testSubquery() throws Exception {
partition = new GenericJdbcPartition();
partition.setConditions("-50 <= ICOL AND ICOL < -16");
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("-16 <= ICOL AND ICOL < 17");
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
partition = new GenericJdbcPartition();
partition.setConditions("17 <= ICOL AND ICOL < 50");
extractor.extract(extractorContext, connectionConfig, jobConfig, partition);
extractor.extract(extractorContext, linkConfig, jobConfig, partition);
}
public class DummyWriter extends DataWriter {

158
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java
View File

@ -116,19 +116,19 @@ public void tearDown() {
@Test
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.tableName = schemalessTableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.tableName = schemalessTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + executor.delimitIdentifier(schemalessTableName)
@ -143,20 +143,20 @@ public void testTableName() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.tableName = schemalessTableName;
jobConf.fromJobConfig.columns = tableColumns;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.tableName = schemalessTableName;
jobConfig.fromJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT ICOL,VCOL FROM " + executor.delimitIdentifier(schemalessTableName)
@ -171,20 +171,20 @@ public void testTableNameWithTableColumns() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.sql = schemalessTableSql;
jobConf.fromJobConfig.partitionColumn = "DCOL";
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.sql = schemalessTableSql;
jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + executor.delimitIdentifier(schemalessTableName)
@ -199,21 +199,21 @@ public void testTableSql() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumns() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.sql = schemalessTableSql;
jobConf.fromJobConfig.columns = tableColumns;
jobConf.fromJobConfig.partitionColumn = "DCOL";
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.sql = schemalessTableSql;
jobConfig.fromJobConfig.columns = tableColumns;
jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "
@ -229,22 +229,22 @@ public void testTableSqlWithTableColumns() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.schemaName = schemaName;
jobConf.fromJobConfig.tableName = tableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.schemaName = schemaName;
jobConfig.fromJobConfig.tableName = tableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + fullTableName
@ -259,23 +259,23 @@ public void testTableNameWithSchema() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.schemaName = schemaName;
jobConf.fromJobConfig.tableName = tableName;
jobConf.fromJobConfig.columns = tableColumns;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.schemaName = schemaName;
jobConfig.fromJobConfig.tableName = tableName;
jobConfig.fromJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT ICOL,VCOL FROM " + fullTableName
@ -290,23 +290,23 @@ public void testTableNameWithTableColumnsWithSchema() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.schemaName = schemaName;
jobConf.fromJobConfig.sql = tableSql;
jobConf.fromJobConfig.partitionColumn = "DCOL";
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.schemaName = schemaName;
jobConfig.fromJobConfig.sql = tableSql;
jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT * FROM " + fullTableName
@ -321,68 +321,68 @@ public void testTableSqlWithSchema() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testGetSchemaForTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.schemaName = schemaName;
jobConf.fromJobConfig.tableName = tableName;
jobConf.fromJobConfig.partitionColumn = "DCOL";
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.schemaName = schemaName;
jobConfig.fromJobConfig.tableName = tableName;
jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
Schema schema = initializer.getSchema(initializerContext, connConf, jobConf);
assertEquals(getSchema(jobConf.fromJobConfig.schemaName + "." + tableName), schema);
initializer.initialize(initializerContext, linkConfig, jobConfig);
Schema schema = initializer.getSchema(initializerContext, linkConfig, jobConfig);
assertEquals(getSchema(jobConfig.fromJobConfig.schemaName + "." + tableName), schema);
}
@Test
@SuppressWarnings("unchecked")
public void testGetSchemaForSql() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.schemaName = schemaName;
jobConf.fromJobConfig.sql = tableSql;
jobConf.fromJobConfig.partitionColumn = "DCOL";
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.schemaName = schemaName;
jobConfig.fromJobConfig.sql = tableSql;
jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
Schema schema = initializer.getSchema(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
Schema schema = initializer.getSchema(initializerContext, linkConfig, jobConfig);
assertEquals(getSchema("Query"), schema);
}
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithTableColumnsWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.fromJobConfig.schemaName = schemaName;
jobConf.fromJobConfig.sql = tableSql;
jobConf.fromJobConfig.columns = tableColumns;
jobConf.fromJobConfig.partitionColumn = "DCOL";
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.fromJobConfig.schemaName = schemaName;
jobConfig.fromJobConfig.sql = tableSql;
jobConfig.fromJobConfig.columns = tableColumns;
jobConfig.fromJobConfig.partitionColumn = "DCOL";
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcFromInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context,
"SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "

8
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestLoader.java
View File

@ -82,10 +82,10 @@ public void tearDown() {
public void testInsert() throws Exception {
MutableContext context = new MutableMapContext();
LinkConfiguration connectionConfig = new LinkConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
connectionConfig.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connectionConfig.link.connectionString = GenericJdbcTestConstants.URL;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
ToJobConfiguration jobConfig = new ToJobConfiguration();
@ -95,7 +95,7 @@ public void testInsert() throws Exception {
Loader loader = new GenericJdbcLoader();
DummyReader reader = new DummyReader();
LoaderContext loaderContext = new LoaderContext(context, reader, null);
loader.load(loaderContext, connectionConfig, jobConfig);
loader.load(loaderContext, linkConfig, jobConfig);
int index = START;
ResultSet rs = executor.executeQuery("SELECT * FROM "

98
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestPartitioner.java
View File

@ -58,12 +58,12 @@ public void testIntegerEvenPartition() throws Exception {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -3",
@ -90,12 +90,12 @@ public void testIntegerUnevenPartition() throws Exception {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -1",
@ -120,12 +120,12 @@ public void testIntegerOverPartition() throws Exception {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf(START + NUMBER_OF_ROWS - 1));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 13, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -4",
@ -157,12 +157,12 @@ public void testFloatingPointEvenPartition() throws Exception {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5.0 <= DCOL AND DCOL < -3.0",
@ -189,12 +189,12 @@ public void testFloatingPointUnevenPartition() throws Exception {
GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
String.valueOf((double)(START + NUMBER_OF_ROWS - 1)));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5.0 <= DCOL AND DCOL < -1.6666666666666665",
@ -211,12 +211,12 @@ public void testNumericEvenPartition() throws Exception {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(START));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(START + NUMBER_OF_ROWS - 1));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"-5 <= ICOL AND ICOL < -3",
@ -235,12 +235,12 @@ public void testNumericUnevenPartition() throws Exception {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(new BigDecimal(START)));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START + NUMBER_OF_ROWS - 1)));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"-5 <= DCOL AND DCOL < -2",
@ -257,12 +257,12 @@ public void testNumericSinglePartition() throws Exception {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE, String.valueOf(new BigDecimal(START)));
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE, String.valueOf(new BigDecimal(START)));
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"DCOL = -5",
@ -282,12 +282,12 @@ public void testDatePartition() throws Exception {
.toString());
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
@ -311,12 +311,12 @@ public void testTimePartition() throws Exception {
Time.valueOf("10:40:50").toString());
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"'01:01:01' <= TCOL AND TCOL < '04:14:17'",
@ -337,12 +337,12 @@ public void testTimestampPartition() throws Exception {
context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE,
Timestamp.valueOf("2013-12-31 10:40:50.654").toString());
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 12:14:17.634'",
"'2013-05-02 12:14:17.634' <= TSCOL AND TSCOL < '2013-08-31 23:27:34.144'",
@ -362,12 +362,12 @@ public void testBooleanPartition() throws Exception {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "1");
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 3, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[]{
"BCOL = TRUE",
"BCOL = FALSE",
@ -386,12 +386,12 @@ public void testVarcharPartition() throws Exception {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "Z");
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 25, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"'A' <= VCCOL AND VCCOL < 'B'",
@ -434,11 +434,11 @@ public void testVarcharPartition2() throws Exception {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "Warty Warthog");
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
assertEquals(partitions.size(), 5);
// First partition needs to contain entire upper bound
assertTrue(partitions.get(0).toString().contains("Breezy Badger"));
@ -458,13 +458,13 @@ public void testVarcharPartitionWithCommonPrefix() throws Exception {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "AAF");
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"'AAA' <= VCCOL AND VCCOL < 'AAB'",
@ -488,14 +488,14 @@ public void testPatitionWithNullValues() throws Exception {
context.setString(GenericJdbcConnectorConstants
.CONNECTOR_JDBC_PARTITION_MAXVALUE, "AAE");
LinkConfiguration connConf = new LinkConfiguration();
FromJobConfiguration jobConf = new FromJobConfiguration();
jobConf.fromJobConfig.partitionColumnNull = true;
LinkConfiguration linkConfig = new LinkConfiguration();
FromJobConfiguration jobConfig = new FromJobConfiguration();
jobConfig.fromJobConfig.partitionColumnNull = true;
Partitioner partitioner = new GenericJdbcPartitioner();
PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf, jobConf);
List<Partition> partitions = partitioner.getPartitions(partitionerContext, linkConfig, jobConfig);
verifyResult(partitions, new String[] {
"VCCOL IS NULL",

188
connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
View File

@ -17,6 +17,10 @@
*/
package org.apache.sqoop.connector.jdbc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.apache.sqoop.common.MutableContext;
import org.apache.sqoop.common.MutableMapContext;
import org.apache.sqoop.common.SqoopException;
@ -24,17 +28,13 @@
import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
import org.apache.sqoop.job.etl.Initializer;
import org.apache.sqoop.job.etl.InitializerContext;
import org.apache.sqoop.validation.ConfigValidationResult;
import org.apache.sqoop.validation.ConfigValidationRunner;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.ValidationResult;
import org.apache.sqoop.validation.ValidationRunner;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestToInitializer {
private final String schemaName;
private final String tableName;
@ -82,21 +82,21 @@ public void tearDown() {
@Test
@SuppressWarnings("unchecked")
public void testTableName() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemalessTableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.tableName = schemalessTableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.tableName = schemalessTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " VALUES (?,?,?)");
}
@ -104,22 +104,22 @@ public void testTableName() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumns() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemalessTableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.tableName = schemalessTableName;
jobConf.toJobConfig.columns = tableColumns;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.tableName = schemalessTableName;
jobConfig.toJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " (" + tableColumns + ") VALUES (?,?)");
}
@ -127,19 +127,19 @@ public void testTableNameWithTableColumns() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableSql() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.sql = schemalessTableSql;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.sql = schemalessTableSql;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(schemalessTableName) + " VALUES (?,?,?)");
}
@ -147,22 +147,22 @@ public void testTableSql() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.schemaName = schemaName;
jobConf.toJobConfig.tableName = tableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.schemaName = schemaName;
jobConfig.toJobConfig.tableName = tableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " VALUES (?,?,?)");
}
@ -170,23 +170,23 @@ public void testTableNameWithSchema() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableNameWithTableColumnsWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullTableName = executor.delimitIdentifier(schemaName) + "." + executor.delimitIdentifier(tableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.schemaName = schemaName;
jobConf.toJobConfig.tableName = tableName;
jobConf.toJobConfig.columns = tableColumns;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.schemaName = schemaName;
jobConfig.toJobConfig.tableName = tableName;
jobConfig.toJobConfig.columns = tableColumns;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullTableName + " (" + tableColumns + ") VALUES (?,?)");
}
@ -194,20 +194,20 @@ public void testTableNameWithTableColumnsWithSchema() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testTableSqlWithSchema() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.schemaName = schemaName;
jobConf.toJobConfig.sql = tableSql;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.schemaName = schemaName;
jobConfig.toJobConfig.sql = tableSql;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + executor.delimitIdentifier(tableName) + " VALUES (?,?,?)");
}
@ -229,13 +229,13 @@ private void createTable(String tableName) {
@Test
public void testNonExistingStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.tableName = schemalessTableName;
jobConf.toJobConfig.stageTableName = stageTableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.tableName = schemalessTableName;
jobConfig.toJobConfig.stageTableName = stageTableName;
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@ -243,7 +243,7 @@ public void testNonExistingStageTable() throws Exception {
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
try {
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
fail("Initialization should fail for non-existing stage table.");
} catch(SqoopException se) {
//expected
@ -253,15 +253,15 @@ public void testNonExistingStageTable() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testNonEmptyStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.tableName = schemalessTableName;
jobConf.toJobConfig.stageTableName = stageTableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.tableName = schemalessTableName;
jobConfig.toJobConfig.stageTableName = stageTableName;
createTable(fullStageTableName);
executor.executeUpdate("INSERT INTO " + fullStageTableName +
" VALUES(1, 1.1, 'one')");
@ -271,7 +271,7 @@ public void testNonEmptyStageTable() throws Exception {
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
try {
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
fail("Initialization should fail for non-empty stage table.");
} catch(SqoopException se) {
//expected
@ -280,17 +280,17 @@ public void testNonEmptyStageTable() throws Exception {
@Test
public void testClearStageTableValidation() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
//specifying clear stage table flag without specifying name of
// the stage table
jobConf.toJobConfig.tableName = schemalessTableName;
jobConf.toJobConfig.clearStageTable = false;
ValidationRunner validationRunner = new ValidationRunner();
ValidationResult result = validationRunner.validate(jobConf);
jobConfig.toJobConfig.tableName = schemalessTableName;
jobConfig.toJobConfig.clearStageTable = false;
ConfigValidationRunner validationRunner = new ConfigValidationRunner();
ConfigValidationResult result = validationRunner.validate(jobConfig);
assertEquals("User should not specify clear stage table flag without " +
"specifying name of the stage table",
Status.UNACCEPTABLE,
@ -298,8 +298,8 @@ public void testClearStageTableValidation() throws Exception {
assertTrue(result.getMessages().containsKey(
"toJobConfig"));
jobConf.toJobConfig.clearStageTable = true;
result = validationRunner.validate(jobConf);
jobConfig.toJobConfig.clearStageTable = true;
result = validationRunner.validate(jobConfig);
assertEquals("User should not specify clear stage table flag without " +
"specifying name of the stage table",
Status.UNACCEPTABLE,
@ -310,17 +310,17 @@ public void testClearStageTableValidation() throws Exception {
@Test
public void testStageTableWithoutTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
//specifying stage table without specifying table name
jobConf.toJobConfig.stageTableName = stageTableName;
jobConf.toJobConfig.sql = "";
jobConfig.toJobConfig.stageTableName = stageTableName;
jobConfig.toJobConfig.sql = "";
ValidationRunner validationRunner = new ValidationRunner();
ValidationResult result = validationRunner.validate(jobConf);
ConfigValidationRunner validationRunner = new ConfigValidationRunner();
ConfigValidationResult result = validationRunner.validate(jobConfig);
assertEquals("Stage table name cannot be specified without specifying " +
"table name", Status.UNACCEPTABLE, result.getStatus());
assertTrue(result.getMessages().containsKey(
@ -330,16 +330,16 @@ public void testStageTableWithoutTable() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testClearStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.tableName = schemalessTableName;
jobConf.toJobConfig.stageTableName = stageTableName;
jobConf.toJobConfig.clearStageTable = true;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.tableName = schemalessTableName;
jobConfig.toJobConfig.stageTableName = stageTableName;
jobConfig.toJobConfig.clearStageTable = true;
createTable(fullStageTableName);
executor.executeUpdate("INSERT INTO " + fullStageTableName +
" VALUES(1, 1.1, 'one')");
@ -348,7 +348,7 @@ public void testClearStageTable() throws Exception {
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
assertEquals("Stage table should have been cleared", 0,
executor.getTableRowCount(stageTableName));
}
@ -356,22 +356,22 @@ public void testClearStageTable() throws Exception {
@Test
@SuppressWarnings("unchecked")
public void testStageTable() throws Exception {
LinkConfiguration connConf = new LinkConfiguration();
ToJobConfiguration jobConf = new ToJobConfiguration();
LinkConfiguration linkConfig = new LinkConfiguration();
ToJobConfiguration jobConfig = new ToJobConfiguration();
String fullStageTableName = executor.delimitIdentifier(stageTableName);
connConf.link.jdbcDriver = GenericJdbcTestConstants.DRIVER;
connConf.link.connectionString = GenericJdbcTestConstants.URL;
jobConf.toJobConfig.tableName = schemalessTableName;
jobConf.toJobConfig.stageTableName = stageTableName;
linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
jobConfig.toJobConfig.tableName = schemalessTableName;
jobConfig.toJobConfig.stageTableName = stageTableName;
createTable(fullStageTableName);
MutableContext context = new MutableMapContext();
InitializerContext initializerContext = new InitializerContext(context);
@SuppressWarnings("rawtypes")
Initializer initializer = new GenericJdbcToInitializer();
initializer.initialize(initializerContext, connConf, jobConf);
initializer.initialize(initializerContext, linkConfig, jobConfig);
verifyResult(context, "INSERT INTO " + fullStageTableName +
" VALUES (?,?,?)");

52
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConfigUpgrader.java
View File

@ -18,21 +18,20 @@
*/
package org.apache.sqoop.connector.hdfs;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.model.MConnectionForms;
import org.apache.sqoop.model.MForm;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MJobForms;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.RepositoryUpgrader;
import org.apache.sqoop.model.MConfigList;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MLinkConfig;
public class HdfsConfigUpgrader extends RepositoryUpgrader {
private static final Logger LOG =
Logger.getLogger(HdfsConfigUpgrader.class);
private static final Logger LOG = Logger.getLogger(HdfsConfigUpgrader.class);
/*
* For now, there is no real upgrade. So copy all data over,
@ -41,37 +40,36 @@ public class HdfsConfigUpgrader extends RepositoryUpgrader {
*/
@Override
public void upgrade(MConnectionForms original,
MConnectionForms upgradeTarget) {
doUpgrade(original.getForms(), upgradeTarget.getForms());
public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@Override
public void upgrade(MJobForms original, MJobForms upgradeTarget) {
doUpgrade(original.getForms(), upgradeTarget.getForms());
public void upgrade(MConfigList original, MConfigList upgradeTarget) {
doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
}
@SuppressWarnings("unchecked")
private void doUpgrade(List<MForm> original, List<MForm> target) {
// Easier to find the form in the original forms list if we use a map.
// Since the constructor of MJobForms takes a list,
private void doUpgrade(List<MConfig> original, List<MConfig> target) {
// Easier to find the config in the original list if we use a map.
// Since the constructor takes a list,
// index is not guaranteed to be the same, so we need to look for
// equivalence
Map<String, MForm> formMap = new HashMap<String, MForm>();
for (MForm form : original) {
formMap.put(form.getName(), form);
Map<String, MConfig> configMap = new HashMap<String, MConfig>();
for (MConfig config : original) {
configMap.put(config.getName(), config);
}
for (MForm form : target) {
List<MInput<?>> inputs = form.getInputs();
MForm originalForm = formMap.get(form.getName());
if (originalForm == null) {
LOG.warn("Form: '" + form.getName() + "' not present in old " +
for (MConfig config : target) {
List<MInput<?>> inputs = config.getInputs();
MConfig originalConfig = configMap.get(config.getName());
if (originalConfig == null) {
LOG.warn("Config: '" + config.getName() + "' not present in old " +
"connector. So it and its inputs will not be transferred by the upgrader.");
continue;
}
for (MInput input : inputs) {
try {
MInput originalInput = originalForm.getInput(input.getName());
MInput originalInput = originalConfig.getInput(input.getName());
input.setValue(originalInput.getValue());
} catch (SqoopException ex) {
LOG.warn("Input: '" + input.getName() + "' not present in old " +

2
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
View File

@ -118,7 +118,7 @@ public To getTo() {
* @return Validator object
*/
@Override
public Validator getValidator() {
public Validator getConfigValidator() {
return hdfsValidator;
}

4
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsExtractor.java
View File

@ -52,8 +52,8 @@ public class HdfsExtractor extends Extractor<LinkConfiguration, FromJobConfigura
@Override
public void extract(ExtractorContext context,
LinkConfiguration connectionConfiguration,
FromJobConfiguration jobConfiguration, HdfsPartition partition) {
LinkConfiguration linkConfig,
FromJobConfiguration fromJobConfig, HdfsPartition partition) {
conf = ((PrefixContext) context.getContext()).getConfiguration();
dataWriter = context.getDataWriter();

6
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsInitializer.java
View File

@ -29,16 +29,16 @@ public class HdfsInitializer extends Initializer {
* promoted to all other part of the workflow automatically.
*
* @param context Initializer context object
* @param linkConf Connector's link configuration object
* @param linkConfig Connector's link configuration object
* @param jobConf Connector's job configuration object
*/
@Override
public void initialize(InitializerContext context, Object linkConf, Object jobConf) {
public void initialize(InitializerContext context, Object linkConfig, Object jobConf) {
}
@Override
public Schema getSchema(InitializerContext context, Object linkConf, Object jobConf) {
public Schema getSchema(InitializerContext context, Object linkConfig, Object jobConfig) {
return new Schema("HDFS file");
}
}

14
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java
View File

@ -42,19 +42,19 @@ public class HdfsLoader extends Loader<LinkConfiguration, ToJobConfiguration> {
* Load data to target.
*
* @param context Loader context object
* @param linkConf Link configuration
* @param toJobConf Job configuration
* @param linkConfig Link configuration
* @param toJobConfig Job configuration
* @throws Exception
*/
@Override
public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfiguration toJobConf) throws Exception {
public void load(LoaderContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) throws Exception {
DataReader reader = context.getDataReader();
Configuration conf = ((PrefixContext)context.getContext()).getConfiguration();
String directoryName = toJobConf.toJobConfig.outputDirectory;
String codecname = getCompressionCodecName(toJobConf);
String directoryName = toJobConfig.toJobConfig.outputDirectory;
String codecname = getCompressionCodecName(toJobConfig);
CompressionCodec codec = null;
if (codecname != null) {
@ -73,12 +73,12 @@ public void load(LoaderContext context, LinkConfiguration linkConf, ToJobConfigu
}
}
String filename = directoryName + "/" + UUID.randomUUID() + getExtension(toJobConf,codec);
String filename = directoryName + "/" + UUID.randomUUID() + getExtension(toJobConfig,codec);
try {
Path filepath = new Path(filename);
GenericHdfsWriter filewriter = getWriter(toJobConf);
GenericHdfsWriter filewriter = getWriter(toJobConfig);
filewriter.initialize(filepath,conf,codec);

6
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsPartitioner.java
View File

@ -68,12 +68,12 @@ public class HdfsPartitioner extends Partitioner<LinkConfiguration, FromJobConfi
@Override
public List<Partition> getPartitions(PartitionerContext context,
LinkConfiguration linkConfiguration, FromJobConfiguration jobConfiguration) {
LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
Configuration conf = ((PrefixContext)context.getContext()).getConfiguration();
try {
long numInputBytes = getInputSize(conf, jobConfiguration.fromJobConfig.inputDirectory);
long numInputBytes = getInputSize(conf, fromJobConfig.fromJobConfig.inputDirectory);
maxSplitSize = numInputBytes / context.getMaxPartitions();
if(numInputBytes % context.getMaxPartitions() != 0 ) {
@ -118,7 +118,7 @@ public List<Partition> getPartitions(PartitionerContext context,
}
// all the files in input set
String indir = jobConfiguration.fromJobConfig.inputDirectory;
String indir = fromJobConfig.fromJobConfig.inputDirectory;
FileSystem fs = FileSystem.get(conf);
List<Path> paths = new LinkedList<Path>();

49
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsValidator.java
View File

@ -19,7 +19,7 @@
import org.apache.sqoop.connector.hdfs.configuration.*;
import org.apache.sqoop.validation.Status;
import org.apache.sqoop.validation.Validation;
import org.apache.sqoop.validation.ConfigValidator;
import org.apache.sqoop.validation.Validator;
/**
@ -28,54 +28,45 @@
public class HdfsValidator extends Validator {
@Override
public Validation validateLink(Object connectionConfiguration) {
Validation validation = new Validation(LinkConfiguration.class);
// No validation on connection object
return validation;
}
@Override
public Validation validateJob(Object jobConfiguration) {
//TODO: I'm pretty sure this needs to call either validateExportJob or validateImportJob, depending on context
return super.validateJob(jobConfiguration);
public ConfigValidator validateConfigForJob(Object jobConfiguration) {
return super.validateConfigForJob(jobConfiguration);
}
@SuppressWarnings("unused")
private Validation validateFromJob(Object jobConfiguration) {
Validation validation = new Validation(FromJobConfiguration.class);
private ConfigValidator validateFromJob(Object jobConfiguration) {
ConfigValidator validation = new ConfigValidator(FromJobConfiguration.class);
FromJobConfiguration configuration = (FromJobConfiguration)jobConfiguration;
validateInputForm(validation, configuration.fromJobConfig);
validateInputConfig(validation, configuration.fromJobConfig);
return validation;
}
@SuppressWarnings("unused")
private Validation validateToJob(Object jobConfiguration) {
Validation validation = new Validation(ToJobConfiguration.class);
private ConfigValidator validateToJob(Object jobConfiguration) {
ConfigValidator validation = new ConfigValidator(ToJobConfiguration.class);
ToJobConfiguration configuration = (ToJobConfiguration)jobConfiguration;
validateOutputForm(validation, configuration.toJobConfig);
validateOutputConfig(validation, configuration.toJobConfig);
return validation;
}
private void validateInputForm(Validation validation, FromJobConfig input) {
if(input.inputDirectory == null || input.inputDirectory.isEmpty()) {
private void validateInputConfig(ConfigValidator validation, FromJobConfig inputConfig) {
if(inputConfig.inputDirectory == null || inputConfig.inputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "input", "inputDirectory", "Input directory is empty");
}
}
private void validateOutputForm(Validation validation, ToJobConfig output) {
if(output.outputDirectory == null || output.outputDirectory.isEmpty()) {
private void validateOutputConfig(ConfigValidator validation, ToJobConfig outputConfig) {
if(outputConfig.outputDirectory == null || outputConfig.outputDirectory.isEmpty()) {
validation.addMessage(Status.UNACCEPTABLE, "output", "outputDirectory", "Output directory is empty");
}
if(output.customCompression != null &&
output.customCompression.trim().length() > 0 &&
output.compression != ToCompression.CUSTOM) {
if(outputConfig.customCompression != null &&
outputConfig.customCompression.trim().length() > 0 &&
outputConfig.compression != ToCompression.CUSTOM) {
validation.addMessage(Status.UNACCEPTABLE, "output", "compression",
"custom compression should be blank as " + output.compression + " is being used.");
"custom compression should be blank as " + outputConfig.compression + " is being used.");
}
if(output.compression == ToCompression.CUSTOM &&
(output.customCompression == null ||
output.customCompression.trim().length() == 0)
if(outputConfig.compression == ToCompression.CUSTOM &&
(outputConfig.customCompression == null ||
outputConfig.customCompression.trim().length() == 0)
) {
validation.addMessage(Status.UNACCEPTABLE, "output", "compression",
"custom compression is blank.");

4
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfig.java
View File

@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
/**
*
*/
@FormClass
@ConfigClass
public class FromJobConfig {
@Input(size = 255) public String inputDirectory;

4
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/FromJobConfiguration.java
View File

@ -18,11 +18,11 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.Config;
@ConfigurationClass
public class FromJobConfiguration {
@Form public FromJobConfig fromJobConfig;
@Config public FromJobConfig fromJobConfig;
public FromJobConfiguration() {
fromJobConfig = new FromJobConfig();

6
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
View File

@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
@FormClass
@ConfigClass
public class LinkConfig {
//Todo: Didn't find anything that belongs here...
// Since empty forms don't work (DERBYREPO_0008:The form contains no input metadata), I'm putting a dummy form here
// Since empty forms don't work (DERBYREPO_0008:The config contains no input metadata), I'm putting a dummy config here
@Input(size = 255) public String dummy;
}

8
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java
View File

@ -18,14 +18,14 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.Config;
@ConfigurationClass
public class LinkConfiguration {
@Form
public LinkConfig link;
@Config
public LinkConfig linkConfig;
public LinkConfiguration() {
link = new LinkConfig();
linkConfig = new LinkConfig();
}
}

4
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java
View File

@ -17,13 +17,13 @@
*/
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.FormClass;
import org.apache.sqoop.model.ConfigClass;
import org.apache.sqoop.model.Input;
/**
*
*/
@FormClass
@ConfigClass
public class ToJobConfig {
@Input public ToFormat outputFormat;

4
connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java
View File

@ -18,11 +18,11 @@
package org.apache.sqoop.connector.hdfs.configuration;
import org.apache.sqoop.model.ConfigurationClass;
import org.apache.sqoop.model.Form;
import org.apache.sqoop.model.Config;
@ConfigurationClass
public class ToJobConfiguration {
@Form
@Config
public ToJobConfig toJobConfig;
public ToJobConfiguration() {

8
connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
View File

@ -18,12 +18,12 @@
############################
# Link Config
#
link.label = Link configuration
link.help = You must supply the information requested in order to \
linkConfig.label = Link configuration
linkConfig.help = You must supply the information requested in order to \
create a connection object.
link.dummy.label = Dummy parameter needed to get HDFS connector to register
link.dummy.help = You can write anything here. Doesn't matter.
linkConfig.dummy.label = Dummy parameter needed to get HDFS connector to register
linkConfig.dummy.help = You can write anything here. Doesn't matter.
# To Job Config
#

14
connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java
View File

@ -18,6 +18,13 @@
*/
package org.apache.sqoop.connector.idf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.schema.Schema;
import org.apache.sqoop.schema.type.Binary;
@ -26,13 +33,6 @@
import org.junit.Before;
import org.junit.Test;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class TestCSVIntermediateDataFormat {
private final String BYTE_FIELD_ENCODING = "ISO-8859-1";

Some files were not shown because too many files have changed in this diff Show More