github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-07 20:03:16 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1687: Sqoop2: Single resource in JSON should not be a list

Browse Source 
(Veena Basavaraj via Abraham Elmahrek)
This commit is contained in:
Abraham Elmahrek 2014-11-06 13:46:27 -08:00
parent 8aa4648130
commit ec3316b100
16 changed files with 771 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
View File

@ -72,9 +72,8 @@ public Map<Long, ResourceBundle> getResourceBundles() {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
public JSONObject extract(boolean skipSensitive) { public JSONObject extract(boolean skipSensitive) {
JSONArray connectorArray = extractConnectors(skipSensitive);
JSONObject connector = new JSONObject(); JSONObject connector = new JSONObject();
connector.put(CONNECTOR, connectorArray); connector.put(CONNECTOR, extractConnector(skipSensitive, connectors.get(0)));
return connector; return connector;
} }
@ -82,91 +81,100 @@ public JSONObject extract(boolean skipSensitive) {
protected JSONArray extractConnectors(boolean skipSensitive) { protected JSONArray extractConnectors(boolean skipSensitive) {
JSONArray connectorArray = new JSONArray(); JSONArray connectorArray = new JSONArray();
for (MConnector connector : connectors) { for (MConnector connector : connectors) {
JSONObject connectorJsonObject = new JSONObject(); connectorArray.add(extractConnector(skipSensitive, connector));
connectorJsonObject.put(ID, connector.getPersistenceId());
connectorJsonObject.put(NAME, connector.getUniqueName());
connectorJsonObject.put(CLASS, connector.getClassName());
connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion());
connectorJsonObject.put(
CONNECTOR_LINK_CONFIG,
extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig()
.getType(), skipSensitive));
connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
// add sub fields to the job config for from and to
if (connector.getFromConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.FROM,
extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig()
.getType(), skipSensitive));
}
if (connector.getToConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.TO,
extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig()
.getType(), skipSensitive));
}
// add the config-param inside each connector
connectorJsonObject.put(ALL_CONFIG_RESOURCES, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIG_RESOURCES,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId())));
}
connectorArray.add(connectorJsonObject);
} }
return connectorArray; return connectorArray;
} }
@SuppressWarnings("unchecked")
private JSONObject extractConnector(boolean skipSensitive, MConnector connector) {
JSONObject connectorJsonObject = new JSONObject();
connectorJsonObject.put(ID, connector.getPersistenceId());
connectorJsonObject.put(NAME, connector.getUniqueName());
connectorJsonObject.put(CLASS, connector.getClassName());
connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion());
connectorJsonObject.put(
CONNECTOR_LINK_CONFIG,
extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig()
.getType(), skipSensitive));
connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
// add sub fields to the job config for from and to
if (connector.getFromConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.FROM,
extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig()
.getType(), skipSensitive));
}
if (connector.getToConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.TO,
extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig()
.getType(), skipSensitive));
}
// add the config-param inside each connector
connectorJsonObject.put(ALL_CONFIGS, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIGS,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId())));
}
return connectorJsonObject;
}
@Override @Override
public void restore(JSONObject jsonObject) { public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(CONNECTOR); connectors = new ArrayList<MConnector>();
restoreConnectors(array); connectorConfigBundles = new HashMap<Long, ResourceBundle>();
JSONObject obj = (JSONObject) jsonObject.get(CONNECTOR);
connectors.add(restoreConnector(obj));
} }
protected void restoreConnectors(JSONArray array) { protected void restoreConnectors(JSONArray array) {
connectors = new ArrayList<MConnector>(); connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>(); connectorConfigBundles = new HashMap<Long, ResourceBundle>();
for (Object obj : array) { for (Object obj : array) {
JSONObject object = (JSONObject) obj; connectors.add(restoreConnector(obj));
long connectorId = (Long) object.get(ID);
String uniqueName = (String) object.get(NAME);
String className = (String) object.get(CLASS);
String version = (String) object.get(CONFIGURABLE_VERSION);
List<MConfig> linkConfigs = restoreConfigList((JSONArray) object
.get(CONNECTOR_LINK_CONFIG));
// parent that encapsulates both the from/to configs
JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (fromJobConfigJson != null) {
List<MConfig> fromJobConfig = restoreConfigList(fromJobConfigJson);
fromConfig = new MFromConfig(fromJobConfig);
}
if (toJobConfigJson != null) {
List<MConfig> toJobConfig = restoreConfigList(toJobConfigJson);
toConfig = new MToConfig(toJobConfig);
}
MLinkConfig linkConfig = new MLinkConfig(linkConfigs);
MConnector connector = new MConnector(uniqueName, className, version, linkConfig, fromConfig,
toConfig);
connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIG_RESOURCES)) {
JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIG_RESOURCES);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle));
}
connectors.add(connector);
} }
} }
private MConnector restoreConnector(Object obj) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(ID);
String uniqueName = (String) object.get(NAME);
String className = (String) object.get(CLASS);
String version = (String) object.get(CONFIGURABLE_VERSION);
List<MConfig> linkConfigs = restoreConfigList((JSONArray) object
.get(CONNECTOR_LINK_CONFIG));
// parent that encapsulates both the from/to configs
JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (fromJobConfigJson != null) {
List<MConfig> fromJobConfig = restoreConfigList(fromJobConfigJson);
fromConfig = new MFromConfig(fromJobConfig);
}
if (toJobConfigJson != null) {
List<MConfig> toJobConfig = restoreConfigList(toJobConfigJson);
toConfig = new MToConfig(toJobConfig);
}
MLinkConfig linkConfig = new MLinkConfig(linkConfigs);
MConnector connector = new MConnector(uniqueName, className, version, linkConfig, fromConfig,
toConfig);
connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIGS)) {
JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIGS);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle));
}
return connector;
}
} }

4
common/src/main/java/org/apache/sqoop/json/DriverBean.java
View File

@ -69,7 +69,7 @@ public JSONObject extract(boolean skipSensitive) {
result.put(ID, driver.getPersistenceId()); result.put(ID, driver.getPersistenceId());
result.put(CONFIGURABLE_VERSION, driver.getVersion()); result.put(CONFIGURABLE_VERSION, driver.getVersion());
result.put(DRIVER_JOB_CONFIG, configs); result.put(DRIVER_JOB_CONFIG, configs);
result.put(ALL_CONFIG_RESOURCES, extractConfigParamBundle(driverConfigBundle)); result.put(ALL_CONFIGS, extractConfigParamBundle(driverConfigBundle));
return result; return result;
} }
@ -80,6 +80,6 @@ public void restore(JSONObject jsonObject) {
List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_JOB_CONFIG)); List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_JOB_CONFIG));
driver = new MDriver(new MDriverConfig(driverConfig), driverVersion); driver = new MDriver(new MDriverConfig(driverConfig), driverVersion);
driver.setPersistenceId(id); driver.setPersistenceId(id);
driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIG_RESOURCES)); driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIGS));
} }
} }

143
common/src/main/java/org/apache/sqoop/json/JobBean.java
View File

@ -101,9 +101,8 @@ public ResourceBundle getDriverConfigBundle() {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) { public JSONObject extract(boolean skipSensitive) {
JSONArray jobArray = extractJobs(skipSensitive);
JSONObject job = new JSONObject(); JSONObject job = new JSONObject();
job.put(JOB, jobArray); job.put(JOB, extractJob(skipSensitive, jobs.get(0)));
return job; return job;
} }
@ -111,83 +110,89 @@ public JSONObject extract(boolean skipSensitive) {
protected JSONArray extractJobs(boolean skipSensitive) { protected JSONArray extractJobs(boolean skipSensitive) {
JSONArray jobArray = new JSONArray(); JSONArray jobArray = new JSONArray();
for (MJob job : jobs) { for (MJob job : jobs) {
JSONObject object = new JSONObject(); jobArray.add(extractJob(skipSensitive, job));
object.put(ID, job.getPersistenceId());
object.put(NAME, job.getName());
object.put(ENABLED, job.getEnabled());
object.put(CREATION_USER, job.getCreationUser());
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
// job link associated connectors
// TODO(SQOOP-1634): fix not to require the connectorIds in the post data
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
// job associated links
object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job configs
MFromConfig fromConfigList = job.getFromJobConfig();
object.put(FROM_CONFIG_VALUES,
extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive));
MToConfig toConfigList = job.getToJobConfig();
object.put(TO_CONFIG_VALUES,
extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive));
MDriverConfig driverConfigList = job.getDriverConfig();
object.put(
DRIVER_CONFIG_VALUES,
extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(),
skipSensitive));
jobArray.add(object);
} }
return jobArray; return jobArray;
} }
@SuppressWarnings("unchecked")
private JSONObject extractJob(boolean skipSensitive, MJob job) {
JSONObject object = new JSONObject();
object.put(ID, job.getPersistenceId());
object.put(NAME, job.getName());
object.put(ENABLED, job.getEnabled());
object.put(CREATION_USER, job.getCreationUser());
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
// job link associated connectors
// TODO(SQOOP-1634): fix not to require the connectorIds in the post data
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
// job associated links
object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job configs
MFromConfig fromConfigList = job.getFromJobConfig();
object.put(FROM_CONFIG_VALUES,
extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive));
MToConfig toConfigList = job.getToJobConfig();
object.put(TO_CONFIG_VALUES,
extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive));
MDriverConfig driverConfigList = job.getDriverConfig();
object.put(
DRIVER_CONFIG_VALUES,
extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(),
skipSensitive));
return object;
}
@Override @Override
public void restore(JSONObject jsonObject) { public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(JOB); jobs = new ArrayList<MJob>();
restoreJobs(array); JSONObject obj = (JSONObject) jsonObject.get(JOB);
jobs.add(restoreJob(obj));
} }
protected void restoreJobs(JSONArray array) { protected void restoreJobs(JSONArray array) {
jobs = new ArrayList<MJob>(); jobs = new ArrayList<MJob>();
for (Object obj : array) { for (Object obj : array) {
JSONObject object = (JSONObject) obj; jobs.add(restoreJob(obj));
long fromConnectorId = (Long) object.get(FROM_CONNECTOR_ID);
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG_VALUES);
JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG_VALUES);
JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG_VALUES);
List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
List<MConfig> toConfig = restoreConfigList(toConfigJson);
List<MConfig> driverConfig = restoreConfigList(driverConfigJson);
MJob job = new MJob(
fromConnectorId,
toConnectorId,
fromConnectionId,
toConnectionId,
new MFromConfig(fromConfig),
new MToConfig(toConfig),
new MDriverConfig(driverConfig)
);
job.setPersistenceId((Long) object.get(ID));
job.setName((String) object.get(NAME));
job.setEnabled((Boolean) object.get(ENABLED));
job.setCreationUser((String) object.get(CREATION_USER));
job.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
job.setLastUpdateUser((String) object.get(UPDATE_USER));
job.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
jobs.add(job);
} }
} }
}
private MJob restoreJob(Object obj) {
JSONObject object = (JSONObject) obj;
long fromConnectorId = (Long) object.get(FROM_CONNECTOR_ID);
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG_VALUES);
JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG_VALUES);
JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG_VALUES);
List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
List<MConfig> toConfig = restoreConfigList(toConfigJson);
List<MConfig> driverConfig = restoreConfigList(driverConfigJson);
MJob job = new MJob(
fromConnectorId,
toConnectorId,
fromConnectionId,
toConnectionId,
new MFromConfig(fromConfig),
new MToConfig(toConfig),
new MDriverConfig(driverConfig)
);
job.setPersistenceId((Long) object.get(ID));
job.setName((String) object.get(NAME));
job.setEnabled((Boolean) object.get(ENABLED));
job.setCreationUser((String) object.get(CREATION_USER));
job.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
job.setLastUpdateUser((String) object.get(UPDATE_USER));
job.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
return job;
}
}

2
common/src/main/java/org/apache/sqoop/json/JsonBean.java
View File

@ -23,7 +23,7 @@ public interface JsonBean {
// common JSON constants for the rest-api response // common JSON constants for the rest-api response
static final String CONFIGURABLE_VERSION = "version"; static final String CONFIGURABLE_VERSION = "version";
static final String ALL_CONFIG_RESOURCES= "all-config-resources"; static final String ALL_CONFIGS= "all-config-resources";
@Deprecated // should not be used anymore in the rest api @Deprecated // should not be used anymore in the rest api
static final String ALL = "all"; static final String ALL = "all";

78
common/src/main/java/org/apache/sqoop/json/LinkBean.java
View File

@ -34,10 +34,9 @@
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
/** /**
* Link representation that is being send across the network between * Link representation that is being send across the network between Sqoop
* Sqoop server and client. Server might optionally send configs * server and client. Server might optionally send configs associated with the
* associated with the links to spare client of sending another HTTP * links to spare client of sending another HTTP requests to obtain them.
* requests to obtain them.
*/ */
public class LinkBean implements JsonBean { public class LinkBean implements JsonBean {
@ -89,54 +88,63 @@ public ResourceBundle getConnectorConfigBundle(Long id) {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
public JSONObject extract(boolean skipSensitive) { public JSONObject extract(boolean skipSensitive) {
JSONArray linkArray = extractLinks(skipSensitive);
JSONObject link = new JSONObject(); JSONObject link = new JSONObject();
link.put(LINK, linkArray); link.put(LINK, extractLink(skipSensitive, links.get(0)));
return link; return link;
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected JSONArray extractLinks(boolean skipSensitive) { protected JSONArray extractLinks(boolean skipSensitive) {
JSONArray linkArray = new JSONArray(); JSONArray linkArray = new JSONArray();
for (MLink link : links) {
for(MLink link : links) { linkArray.add(extractLink(skipSensitive, link));
JSONObject linkJsonObject = new JSONObject();
linkJsonObject.put(ID, link.getPersistenceId());
linkJsonObject.put(NAME, link.getName());
linkJsonObject.put(ENABLED, link.getEnabled());
linkJsonObject.put(CREATION_USER, link.getCreationUser());
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive));
linkArray.add(linkJsonObject);
} }
return linkArray; return linkArray;
} }
@SuppressWarnings("unchecked")
private JSONObject extractLink(boolean skipSensitive, MLink link) {
JSONObject linkJsonObject = new JSONObject();
linkJsonObject.put(ID, link.getPersistenceId());
linkJsonObject.put(NAME, link.getName());
linkJsonObject.put(ENABLED, link.getEnabled());
linkJsonObject.put(CREATION_USER, link.getCreationUser());
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive));
return linkJsonObject;
}
@Override @Override
public void restore(JSONObject jsonObject) { public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(LINK); links = new ArrayList<MLink>();
restoreLinks(array); JSONObject obj = (JSONObject) jsonObject.get(LINK);
links.add(restoreLink(obj));
} }
protected void restoreLinks(JSONArray array) { protected void restoreLinks(JSONArray array) {
links = new ArrayList<MLink>(); links = new ArrayList<MLink>();
for (Object obj : array) { for (Object obj : array) {
JSONObject object = (JSONObject) obj; links.add(restoreLink(obj));
long connectorId = (Long) object.get(CONNECTOR_ID);
JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG_VALUES);
List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID));
link.setName((String) object.get(NAME));
link.setEnabled((Boolean) object.get(ENABLED));
link.setCreationUser((String) object.get(CREATION_USER));
link.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
link.setLastUpdateUser((String) object.get(UPDATE_USER));
link.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
links.add(link);
} }
} }
private MLink restoreLink(Object obj) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(CONNECTOR_ID);
JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG_VALUES);
List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID));
link.setName((String) object.get(NAME));
link.setEnabled((Boolean) object.get(ENABLED));
link.setCreationUser((String) object.get(CREATION_USER));
link.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
link.setLastUpdateUser((String) object.get(UPDATE_USER));
link.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
return link;
}
} }

186
common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
View File

@ -80,61 +80,64 @@ public SubmissionBean() {
@Override @Override
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) { public JSONObject extract(boolean skipSensitive) {
JSONArray submissionArray = extractSubmissions();
JSONObject submission = new JSONObject(); JSONObject submission = new JSONObject();
submission.put(SUBMISSION, submissionArray); submission.put(SUBMISSION, extractSubmission(submissions.get(0)));
return submission; return submission;
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected JSONArray extractSubmissions() { protected JSONArray extractSubmissions() {
JSONArray submissionsArray = new JSONArray(); JSONArray submissionsArray = new JSONArray();
for (MSubmission submission : submissions) {
for (MSubmission submission : this.submissions) { submissionsArray.add(extractSubmission(submission));
JSONObject object = new JSONObject();
object.put(JOB, submission.getJobId());
object.put(STATUS, submission.getStatus().name());
object.put(PROGRESS, submission.getProgress());
if (submission.getCreationUser() != null) {
object.put(CREATION_USER, submission.getCreationUser());
}
if (submission.getCreationDate() != null) {
object.put(CREATION_DATE, submission.getCreationDate().getTime());
}
if (submission.getLastUpdateUser() != null) {
object.put(LAST_UPDATE_USER, submission.getLastUpdateUser());
}
if (submission.getLastUpdateDate() != null) {
object.put(LAST_UPDATE_DATE, submission.getLastUpdateDate().getTime());
}
if (submission.getExternalId() != null) {
object.put(EXTERNAL_ID, submission.getExternalId());
}
if (submission.getExternalLink() != null) {
object.put(EXTERNAL_LINK, submission.getExternalLink());
}
if (submission.getExceptionInfo() != null) {
object.put(EXCEPTION, submission.getExceptionInfo());
}
if (submission.getExceptionStackTrace() != null) {
object.put(EXCEPTION_TRACE, submission.getExceptionStackTrace());
}
if (submission.getCounters() != null) {
object.put(COUNTERS, extractCounters(submission.getCounters()));
}
if (submission.getFromSchema() != null) {
object.put(FROM_SCHEMA, extractSchema(submission.getFromSchema()));
}
if (submission.getToSchema() != null) {
object.put(TO_SCHEMA, extractSchema(submission.getToSchema()));
}
submissionsArray.add(object);
} }
return submissionsArray; return submissionsArray;
} }
@SuppressWarnings("unchecked")
private JSONObject extractSubmission(MSubmission submission) {
JSONObject object = new JSONObject();
object.put(JOB, submission.getJobId());
object.put(STATUS, submission.getStatus().name());
object.put(PROGRESS, submission.getProgress());
if (submission.getCreationUser() != null) {
object.put(CREATION_USER, submission.getCreationUser());
}
if (submission.getCreationDate() != null) {
object.put(CREATION_DATE, submission.getCreationDate().getTime());
}
if (submission.getLastUpdateUser() != null) {
object.put(LAST_UPDATE_USER, submission.getLastUpdateUser());
}
if (submission.getLastUpdateDate() != null) {
object.put(LAST_UPDATE_DATE, submission.getLastUpdateDate().getTime());
}
if (submission.getExternalId() != null) {
object.put(EXTERNAL_ID, submission.getExternalId());
}
if (submission.getExternalLink() != null) {
object.put(EXTERNAL_LINK, submission.getExternalLink());
}
if (submission.getExceptionInfo() != null) {
object.put(EXCEPTION, submission.getExceptionInfo());
}
if (submission.getExceptionStackTrace() != null) {
object.put(EXCEPTION_TRACE, submission.getExceptionStackTrace());
}
if (submission.getCounters() != null) {
object.put(COUNTERS, extractCounters(submission.getCounters()));
}
if (submission.getFromSchema() != null) {
object.put(FROM_SCHEMA, extractSchema(submission.getFromSchema()));
}
if (submission.getToSchema() != null) {
object.put(TO_SCHEMA, extractSchema(submission.getToSchema()));
}
return object;
}
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private JSONObject extractCounters(Counters counters) { private JSONObject extractCounters(Counters counters) {
JSONObject counterArray = new JSONObject(); JSONObject counterArray = new JSONObject();
@ -151,59 +154,62 @@ private JSONObject extractCounters(Counters counters) {
@Override @Override
public void restore(JSONObject json) { public void restore(JSONObject json) {
JSONArray submissionArray = (JSONArray) json.get(SUBMISSION); submissions = new ArrayList<MSubmission>();
restoreSubmissions(submissionArray); JSONObject obj = (JSONObject) json.get(SUBMISSION);
submissions.add(restoreSubmission(obj));
} }
protected void restoreSubmissions(JSONArray array) { protected void restoreSubmissions(JSONArray array) {
this.submissions = new ArrayList<MSubmission>(); submissions = new ArrayList<MSubmission>();
for (Object obj : array) { for (Object obj : array) {
JSONObject object = (JSONObject) obj; submissions.add(restoreSubmission(obj));
MSubmission submission = new MSubmission();
submission.setJobId((Long) object.get(JOB));
submission.setStatus(SubmissionStatus.valueOf((String) object.get(STATUS)));
submission.setProgress((Double) object.get(PROGRESS));
if (object.containsKey(CREATION_USER)) {
submission.setCreationUser((String) object.get(CREATION_USER));
}
if (object.containsKey(CREATION_DATE)) {
submission.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
}
if (object.containsKey(LAST_UPDATE_USER)) {
submission.setLastUpdateUser((String) object.get(LAST_UPDATE_USER));
}
if (object.containsKey(LAST_UPDATE_DATE)) {
submission.setLastUpdateDate(new Date((Long) object.get(LAST_UPDATE_DATE)));
}
if (object.containsKey(EXTERNAL_ID)) {
submission.setExternalId((String) object.get(EXTERNAL_ID));
}
if (object.containsKey(EXTERNAL_LINK)) {
submission.setExternalLink((String) object.get(EXTERNAL_LINK));
}
if (object.containsKey(EXCEPTION)) {
submission.setExceptionInfo((String) object.get(EXCEPTION));
}
if (object.containsKey(EXCEPTION_TRACE)) {
submission.setExceptionStackTrace((String) object.get(EXCEPTION_TRACE));
}
if (object.containsKey(COUNTERS)) {
submission.setCounters(restoreCounters((JSONObject) object.get(COUNTERS)));
}
if (object.containsKey(FROM_SCHEMA)) {
submission.setFromSchema(restoreSchema((JSONObject) object.get(FROM_SCHEMA)));
}
if (object.containsKey(TO_SCHEMA)) {
submission.setToSchema(restoreSchema((JSONObject) object.get(TO_SCHEMA)));
}
this.submissions.add(submission);
} }
} }
private MSubmission restoreSubmission(Object obj) {
JSONObject object = (JSONObject) obj;
MSubmission submission = new MSubmission();
submission.setJobId((Long) object.get(JOB));
submission.setStatus(SubmissionStatus.valueOf((String) object.get(STATUS)));
submission.setProgress((Double) object.get(PROGRESS));
if (object.containsKey(CREATION_USER)) {
submission.setCreationUser((String) object.get(CREATION_USER));
}
if (object.containsKey(CREATION_DATE)) {
submission.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
}
if (object.containsKey(LAST_UPDATE_USER)) {
submission.setLastUpdateUser((String) object.get(LAST_UPDATE_USER));
}
if (object.containsKey(LAST_UPDATE_DATE)) {
submission.setLastUpdateDate(new Date((Long) object.get(LAST_UPDATE_DATE)));
}
if (object.containsKey(EXTERNAL_ID)) {
submission.setExternalId((String) object.get(EXTERNAL_ID));
}
if (object.containsKey(EXTERNAL_LINK)) {
submission.setExternalLink((String) object.get(EXTERNAL_LINK));
}
if (object.containsKey(EXCEPTION)) {
submission.setExceptionInfo((String) object.get(EXCEPTION));
}
if (object.containsKey(EXCEPTION_TRACE)) {
submission.setExceptionStackTrace((String) object.get(EXCEPTION_TRACE));
}
if (object.containsKey(COUNTERS)) {
submission.setCounters(restoreCounters((JSONObject) object.get(COUNTERS)));
}
if (object.containsKey(FROM_SCHEMA)) {
submission.setFromSchema(restoreSchema((JSONObject) object.get(FROM_SCHEMA)));
}
if (object.containsKey(TO_SCHEMA)) {
submission.setToSchema(restoreSchema((JSONObject) object.get(TO_SCHEMA)));
}
return submission;
}
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Counters restoreCounters(JSONObject object) { public Counters restoreCounters(JSONObject object) {
Set<Map.Entry<String, JSONObject>> groupSet = object.entrySet(); Set<Map.Entry<String, JSONObject>> groupSet = object.entrySet();

84
common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
View File

@ -17,9 +17,8 @@
*/ */
package org.apache.sqoop.json; package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getConnector; import static org.junit.Assert.assertEquals;
import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.*;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
@ -27,14 +26,13 @@
import java.util.Map; import java.util.Map;
import java.util.ResourceBundle; import java.util.ResourceBundle;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigTestUtil;
import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
import org.json.simple.JSONValue; import org.json.simple.JSONValue;
import org.junit.Test; import org.junit.Test;
/**
*
*/
public class TestConnectorBean { public class TestConnectorBean {
/** /**
@ -42,16 +40,14 @@ public class TestConnectorBean {
* equal connector object. * equal connector object.
*/ */
@Test @Test
public void testSerialization() { public void testConnectorSerialization() {
// Create testing connector // Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>(); List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(getConnector(1L, "jdbc")); connectors.add(BeanTestUtil.getConnector(1L, "jdbc"));
connectors.add(getConnector(2L, "mysql"));
// Create testing bundles // Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>(); Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, getResourceBundle()); configBundles.put(1L, ConfigTestUtil.getResourceBundle());
configBundles.put(2L, getResourceBundle());
// Serialize it to JSON object // Serialize it to JSON object
ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles); ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles);
@ -61,75 +57,15 @@ public void testSerialization() {
String connectorJSONString = connectorJSON.toJSONString(); String connectorJSONString = connectorJSON.toJSONString();
// Retrieved transferred object // Retrieved transferred object
JSONObject parsedConnector = (JSONObject) JSONValue.parse(connectorJSONString); JSONObject parsedConnectors = (JSONObject) JSONValue.parse(connectorJSONString);
ConnectorBean parsedConnectorBean = new ConnectorBean(); ConnectorBean parsedConnectorBean = new ConnectorBean();
parsedConnectorBean.restore(parsedConnector); parsedConnectorBean.restore(parsedConnectors);
assertEquals(connectors.size(), 1);
assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size()); assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0)); assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0));
assertEquals(connectors.get(1), parsedConnectorBean.getConnectors().get(1));
ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L); ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L);
assertNotNull(retrievedBundle); assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a")); assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b")); assertEquals("b", retrievedBundle.getString("b"));
} }
@Test
public void testSingleDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(getConnector(1L, "jdbc", true, false));
connectors.add(getConnector(2L, "mysql", false, true));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, getResourceBundle());
bundles.put(2L, getResourceBundle());
// Serialize it to JSON object
ConnectorBean bean = new ConnectorBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorBean retrievedBean = new ConnectorBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
@Test
public void testNoDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(getConnector(1L, "jdbc", false, false));
connectors.add(getConnector(2L, "mysql", false, false));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, getResourceBundle());
bundles.put(2L, getResourceBundle());
// Serialize it to JSON object
ConnectorBean bean = new ConnectorBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorBean retrievedBean = new ConnectorBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
} }

130
common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java Normal file
View File

@ -0,0 +1,130 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigTestUtil;
import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
public class TestConnectorsBean {
@Test
public void testConnectorsSerialization() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(BeanTestUtil.getConnector(1L, "jdbc"));
connectors.add(BeanTestUtil.getConnector(2L, "mysql"));
// Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, ConfigTestUtil.getResourceBundle());
configBundles.put(2L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean connectorsBean = new ConnectorsBean(connectors, configBundles);
JSONObject connectorsJSON = connectorsBean.extract(false);
// "Move" it across network in text form
String connectorsJSONString = connectorsJSON.toJSONString();
// Retrieved transferred object
JSONObject parsedConnectors = (JSONObject) JSONValue.parse(connectorsJSONString);
ConnectorsBean parsedConnectorsBean = new ConnectorsBean();
parsedConnectorsBean.restore(parsedConnectors);
assertEquals(connectors.size(), parsedConnectorsBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorsBean.getConnectors().get(0));
assertEquals(connectors.get(1), parsedConnectorsBean.getConnectors().get(1));
ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get(1L);
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
}
@Test
public void testSingleDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(BeanTestUtil.getConnector(1L, "jdbc", true, false));
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, true));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorsBean retrievedBean = new ConnectorsBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
@Test
public void testNoDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(BeanTestUtil.getConnector(1L, "jdbc", false, false));
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, false));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorsBean retrievedBean = new ConnectorsBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
}

4
common/src/test/java/org/apache/sqoop/json/TestDriverBean.java
View File

@ -17,11 +17,11 @@
*/ */
package org.apache.sqoop.json; package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import java.util.ResourceBundle; import java.util.ResourceBundle;
import org.apache.sqoop.json.util.ConfigTestUtil;
import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MDriver;
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
import org.json.simple.JSONValue; import org.json.simple.JSONValue;
@ -41,7 +41,7 @@ public void testSerialization() {
MDriver driver = new MDriver(ConfigTestUtil.getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION); MDriver driver = new MDriver(ConfigTestUtil.getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION);
// Serialize it to JSON object // Serialize it to JSON object
DriverBean bean = new DriverBean(driver, getResourceBundle()); DriverBean bean = new DriverBean(driver, ConfigTestUtil.getResourceBundle());
JSONObject json = bean.extract(false); JSONObject json = bean.extract(false);
// "Move" it across network in text form // "Move" it across network in text form

39
common/src/test/java/org/apache/sqoop/json/TestJobBean.java
View File

@ -17,12 +17,12 @@
*/ */
package org.apache.sqoop.json; package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getJob;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import java.util.Date; import java.util.Date;
import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.Direction;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput; import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
@ -30,27 +30,19 @@
import org.json.simple.parser.ParseException; import org.json.simple.parser.ParseException;
import org.junit.Test; import org.junit.Test;
/**
*
*/
public class TestJobBean { public class TestJobBean {
@Test @Test
public void testSerialization() throws ParseException { public void testJobSerialization() throws ParseException {
Date created = new Date(); Date created = new Date();
Date updated = new Date(); Date updated = new Date();
MJob job = getJob("ahoj"); MJob job = BeanTestUtil.createJob("ahoj", "The big Job", 22L, created, updated);
job.setName("The big job");
job.setPersistenceId(666);
job.setCreationDate(created);
job.setLastUpdateDate(updated);
job.setEnabled(false);
// Fill some data at the beginning // Fill some data at the beginning
MStringInput input = (MStringInput) job.getJobConfig(Direction.FROM) MStringInput input = (MStringInput) job.getJobConfig(Direction.FROM).getConfigs().get(0)
.getConfigs().get(0).getInputs().get(0); .getInputs().get(0);
input.setValue("Hi there!"); input.setValue("Hi there!");
input = (MStringInput) job.getJobConfig(Direction.TO) input = (MStringInput) job.getJobConfig(Direction.TO).getConfigs().get(0).getInputs().get(0);
.getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!"); input.setValue("Hi there again!");
// Serialize it to JSON object // Serialize it to JSON object
@ -61,28 +53,31 @@ public void testSerialization() throws ParseException {
String jobJsonString = json.toJSONString(); String jobJsonString = json.toJSONString();
// Retrieved transferred object // Retrieved transferred object
JSONObject parsedJobJson = (JSONObject)JSONValue.parse(jobJsonString); JSONObject parsedJobJson = (JSONObject) JSONValue.parse(jobJsonString);
JobBean parsedJobBean = new JobBean(); JobBean parsedJobBean = new JobBean();
parsedJobBean.restore(parsedJobJson); parsedJobBean.restore(parsedJobJson);
MJob target = parsedJobBean.getJobs().get(0); MJob target = parsedJobBean.getJobs().get(0);
// Check id and name // Check id and name
assertEquals(666, target.getPersistenceId()); assertEquals(22L, target.getPersistenceId());
assertEquals("The big Job", target.getName());
assertEquals(target.getLinkId(Direction.FROM), 1); assertEquals(target.getLinkId(Direction.FROM), 1);
assertEquals(target.getLinkId(Direction.TO), 2); assertEquals(target.getLinkId(Direction.TO), 2);
assertEquals(target.getConnectorId(Direction.FROM), 1); assertEquals(target.getConnectorId(Direction.FROM), 1);
assertEquals(target.getConnectorId(Direction.TO), 2); assertEquals(target.getConnectorId(Direction.TO), 2);
assertEquals("The big job", target.getName());
assertEquals(created, target.getCreationDate()); assertEquals(created, target.getCreationDate());
assertEquals(updated, target.getLastUpdateDate()); assertEquals(updated, target.getLastUpdateDate());
assertEquals(false, target.getEnabled()); assertEquals(false, target.getEnabled());
// Test that value was correctly moved // Test that value was correctly moved
MStringInput targetInput = (MStringInput) target.getJobConfig(Direction.FROM) MStringInput targetInput = (MStringInput) target.getJobConfig(Direction.FROM).getConfigs()
.getConfigs().get(0).getInputs().get(0); .get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue()); assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) target.getJobConfig(Direction.TO) targetInput = (MStringInput) target.getJobConfig(Direction.TO).getConfigs().get(0).getInputs()
.getConfigs().get(0).getInputs().get(0); .get(0);
assertEquals("Hi there again!", targetInput.getValue()); assertEquals("Hi there again!", targetInput.getValue());
} }
} }

92
common/src/test/java/org/apache/sqoop/json/TestJobsBean.java Normal file
View File

@ -0,0 +1,92 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
import org.junit.Test;
public class TestJobsBean {
@Test
public void testJobsSerialization() throws ParseException {
Date created = new Date();
Date updated = new Date();
MJob job1 = BeanTestUtil.createJob("ahoj", "The big Job", 22L, created, updated);
MJob job2 = BeanTestUtil.createJob("ahoj", "The small Job", 44L, created, updated);
List<MJob> jobs = new ArrayList<MJob>();
jobs.add(job1);
jobs.add(job2);
// Fill some data at the beginning
MStringInput input = (MStringInput) job1.getJobConfig(Direction.FROM).getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
input = (MStringInput) job1.getJobConfig(Direction.TO).getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
JobsBean jobsBean = new JobsBean(jobs);
JSONObject json = jobsBean.extract(false);
// "Move" it across network in text form
String jobJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedJobsJson = (JSONObject) JSONValue.parse(jobJsonString);
JobsBean parsedJobsBean = new JobsBean();
parsedJobsBean.restore(parsedJobsJson);
MJob retrievedJob1 = parsedJobsBean.getJobs().get(0);
MJob retrievedJob2 = parsedJobsBean.getJobs().get(1);
// Check id and name
assertEquals(22L, retrievedJob1.getPersistenceId());
assertEquals("The big Job", retrievedJob1.getName());
assertEquals(44L, retrievedJob2.getPersistenceId());
assertEquals("The small Job", retrievedJob2.getName());
assertEquals(retrievedJob1.getLinkId(Direction.FROM), 1);
assertEquals(retrievedJob1.getLinkId(Direction.TO), 2);
assertEquals(retrievedJob1.getConnectorId(Direction.FROM), 1);
assertEquals(retrievedJob1.getConnectorId(Direction.TO), 2);
assertEquals(created, retrievedJob1.getCreationDate());
assertEquals(updated, retrievedJob1.getLastUpdateDate());
assertEquals(false, retrievedJob1.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) retrievedJob1.getJobConfig(Direction.FROM)
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) retrievedJob1.getJobConfig(Direction.TO).getConfigs().get(0)
.getInputs().get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}

94
common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
View File

@ -17,13 +17,13 @@
*/ */
package org.apache.sqoop.json; package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getLink;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import java.util.Date; import java.util.Date;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigInputConstants; import org.apache.sqoop.json.util.ConfigInputConstants;
import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MStringInput; import org.apache.sqoop.model.MStringInput;
@ -32,26 +32,17 @@
import org.json.simple.JSONValue; import org.json.simple.JSONValue;
import org.junit.Test; import org.junit.Test;
/**
*
*/
public class TestLinkBean { public class TestLinkBean {
@Test @Test
public void testSerialization() { public void testLinkSerialization() {
Date created = new Date(); Date created = new Date();
Date updated = new Date(); Date updated = new Date();
MLink link = getLink("ahoj"); MLink link = BeanTestUtil.createLink("ahoj", "link1", 22L, created, updated);
link.setName("Connection");
link.setPersistenceId(666);
link.setCreationUser("admin");
link.setCreationDate(created);
link.setLastUpdateUser("user");
link.setLastUpdateDate(updated);
link.setEnabled(false);
// Fill some data at the beginning // Fill some data at the beginning
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs() MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs().get(0)
.get(0).getInputs().get(0); .getInputs().get(0);
input.setValue("Hi there!"); input.setValue("Hi there!");
// Serialize it to JSON object // Serialize it to JSON object
@ -59,13 +50,12 @@ public void testSerialization() {
JSONObject json = linkBean.extract(false); JSONObject json = linkBean.extract(false);
// Check for sensitivity // Check for sensitivity
JSONArray linkArray = (JSONArray)json.get(LinkBean.LINK); JSONObject linkObj = (JSONObject) json.get(LinkBean.LINK);
JSONObject linkItem = (JSONObject)linkArray.get(0); JSONArray linkConfigs = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
JSONArray connectors = (JSONArray)linkItem.get(LinkBean.LINK_CONFIG_VALUES); JSONObject linkConfig = (JSONObject) linkConfigs.get(0);
JSONObject connector = (JSONObject)connectors.get(0); JSONArray inputs = (JSONArray) linkConfig.get(ConfigInputConstants.CONFIG_INPUTS);
JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); for (Object in : inputs) {
for (Object input1 : inputs) { assertTrue(((JSONObject) in).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE));
assertTrue(((JSONObject)input1).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE));
} }
// "Move" it across network in text form // "Move" it across network in text form
@ -75,39 +65,33 @@ public void testSerialization() {
JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString); JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString);
LinkBean retrievedBean = new LinkBean(); LinkBean retrievedBean = new LinkBean();
retrievedBean.restore(parsedLinkJson); retrievedBean.restore(parsedLinkJson);
MLink target = retrievedBean.getLinks().get(0); MLink retrievedLink = retrievedBean.getLinks().get(0);
// Check id and name // Check id and name
assertEquals(666, target.getPersistenceId()); assertEquals(22L, retrievedLink.getPersistenceId());
assertEquals("Connection", target.getName()); assertEquals("link1", retrievedLink.getName());
assertEquals("admin", target.getCreationUser()); assertEquals("admin", retrievedLink.getCreationUser());
assertEquals(created, target.getCreationDate()); assertEquals(created, retrievedLink.getCreationDate());
assertEquals("user", target.getLastUpdateUser()); assertEquals("user", retrievedLink.getLastUpdateUser());
assertEquals(updated, target.getLastUpdateDate()); assertEquals(updated, retrievedLink.getLastUpdateDate());
assertEquals(false, target.getEnabled()); assertEquals(false, retrievedLink.getEnabled());
// Test that value was correctly moved // Test that value was correctly moved
MStringInput targetInput = (MStringInput) target.getConnectorLinkConfig() MStringInput retrievedLinkInput = (MStringInput) retrievedLink.getConnectorLinkConfig().getConfigs().get(0)
.getConfigs().get(0).getInputs().get(0); .getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue()); assertEquals("Hi there!", retrievedLinkInput.getValue());
} }
@Test @Test
public void testSensitivityFilter() { public void testSensitivityFilter() {
Date created = new Date(); Date created = new Date();
Date updated = new Date(); Date updated = new Date();
MLink link = getLink("ahoj"); MLink link = BeanTestUtil.createLink("ahoj", "link1", 22L, created, updated);
link.setName("Connection");
link.setPersistenceId(666);
link.setCreationUser("admin");
link.setCreationDate(created);
link.setLastUpdateUser("user");
link.setLastUpdateDate(updated);
link.setEnabled(true);
// Fill some data at the beginning // Fill some data at the beginning
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs() MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs().get(0)
.get(0).getInputs().get(0); .getInputs().get(0);
input.setValue("Hi there!"); input.setValue("Hi there!");
// Serialize it to JSON object // Serialize it to JSON object
@ -116,25 +100,23 @@ public void testSensitivityFilter() {
JSONObject jsonFiltered = bean.extract(true); JSONObject jsonFiltered = bean.extract(true);
// Sensitive values should exist // Sensitive values should exist
JSONArray all = (JSONArray)json.get(LinkBean.LINK); JSONObject linkObj = (JSONObject) json.get(LinkBean.LINK);
JSONObject allItem = (JSONObject)all.get(0); JSONArray linkConfigsObj = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG_VALUES); JSONObject linkConfigObj = (JSONObject) linkConfigsObj.get(0);
JSONObject connector = (JSONObject)connectors.get(0); JSONArray inputs = (JSONArray) linkConfigObj.get(ConfigInputConstants.CONFIG_INPUTS);
JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
assertEquals(3, inputs.size()); assertEquals(3, inputs.size());
// Inputs are ordered when creating link // Inputs are ordered when creating link
JSONObject password = (JSONObject)inputs.get(2); JSONObject password = (JSONObject) inputs.get(2);
assertTrue(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE)); assertTrue(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE));
// Sensitive values should not exist // Sensitive values should not exist
all = (JSONArray)jsonFiltered.get(LinkBean.LINK); linkObj = (JSONObject) jsonFiltered.get(LinkBean.LINK);
allItem = (JSONObject)all.get(0); linkConfigsObj = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG_VALUES); linkConfigObj = (JSONObject) linkConfigsObj.get(0);
connector = (JSONObject)connectors.get(0); inputs = (JSONArray) linkConfigObj.get(ConfigInputConstants.CONFIG_INPUTS);
inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
assertEquals(3, inputs.size()); assertEquals(3, inputs.size());
// Inputs are ordered when creating link // Inputs are ordered when creating link
password = (JSONObject)inputs.get(2); password = (JSONObject) inputs.get(2);
assertFalse(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE)); assertFalse(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE));
} }
} }

98
common/src/test/java/org/apache/sqoop/json/TestLinksBean.java Normal file
View File

@ -0,0 +1,98 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigInputConstants;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
public class TestLinksBean {
@Test
public void testLinksSerialization() {
Date created = new Date();
Date updated = new Date();
MLink link1 = BeanTestUtil.createLink("ahoj", "link1", 666L, created, updated);
MLink link2 = BeanTestUtil.createLink("jhoa", "link2", 888L, created, updated);
List<MLink> links = new ArrayList<MLink>();
links.add(link1);
links.add(link2);
// Fill some data at the beginning
MStringInput input = (MStringInput) link1.getConnectorLinkConfig().getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
// Serialize it to JSON object
LinksBean linkBean = new LinksBean(links);
JSONObject json = linkBean.extract(false);
// Check for sensitivity
JSONArray linksObj = (JSONArray) json.get(LinksBean.LINKS);
JSONObject linkObj = (JSONObject) linksObj.get(0);
JSONArray linkConfigs = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject linkConfig = (JSONObject) linkConfigs.get(0);
JSONArray inputs = (JSONArray) linkConfig.get(ConfigInputConstants.CONFIG_INPUTS);
for (Object inp : inputs) {
assertTrue(((JSONObject) inp).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE));
}
// "Move" it across network in text form
String linkJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString);
LinksBean retrievedBean = new LinksBean();
retrievedBean.restore(parsedLinkJson);
MLink targetLink1 = retrievedBean.getLinks().get(0);
MLink targetLink2 = retrievedBean.getLinks().get(1);
// Check id and name
assertEquals(666L, targetLink1.getPersistenceId());
assertEquals(888L, targetLink2.getPersistenceId());
assertEquals("link1", targetLink1.getName());
assertEquals("link2", targetLink2.getName());
assertEquals("admin", targetLink1.getCreationUser());
assertEquals(created, targetLink1.getCreationDate());
assertEquals("user", targetLink1.getLastUpdateUser());
assertEquals(updated, targetLink1.getLastUpdateDate());
assertEquals(false, targetLink1.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) targetLink1.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
}
}

4
common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
View File

@ -36,9 +36,7 @@
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
/** // NOTE: This tests both the submission and submissions list bean
*
*/
public class TestSubmissionBean { public class TestSubmissionBean {
private static final double EPSILON = 0.01; private static final double EPSILON = 0.01;

83
common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java Normal file
View File

@ -0,0 +1,83 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json.util;
import java.util.Date;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MToConfig;
public class BeanTestUtil {
public static MLink createLink(String connectorName, String linkName, Long linkId, Date created,
Date updated) {
MLink link1 = getLink(connectorName);
link1.setName(linkName);
link1.setPersistenceId(linkId);
link1.setCreationUser("admin");
link1.setCreationDate(created);
link1.setLastUpdateUser("user");
link1.setLastUpdateDate(updated);
link1.setEnabled(false);
return link1;
}
public static MJob createJob(String connectorName, String jobName, Long jobId, Date created, Date updated) {
MJob job = BeanTestUtil.getJob(connectorName);
job.setName(jobName);
job.setPersistenceId(jobId);
job.setCreationDate(created);
job.setLastUpdateDate(updated);
job.setEnabled(false);
return job;
}
public static MLink getLink(String connectorName) {
return new MLink(1, getConnector(1L, connectorName).getLinkConfig());
}
public static MConnector getConnector(Long connectorId, String connectorName) {
return getConnector(connectorId, connectorName, true, true);
}
public static MConnector getConnector(Long id, String name, boolean from, boolean to) {
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (from) {
fromConfig = ConfigTestUtil.getFromConfig();
}
if (to) {
toConfig = ConfigTestUtil.getToConfig();
}
MConnector connector = new MConnector(name, name + ".class", "1.0-test",
ConfigTestUtil.getLinkConfig(), fromConfig, toConfig);
// simulate a persistence id
connector.setPersistenceId(id);
return connector;
}
public static MJob getJob(String connectorName) {
return new MJob(1, 2, 1, 2, getConnector(1L, connectorName).getFromConfig(), getConnector(1L, connectorName)
.getToConfig(), ConfigTestUtil.getDriverConfig());
}
}

38
common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java → common/src/test/java/org/apache/sqoop/json/util/ConfigTestUtil.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.sqoop.json; package org.apache.sqoop.json.util;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -24,52 +24,16 @@
import java.util.ResourceBundle; import java.util.ResourceBundle;
import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig; import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput; import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig; import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MStringInput; import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MToConfig; import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.utils.MapResourceBundle; import org.apache.sqoop.utils.MapResourceBundle;
/**
*
*/
public class ConfigTestUtil { public class ConfigTestUtil {
public static MConnector getConnector(Long id, String name) {
return getConnector(id, name, true, true);
}
public static MConnector getConnector(Long id, String name, boolean from, boolean to) {
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (from) {
fromConfig = getFromConfig();
}
if (to) {
toConfig = getToConfig();
}
MConnector connector = new MConnector(name, name + ".class", "1.0-test",
getLinkConfig(), fromConfig, toConfig);
// simulate a persistence id
connector.setPersistenceId(id);
return connector;
}
public static MLink getLink(String name) {
return new MLink(1, getConnector(1L, name).getLinkConfig());
}
public static MJob getJob(String name) {
return new MJob(1, 2, 1, 2, getConnector(1L, name).getFromConfig(), getConnector(1L, name)
.getToConfig(), getDriverConfig());
}
public static MDriverConfig getDriverConfig() { public static MDriverConfig getDriverConfig() {
List<MInput<?>> inputs; List<MInput<?>> inputs;