github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-06 10:20:14 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1687: Sqoop2: Single resource in JSON should not be a list

Browse Source 
(Veena Basavaraj via Abraham Elmahrek)
This commit is contained in:
Abraham Elmahrek 2014-11-06 13:46:27 -08:00
parent 8aa4648130
commit ec3316b100
16 changed files with 771 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
View File

@ -72,9 +72,8 @@ public Map<Long, ResourceBundle> getResourceBundles() {
@SuppressWarnings("unchecked")
@Override
public JSONObject extract(boolean skipSensitive) {
JSONArray connectorArray = extractConnectors(skipSensitive);
JSONObject connector = new JSONObject();
connector.put(CONNECTOR, connectorArray);
connector.put(CONNECTOR, extractConnector(skipSensitive, connectors.get(0)));
return connector;
}
@ -82,91 +81,100 @@ public JSONObject extract(boolean skipSensitive) {
protected JSONArray extractConnectors(boolean skipSensitive) {
JSONArray connectorArray = new JSONArray();
for (MConnector connector : connectors) {
JSONObject connectorJsonObject = new JSONObject();
connectorJsonObject.put(ID, connector.getPersistenceId());
connectorJsonObject.put(NAME, connector.getUniqueName());
connectorJsonObject.put(CLASS, connector.getClassName());
connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion());
connectorJsonObject.put(
CONNECTOR_LINK_CONFIG,
extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig()
.getType(), skipSensitive));
connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
// add sub fields to the job config for from and to
if (connector.getFromConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.FROM,
extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig()
.getType(), skipSensitive));
}
if (connector.getToConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.TO,
extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig()
.getType(), skipSensitive));
}
// add the config-param inside each connector
connectorJsonObject.put(ALL_CONFIG_RESOURCES, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIG_RESOURCES,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId())));
}
connectorArray.add(connectorJsonObject);
connectorArray.add(extractConnector(skipSensitive, connector));
}
return connectorArray;
}
@SuppressWarnings("unchecked")
private JSONObject extractConnector(boolean skipSensitive, MConnector connector) {
JSONObject connectorJsonObject = new JSONObject();
connectorJsonObject.put(ID, connector.getPersistenceId());
connectorJsonObject.put(NAME, connector.getUniqueName());
connectorJsonObject.put(CLASS, connector.getClassName());
connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion());
connectorJsonObject.put(
CONNECTOR_LINK_CONFIG,
extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig()
.getType(), skipSensitive));
connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
// add sub fields to the job config for from and to
if (connector.getFromConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.FROM,
extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig()
.getType(), skipSensitive));
}
if (connector.getToConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.TO,
extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig()
.getType(), skipSensitive));
}
// add the config-param inside each connector
connectorJsonObject.put(ALL_CONFIGS, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIGS,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId())));
}
return connectorJsonObject;
}
@Override
public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(CONNECTOR);
restoreConnectors(array);
connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
JSONObject obj = (JSONObject) jsonObject.get(CONNECTOR);
connectors.add(restoreConnector(obj));
}
protected void restoreConnectors(JSONArray array) {
connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
for (Object obj : array) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(ID);
String uniqueName = (String) object.get(NAME);
String className = (String) object.get(CLASS);
String version = (String) object.get(CONFIGURABLE_VERSION);
List<MConfig> linkConfigs = restoreConfigList((JSONArray) object
.get(CONNECTOR_LINK_CONFIG));
// parent that encapsulates both the from/to configs
JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (fromJobConfigJson != null) {
List<MConfig> fromJobConfig = restoreConfigList(fromJobConfigJson);
fromConfig = new MFromConfig(fromJobConfig);
}
if (toJobConfigJson != null) {
List<MConfig> toJobConfig = restoreConfigList(toJobConfigJson);
toConfig = new MToConfig(toJobConfig);
}
MLinkConfig linkConfig = new MLinkConfig(linkConfigs);
MConnector connector = new MConnector(uniqueName, className, version, linkConfig, fromConfig,
toConfig);
connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIG_RESOURCES)) {
JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIG_RESOURCES);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle));
}
connectors.add(connector);
connectors.add(restoreConnector(obj));
}
}
private MConnector restoreConnector(Object obj) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(ID);
String uniqueName = (String) object.get(NAME);
String className = (String) object.get(CLASS);
String version = (String) object.get(CONFIGURABLE_VERSION);
List<MConfig> linkConfigs = restoreConfigList((JSONArray) object
.get(CONNECTOR_LINK_CONFIG));
// parent that encapsulates both the from/to configs
JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (fromJobConfigJson != null) {
List<MConfig> fromJobConfig = restoreConfigList(fromJobConfigJson);
fromConfig = new MFromConfig(fromJobConfig);
}
if (toJobConfigJson != null) {
List<MConfig> toJobConfig = restoreConfigList(toJobConfigJson);
toConfig = new MToConfig(toJobConfig);
}
MLinkConfig linkConfig = new MLinkConfig(linkConfigs);
MConnector connector = new MConnector(uniqueName, className, version, linkConfig, fromConfig,
toConfig);
connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIGS)) {
JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIGS);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle));
}
return connector;
}
}

4
common/src/main/java/org/apache/sqoop/json/DriverBean.java
View File

@ -69,7 +69,7 @@ public JSONObject extract(boolean skipSensitive) {
result.put(ID, driver.getPersistenceId());
result.put(CONFIGURABLE_VERSION, driver.getVersion());
result.put(DRIVER_JOB_CONFIG, configs);
result.put(ALL_CONFIG_RESOURCES, extractConfigParamBundle(driverConfigBundle));
result.put(ALL_CONFIGS, extractConfigParamBundle(driverConfigBundle));
return result;
}
@ -80,6 +80,6 @@ public void restore(JSONObject jsonObject) {
List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_JOB_CONFIG));
driver = new MDriver(new MDriverConfig(driverConfig), driverVersion);
driver.setPersistenceId(id);
driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIG_RESOURCES));
driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIGS));
}
}

143
common/src/main/java/org/apache/sqoop/json/JobBean.java
View File

@ -101,9 +101,8 @@ public ResourceBundle getDriverConfigBundle() {
@Override
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONArray jobArray = extractJobs(skipSensitive);
JSONObject job = new JSONObject();
job.put(JOB, jobArray);
job.put(JOB, extractJob(skipSensitive, jobs.get(0)));
return job;
}
@ -111,83 +110,89 @@ public JSONObject extract(boolean skipSensitive) {
protected JSONArray extractJobs(boolean skipSensitive) {
JSONArray jobArray = new JSONArray();
for (MJob job : jobs) {
JSONObject object = new JSONObject();
object.put(ID, job.getPersistenceId());
object.put(NAME, job.getName());
object.put(ENABLED, job.getEnabled());
object.put(CREATION_USER, job.getCreationUser());
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
// job link associated connectors
// TODO(SQOOP-1634): fix not to require the connectorIds in the post data
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
// job associated links
object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job configs
MFromConfig fromConfigList = job.getFromJobConfig();
object.put(FROM_CONFIG_VALUES,
extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive));
MToConfig toConfigList = job.getToJobConfig();
object.put(TO_CONFIG_VALUES,
extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive));
MDriverConfig driverConfigList = job.getDriverConfig();
object.put(
DRIVER_CONFIG_VALUES,
extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(),
skipSensitive));
jobArray.add(object);
jobArray.add(extractJob(skipSensitive, job));
}
return jobArray;
}
@SuppressWarnings("unchecked")
private JSONObject extractJob(boolean skipSensitive, MJob job) {
JSONObject object = new JSONObject();
object.put(ID, job.getPersistenceId());
object.put(NAME, job.getName());
object.put(ENABLED, job.getEnabled());
object.put(CREATION_USER, job.getCreationUser());
object.put(CREATION_DATE, job.getCreationDate().getTime());
object.put(UPDATE_USER, job.getLastUpdateUser());
object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
// job link associated connectors
// TODO(SQOOP-1634): fix not to require the connectorIds in the post data
object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
// job associated links
object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job configs
MFromConfig fromConfigList = job.getFromJobConfig();
object.put(FROM_CONFIG_VALUES,
extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive));
MToConfig toConfigList = job.getToJobConfig();
object.put(TO_CONFIG_VALUES,
extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive));
MDriverConfig driverConfigList = job.getDriverConfig();
object.put(
DRIVER_CONFIG_VALUES,
extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(),
skipSensitive));
return object;
}
@Override
public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(JOB);
restoreJobs(array);
jobs = new ArrayList<MJob>();
JSONObject obj = (JSONObject) jsonObject.get(JOB);
jobs.add(restoreJob(obj));
}
protected void restoreJobs(JSONArray array) {
jobs = new ArrayList<MJob>();
for (Object obj : array) {
JSONObject object = (JSONObject) obj;
long fromConnectorId = (Long) object.get(FROM_CONNECTOR_ID);
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG_VALUES);
JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG_VALUES);
JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG_VALUES);
List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
List<MConfig> toConfig = restoreConfigList(toConfigJson);
List<MConfig> driverConfig = restoreConfigList(driverConfigJson);
MJob job = new MJob(
fromConnectorId,
toConnectorId,
fromConnectionId,
toConnectionId,
new MFromConfig(fromConfig),
new MToConfig(toConfig),
new MDriverConfig(driverConfig)
);
job.setPersistenceId((Long) object.get(ID));
job.setName((String) object.get(NAME));
job.setEnabled((Boolean) object.get(ENABLED));
job.setCreationUser((String) object.get(CREATION_USER));
job.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
job.setLastUpdateUser((String) object.get(UPDATE_USER));
job.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
jobs.add(job);
jobs.add(restoreJob(obj));
}
}
}
private MJob restoreJob(Object obj) {
JSONObject object = (JSONObject) obj;
long fromConnectorId = (Long) object.get(FROM_CONNECTOR_ID);
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG_VALUES);
JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG_VALUES);
JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG_VALUES);
List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
List<MConfig> toConfig = restoreConfigList(toConfigJson);
List<MConfig> driverConfig = restoreConfigList(driverConfigJson);
MJob job = new MJob(
fromConnectorId,
toConnectorId,
fromConnectionId,
toConnectionId,
new MFromConfig(fromConfig),
new MToConfig(toConfig),
new MDriverConfig(driverConfig)
);
job.setPersistenceId((Long) object.get(ID));
job.setName((String) object.get(NAME));
job.setEnabled((Boolean) object.get(ENABLED));
job.setCreationUser((String) object.get(CREATION_USER));
job.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
job.setLastUpdateUser((String) object.get(UPDATE_USER));
job.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
return job;
}
}

2
common/src/main/java/org/apache/sqoop/json/JsonBean.java
View File

@ -23,7 +23,7 @@ public interface JsonBean {
// common JSON constants for the rest-api response
static final String CONFIGURABLE_VERSION = "version";
static final String ALL_CONFIG_RESOURCES= "all-config-resources";
static final String ALL_CONFIGS= "all-config-resources";
@Deprecated // should not be used anymore in the rest api
static final String ALL = "all";

78
common/src/main/java/org/apache/sqoop/json/LinkBean.java
View File

@ -34,10 +34,9 @@
import org.json.simple.JSONObject;
/**
* Link representation that is being send across the network between
* Sqoop server and client. Server might optionally send configs
* associated with the links to spare client of sending another HTTP
* requests to obtain them.
* Link representation that is being send across the network between Sqoop
* server and client. Server might optionally send configs associated with the
* links to spare client of sending another HTTP requests to obtain them.
*/
public class LinkBean implements JsonBean {
@ -89,54 +88,63 @@ public ResourceBundle getConnectorConfigBundle(Long id) {
@SuppressWarnings("unchecked")
@Override
public JSONObject extract(boolean skipSensitive) {
JSONArray linkArray = extractLinks(skipSensitive);
JSONObject link = new JSONObject();
link.put(LINK, linkArray);
link.put(LINK, extractLink(skipSensitive, links.get(0)));
return link;
}
@SuppressWarnings("unchecked")
protected JSONArray extractLinks(boolean skipSensitive) {
JSONArray linkArray = new JSONArray();
for(MLink link : links) {
JSONObject linkJsonObject = new JSONObject();
linkJsonObject.put(ID, link.getPersistenceId());
linkJsonObject.put(NAME, link.getName());
linkJsonObject.put(ENABLED, link.getEnabled());
linkJsonObject.put(CREATION_USER, link.getCreationUser());
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive));
linkArray.add(linkJsonObject);
for (MLink link : links) {
linkArray.add(extractLink(skipSensitive, link));
}
return linkArray;
}
@SuppressWarnings("unchecked")
private JSONObject extractLink(boolean skipSensitive, MLink link) {
JSONObject linkJsonObject = new JSONObject();
linkJsonObject.put(ID, link.getPersistenceId());
linkJsonObject.put(NAME, link.getName());
linkJsonObject.put(ENABLED, link.getEnabled());
linkJsonObject.put(CREATION_USER, link.getCreationUser());
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive));
return linkJsonObject;
}
@Override
public void restore(JSONObject jsonObject) {
JSONArray array = (JSONArray) jsonObject.get(LINK);
restoreLinks(array);
links = new ArrayList<MLink>();
JSONObject obj = (JSONObject) jsonObject.get(LINK);
links.add(restoreLink(obj));
}
protected void restoreLinks(JSONArray array) {
links = new ArrayList<MLink>();
for (Object obj : array) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(CONNECTOR_ID);
JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG_VALUES);
List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID));
link.setName((String) object.get(NAME));
link.setEnabled((Boolean) object.get(ENABLED));
link.setCreationUser((String) object.get(CREATION_USER));
link.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
link.setLastUpdateUser((String) object.get(UPDATE_USER));
link.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
links.add(link);
links.add(restoreLink(obj));
}
}
private MLink restoreLink(Object obj) {
JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(CONNECTOR_ID);
JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG_VALUES);
List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID));
link.setName((String) object.get(NAME));
link.setEnabled((Boolean) object.get(ENABLED));
link.setCreationUser((String) object.get(CREATION_USER));
link.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
link.setLastUpdateUser((String) object.get(UPDATE_USER));
link.setLastUpdateDate(new Date((Long) object.get(UPDATE_DATE)));
return link;
}
}

186
common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
View File

@ -80,61 +80,64 @@ public SubmissionBean() {
@Override
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONArray submissionArray = extractSubmissions();
JSONObject submission = new JSONObject();
submission.put(SUBMISSION, submissionArray);
submission.put(SUBMISSION, extractSubmission(submissions.get(0)));
return submission;
}
@SuppressWarnings("unchecked")
protected JSONArray extractSubmissions() {
JSONArray submissionsArray = new JSONArray();
for (MSubmission submission : this.submissions) {
JSONObject object = new JSONObject();
object.put(JOB, submission.getJobId());
object.put(STATUS, submission.getStatus().name());
object.put(PROGRESS, submission.getProgress());
if (submission.getCreationUser() != null) {
object.put(CREATION_USER, submission.getCreationUser());
}
if (submission.getCreationDate() != null) {
object.put(CREATION_DATE, submission.getCreationDate().getTime());
}
if (submission.getLastUpdateUser() != null) {
object.put(LAST_UPDATE_USER, submission.getLastUpdateUser());
}
if (submission.getLastUpdateDate() != null) {
object.put(LAST_UPDATE_DATE, submission.getLastUpdateDate().getTime());
}
if (submission.getExternalId() != null) {
object.put(EXTERNAL_ID, submission.getExternalId());
}
if (submission.getExternalLink() != null) {
object.put(EXTERNAL_LINK, submission.getExternalLink());
}
if (submission.getExceptionInfo() != null) {
object.put(EXCEPTION, submission.getExceptionInfo());
}
if (submission.getExceptionStackTrace() != null) {
object.put(EXCEPTION_TRACE, submission.getExceptionStackTrace());
}
if (submission.getCounters() != null) {
object.put(COUNTERS, extractCounters(submission.getCounters()));
}
if (submission.getFromSchema() != null) {
object.put(FROM_SCHEMA, extractSchema(submission.getFromSchema()));
}
if (submission.getToSchema() != null) {
object.put(TO_SCHEMA, extractSchema(submission.getToSchema()));
}
submissionsArray.add(object);
for (MSubmission submission : submissions) {
submissionsArray.add(extractSubmission(submission));
}
return submissionsArray;
}
@SuppressWarnings("unchecked")
private JSONObject extractSubmission(MSubmission submission) {
JSONObject object = new JSONObject();
object.put(JOB, submission.getJobId());
object.put(STATUS, submission.getStatus().name());
object.put(PROGRESS, submission.getProgress());
if (submission.getCreationUser() != null) {
object.put(CREATION_USER, submission.getCreationUser());
}
if (submission.getCreationDate() != null) {
object.put(CREATION_DATE, submission.getCreationDate().getTime());
}
if (submission.getLastUpdateUser() != null) {
object.put(LAST_UPDATE_USER, submission.getLastUpdateUser());
}
if (submission.getLastUpdateDate() != null) {
object.put(LAST_UPDATE_DATE, submission.getLastUpdateDate().getTime());
}
if (submission.getExternalId() != null) {
object.put(EXTERNAL_ID, submission.getExternalId());
}
if (submission.getExternalLink() != null) {
object.put(EXTERNAL_LINK, submission.getExternalLink());
}
if (submission.getExceptionInfo() != null) {
object.put(EXCEPTION, submission.getExceptionInfo());
}
if (submission.getExceptionStackTrace() != null) {
object.put(EXCEPTION_TRACE, submission.getExceptionStackTrace());
}
if (submission.getCounters() != null) {
object.put(COUNTERS, extractCounters(submission.getCounters()));
}
if (submission.getFromSchema() != null) {
object.put(FROM_SCHEMA, extractSchema(submission.getFromSchema()));
}
if (submission.getToSchema() != null) {
object.put(TO_SCHEMA, extractSchema(submission.getToSchema()));
}
return object;
}
@SuppressWarnings("unchecked")
private JSONObject extractCounters(Counters counters) {
JSONObject counterArray = new JSONObject();
@ -151,59 +154,62 @@ private JSONObject extractCounters(Counters counters) {
@Override
public void restore(JSONObject json) {
JSONArray submissionArray = (JSONArray) json.get(SUBMISSION);
restoreSubmissions(submissionArray);
submissions = new ArrayList<MSubmission>();
JSONObject obj = (JSONObject) json.get(SUBMISSION);
submissions.add(restoreSubmission(obj));
}
protected void restoreSubmissions(JSONArray array) {
this.submissions = new ArrayList<MSubmission>();
submissions = new ArrayList<MSubmission>();
for (Object obj : array) {
JSONObject object = (JSONObject) obj;
MSubmission submission = new MSubmission();
submission.setJobId((Long) object.get(JOB));
submission.setStatus(SubmissionStatus.valueOf((String) object.get(STATUS)));
submission.setProgress((Double) object.get(PROGRESS));
if (object.containsKey(CREATION_USER)) {
submission.setCreationUser((String) object.get(CREATION_USER));
}
if (object.containsKey(CREATION_DATE)) {
submission.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
}
if (object.containsKey(LAST_UPDATE_USER)) {
submission.setLastUpdateUser((String) object.get(LAST_UPDATE_USER));
}
if (object.containsKey(LAST_UPDATE_DATE)) {
submission.setLastUpdateDate(new Date((Long) object.get(LAST_UPDATE_DATE)));
}
if (object.containsKey(EXTERNAL_ID)) {
submission.setExternalId((String) object.get(EXTERNAL_ID));
}
if (object.containsKey(EXTERNAL_LINK)) {
submission.setExternalLink((String) object.get(EXTERNAL_LINK));
}
if (object.containsKey(EXCEPTION)) {
submission.setExceptionInfo((String) object.get(EXCEPTION));
}
if (object.containsKey(EXCEPTION_TRACE)) {
submission.setExceptionStackTrace((String) object.get(EXCEPTION_TRACE));
}
if (object.containsKey(COUNTERS)) {
submission.setCounters(restoreCounters((JSONObject) object.get(COUNTERS)));
}
if (object.containsKey(FROM_SCHEMA)) {
submission.setFromSchema(restoreSchema((JSONObject) object.get(FROM_SCHEMA)));
}
if (object.containsKey(TO_SCHEMA)) {
submission.setToSchema(restoreSchema((JSONObject) object.get(TO_SCHEMA)));
}
this.submissions.add(submission);
submissions.add(restoreSubmission(obj));
}
}
private MSubmission restoreSubmission(Object obj) {
JSONObject object = (JSONObject) obj;
MSubmission submission = new MSubmission();
submission.setJobId((Long) object.get(JOB));
submission.setStatus(SubmissionStatus.valueOf((String) object.get(STATUS)));
submission.setProgress((Double) object.get(PROGRESS));
if (object.containsKey(CREATION_USER)) {
submission.setCreationUser((String) object.get(CREATION_USER));
}
if (object.containsKey(CREATION_DATE)) {
submission.setCreationDate(new Date((Long) object.get(CREATION_DATE)));
}
if (object.containsKey(LAST_UPDATE_USER)) {
submission.setLastUpdateUser((String) object.get(LAST_UPDATE_USER));
}
if (object.containsKey(LAST_UPDATE_DATE)) {
submission.setLastUpdateDate(new Date((Long) object.get(LAST_UPDATE_DATE)));
}
if (object.containsKey(EXTERNAL_ID)) {
submission.setExternalId((String) object.get(EXTERNAL_ID));
}
if (object.containsKey(EXTERNAL_LINK)) {
submission.setExternalLink((String) object.get(EXTERNAL_LINK));
}
if (object.containsKey(EXCEPTION)) {
submission.setExceptionInfo((String) object.get(EXCEPTION));
}
if (object.containsKey(EXCEPTION_TRACE)) {
submission.setExceptionStackTrace((String) object.get(EXCEPTION_TRACE));
}
if (object.containsKey(COUNTERS)) {
submission.setCounters(restoreCounters((JSONObject) object.get(COUNTERS)));
}
if (object.containsKey(FROM_SCHEMA)) {
submission.setFromSchema(restoreSchema((JSONObject) object.get(FROM_SCHEMA)));
}
if (object.containsKey(TO_SCHEMA)) {
submission.setToSchema(restoreSchema((JSONObject) object.get(TO_SCHEMA)));
}
return submission;
}
@SuppressWarnings("unchecked")
public Counters restoreCounters(JSONObject object) {
Set<Map.Entry<String, JSONObject>> groupSet = object.entrySet();

84
common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
View File

@ -17,9 +17,8 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getConnector;
import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.util.HashMap;
import java.util.LinkedList;
@ -27,14 +26,13 @@
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigTestUtil;
import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
/**
*
*/
public class TestConnectorBean {
/**
@ -42,16 +40,14 @@ public class TestConnectorBean {
* equal connector object.
*/
@Test
public void testSerialization() {
public void testConnectorSerialization() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(getConnector(1L, "jdbc"));
connectors.add(getConnector(2L, "mysql"));
connectors.add(BeanTestUtil.getConnector(1L, "jdbc"));
// Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, getResourceBundle());
configBundles.put(2L, getResourceBundle());
configBundles.put(1L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles);
@ -61,75 +57,15 @@ public void testSerialization() {
String connectorJSONString = connectorJSON.toJSONString();
// Retrieved transferred object
JSONObject parsedConnector = (JSONObject) JSONValue.parse(connectorJSONString);
JSONObject parsedConnectors = (JSONObject) JSONValue.parse(connectorJSONString);
ConnectorBean parsedConnectorBean = new ConnectorBean();
parsedConnectorBean.restore(parsedConnector);
parsedConnectorBean.restore(parsedConnectors);
assertEquals(connectors.size(), 1);
assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0));
assertEquals(connectors.get(1), parsedConnectorBean.getConnectors().get(1));
ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L);
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
}
@Test
public void testSingleDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(getConnector(1L, "jdbc", true, false));
connectors.add(getConnector(2L, "mysql", false, true));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, getResourceBundle());
bundles.put(2L, getResourceBundle());
// Serialize it to JSON object
ConnectorBean bean = new ConnectorBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorBean retrievedBean = new ConnectorBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
@Test
public void testNoDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(getConnector(1L, "jdbc", false, false));
connectors.add(getConnector(2L, "mysql", false, false));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, getResourceBundle());
bundles.put(2L, getResourceBundle());
// Serialize it to JSON object
ConnectorBean bean = new ConnectorBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorBean retrievedBean = new ConnectorBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
}

130
common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java Normal file
View File

@ -0,0 +1,130 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.ResourceBundle;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigTestUtil;
import org.apache.sqoop.model.MConnector;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
public class TestConnectorsBean {
@Test
public void testConnectorsSerialization() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(BeanTestUtil.getConnector(1L, "jdbc"));
connectors.add(BeanTestUtil.getConnector(2L, "mysql"));
// Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, ConfigTestUtil.getResourceBundle());
configBundles.put(2L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean connectorsBean = new ConnectorsBean(connectors, configBundles);
JSONObject connectorsJSON = connectorsBean.extract(false);
// "Move" it across network in text form
String connectorsJSONString = connectorsJSON.toJSONString();
// Retrieved transferred object
JSONObject parsedConnectors = (JSONObject) JSONValue.parse(connectorsJSONString);
ConnectorsBean parsedConnectorsBean = new ConnectorsBean();
parsedConnectorsBean.restore(parsedConnectors);
assertEquals(connectors.size(), parsedConnectorsBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorsBean.getConnectors().get(0));
assertEquals(connectors.get(1), parsedConnectorsBean.getConnectors().get(1));
ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get(1L);
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
}
@Test
public void testSingleDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(BeanTestUtil.getConnector(1L, "jdbc", true, false));
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, true));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorsBean retrievedBean = new ConnectorsBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
@Test
public void testNoDirection() {
// Create testing connector
List<MConnector> connectors = new LinkedList<MConnector>();
connectors.add(BeanTestUtil.getConnector(1L, "jdbc", false, false));
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, false));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles);
JSONObject json = bean.extract(false);
// "Move" it across network in text form
String string = json.toJSONString();
// Retrieved transferred object
JSONObject retrievedJson = (JSONObject) JSONValue.parse(string);
ConnectorsBean retrievedBean = new ConnectorsBean();
retrievedBean.restore(retrievedJson);
assertEquals(connectors.size(), retrievedBean.getConnectors().size());
assertEquals(connectors.get(0), retrievedBean.getConnectors().get(0));
assertEquals(connectors.get(1), retrievedBean.getConnectors().get(1));
}
}

4
common/src/test/java/org/apache/sqoop/json/TestDriverBean.java
View File

@ -17,11 +17,11 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getResourceBundle;
import static org.junit.Assert.assertEquals;
import java.util.ResourceBundle;
import org.apache.sqoop.json.util.ConfigTestUtil;
import org.apache.sqoop.model.MDriver;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
@ -41,7 +41,7 @@ public void testSerialization() {
MDriver driver = new MDriver(ConfigTestUtil.getDriverConfig(), DriverBean.CURRENT_DRIVER_VERSION);
// Serialize it to JSON object
DriverBean bean = new DriverBean(driver, getResourceBundle());
DriverBean bean = new DriverBean(driver, ConfigTestUtil.getResourceBundle());
JSONObject json = bean.extract(false);
// "Move" it across network in text form

39
common/src/test/java/org/apache/sqoop/json/TestJobBean.java
View File

@ -17,12 +17,12 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getJob;
import static org.junit.Assert.assertEquals;
import java.util.Date;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONObject;
@ -30,27 +30,19 @@
import org.json.simple.parser.ParseException;
import org.junit.Test;
/**
*
*/
public class TestJobBean {
@Test
public void testSerialization() throws ParseException {
public void testJobSerialization() throws ParseException {
Date created = new Date();
Date updated = new Date();
MJob job = getJob("ahoj");
job.setName("The big job");
job.setPersistenceId(666);
job.setCreationDate(created);
job.setLastUpdateDate(updated);
job.setEnabled(false);
MJob job = BeanTestUtil.createJob("ahoj", "The big Job", 22L, created, updated);
// Fill some data at the beginning
MStringInput input = (MStringInput) job.getJobConfig(Direction.FROM)
.getConfigs().get(0).getInputs().get(0);
MStringInput input = (MStringInput) job.getJobConfig(Direction.FROM).getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
input = (MStringInput) job.getJobConfig(Direction.TO)
.getConfigs().get(0).getInputs().get(0);
input = (MStringInput) job.getJobConfig(Direction.TO).getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
@ -61,28 +53,31 @@ public void testSerialization() throws ParseException {
String jobJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedJobJson = (JSONObject)JSONValue.parse(jobJsonString);
JSONObject parsedJobJson = (JSONObject) JSONValue.parse(jobJsonString);
JobBean parsedJobBean = new JobBean();
parsedJobBean.restore(parsedJobJson);
MJob target = parsedJobBean.getJobs().get(0);
// Check id and name
assertEquals(666, target.getPersistenceId());
assertEquals(22L, target.getPersistenceId());
assertEquals("The big Job", target.getName());
assertEquals(target.getLinkId(Direction.FROM), 1);
assertEquals(target.getLinkId(Direction.TO), 2);
assertEquals(target.getConnectorId(Direction.FROM), 1);
assertEquals(target.getConnectorId(Direction.TO), 2);
assertEquals("The big job", target.getName());
assertEquals(created, target.getCreationDate());
assertEquals(updated, target.getLastUpdateDate());
assertEquals(false, target.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) target.getJobConfig(Direction.FROM)
.getConfigs().get(0).getInputs().get(0);
MStringInput targetInput = (MStringInput) target.getJobConfig(Direction.FROM).getConfigs()
.get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) target.getJobConfig(Direction.TO)
.getConfigs().get(0).getInputs().get(0);
targetInput = (MStringInput) target.getJobConfig(Direction.TO).getConfigs().get(0).getInputs()
.get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}

92
common/src/test/java/org/apache/sqoop/json/TestJobsBean.java Normal file
View File

@ -0,0 +1,92 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
import org.junit.Test;
public class TestJobsBean {
@Test
public void testJobsSerialization() throws ParseException {
Date created = new Date();
Date updated = new Date();
MJob job1 = BeanTestUtil.createJob("ahoj", "The big Job", 22L, created, updated);
MJob job2 = BeanTestUtil.createJob("ahoj", "The small Job", 44L, created, updated);
List<MJob> jobs = new ArrayList<MJob>();
jobs.add(job1);
jobs.add(job2);
// Fill some data at the beginning
MStringInput input = (MStringInput) job1.getJobConfig(Direction.FROM).getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
input = (MStringInput) job1.getJobConfig(Direction.TO).getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
JobsBean jobsBean = new JobsBean(jobs);
JSONObject json = jobsBean.extract(false);
// "Move" it across network in text form
String jobJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedJobsJson = (JSONObject) JSONValue.parse(jobJsonString);
JobsBean parsedJobsBean = new JobsBean();
parsedJobsBean.restore(parsedJobsJson);
MJob retrievedJob1 = parsedJobsBean.getJobs().get(0);
MJob retrievedJob2 = parsedJobsBean.getJobs().get(1);
// Check id and name
assertEquals(22L, retrievedJob1.getPersistenceId());
assertEquals("The big Job", retrievedJob1.getName());
assertEquals(44L, retrievedJob2.getPersistenceId());
assertEquals("The small Job", retrievedJob2.getName());
assertEquals(retrievedJob1.getLinkId(Direction.FROM), 1);
assertEquals(retrievedJob1.getLinkId(Direction.TO), 2);
assertEquals(retrievedJob1.getConnectorId(Direction.FROM), 1);
assertEquals(retrievedJob1.getConnectorId(Direction.TO), 2);
assertEquals(created, retrievedJob1.getCreationDate());
assertEquals(updated, retrievedJob1.getLastUpdateDate());
assertEquals(false, retrievedJob1.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) retrievedJob1.getJobConfig(Direction.FROM)
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) retrievedJob1.getJobConfig(Direction.TO).getConfigs().get(0)
.getInputs().get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}

94
common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
View File

@ -17,13 +17,13 @@
*/
package org.apache.sqoop.json;
import static org.apache.sqoop.json.ConfigTestUtil.getLink;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.Date;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigInputConstants;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MStringInput;
@ -32,26 +32,17 @@
import org.json.simple.JSONValue;
import org.junit.Test;
/**
*
*/
public class TestLinkBean {
@Test
public void testSerialization() {
public void testLinkSerialization() {
Date created = new Date();
Date updated = new Date();
MLink link = getLink("ahoj");
link.setName("Connection");
link.setPersistenceId(666);
link.setCreationUser("admin");
link.setCreationDate(created);
link.setLastUpdateUser("user");
link.setLastUpdateDate(updated);
link.setEnabled(false);
MLink link = BeanTestUtil.createLink("ahoj", "link1", 22L, created, updated);
// Fill some data at the beginning
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
// Serialize it to JSON object
@ -59,13 +50,12 @@ public void testSerialization() {
JSONObject json = linkBean.extract(false);
// Check for sensitivity
JSONArray linkArray = (JSONArray)json.get(LinkBean.LINK);
JSONObject linkItem = (JSONObject)linkArray.get(0);
JSONArray connectors = (JSONArray)linkItem.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject connector = (JSONObject)connectors.get(0);
JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
for (Object input1 : inputs) {
assertTrue(((JSONObject)input1).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE));
JSONObject linkObj = (JSONObject) json.get(LinkBean.LINK);
JSONArray linkConfigs = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject linkConfig = (JSONObject) linkConfigs.get(0);
JSONArray inputs = (JSONArray) linkConfig.get(ConfigInputConstants.CONFIG_INPUTS);
for (Object in : inputs) {
assertTrue(((JSONObject) in).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE));
}
// "Move" it across network in text form
@ -75,39 +65,33 @@ public void testSerialization() {
JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString);
LinkBean retrievedBean = new LinkBean();
retrievedBean.restore(parsedLinkJson);
MLink target = retrievedBean.getLinks().get(0);
MLink retrievedLink = retrievedBean.getLinks().get(0);
// Check id and name
assertEquals(666, target.getPersistenceId());
assertEquals("Connection", target.getName());
assertEquals("admin", target.getCreationUser());
assertEquals(created, target.getCreationDate());
assertEquals("user", target.getLastUpdateUser());
assertEquals(updated, target.getLastUpdateDate());
assertEquals(false, target.getEnabled());
assertEquals(22L, retrievedLink.getPersistenceId());
assertEquals("link1", retrievedLink.getName());
assertEquals("admin", retrievedLink.getCreationUser());
assertEquals(created, retrievedLink.getCreationDate());
assertEquals("user", retrievedLink.getLastUpdateUser());
assertEquals(updated, retrievedLink.getLastUpdateDate());
assertEquals(false, retrievedLink.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) target.getConnectorLinkConfig()
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
MStringInput retrievedLinkInput = (MStringInput) retrievedLink.getConnectorLinkConfig().getConfigs().get(0)
.getInputs().get(0);
assertEquals("Hi there!", retrievedLinkInput.getValue());
}
@Test
public void testSensitivityFilter() {
Date created = new Date();
Date updated = new Date();
MLink link = getLink("ahoj");
link.setName("Connection");
link.setPersistenceId(666);
link.setCreationUser("admin");
link.setCreationDate(created);
link.setLastUpdateUser("user");
link.setLastUpdateDate(updated);
link.setEnabled(true);
MLink link = BeanTestUtil.createLink("ahoj", "link1", 22L, created, updated);
// Fill some data at the beginning
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
MStringInput input = (MStringInput) link.getConnectorLinkConfig().getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
// Serialize it to JSON object
@ -116,25 +100,23 @@ public void testSensitivityFilter() {
JSONObject jsonFiltered = bean.extract(true);
// Sensitive values should exist
JSONArray all = (JSONArray)json.get(LinkBean.LINK);
JSONObject allItem = (JSONObject)all.get(0);
JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject connector = (JSONObject)connectors.get(0);
JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
JSONObject linkObj = (JSONObject) json.get(LinkBean.LINK);
JSONArray linkConfigsObj = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject linkConfigObj = (JSONObject) linkConfigsObj.get(0);
JSONArray inputs = (JSONArray) linkConfigObj.get(ConfigInputConstants.CONFIG_INPUTS);
assertEquals(3, inputs.size());
// Inputs are ordered when creating link
JSONObject password = (JSONObject)inputs.get(2);
JSONObject password = (JSONObject) inputs.get(2);
assertTrue(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE));
// Sensitive values should not exist
all = (JSONArray)jsonFiltered.get(LinkBean.LINK);
allItem = (JSONObject)all.get(0);
connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG_VALUES);
connector = (JSONObject)connectors.get(0);
inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
linkObj = (JSONObject) jsonFiltered.get(LinkBean.LINK);
linkConfigsObj = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
linkConfigObj = (JSONObject) linkConfigsObj.get(0);
inputs = (JSONArray) linkConfigObj.get(ConfigInputConstants.CONFIG_INPUTS);
assertEquals(3, inputs.size());
// Inputs are ordered when creating link
password = (JSONObject)inputs.get(2);
password = (JSONObject) inputs.get(2);
assertFalse(password.containsKey(ConfigInputConstants.CONFIG_INPUT_VALUE));
}
}
}

98
common/src/test/java/org/apache/sqoop/json/TestLinksBean.java Normal file
View File

@ -0,0 +1,98 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.json.util.ConfigInputConstants;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.junit.Test;
public class TestLinksBean {
@Test
public void testLinksSerialization() {
Date created = new Date();
Date updated = new Date();
MLink link1 = BeanTestUtil.createLink("ahoj", "link1", 666L, created, updated);
MLink link2 = BeanTestUtil.createLink("jhoa", "link2", 888L, created, updated);
List<MLink> links = new ArrayList<MLink>();
links.add(link1);
links.add(link2);
// Fill some data at the beginning
MStringInput input = (MStringInput) link1.getConnectorLinkConfig().getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
// Serialize it to JSON object
LinksBean linkBean = new LinksBean(links);
JSONObject json = linkBean.extract(false);
// Check for sensitivity
JSONArray linksObj = (JSONArray) json.get(LinksBean.LINKS);
JSONObject linkObj = (JSONObject) linksObj.get(0);
JSONArray linkConfigs = (JSONArray) linkObj.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject linkConfig = (JSONObject) linkConfigs.get(0);
JSONArray inputs = (JSONArray) linkConfig.get(ConfigInputConstants.CONFIG_INPUTS);
for (Object inp : inputs) {
assertTrue(((JSONObject) inp).containsKey(ConfigInputConstants.CONFIG_INPUT_SENSITIVE));
}
// "Move" it across network in text form
String linkJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedLinkJson = (JSONObject) JSONValue.parse(linkJsonString);
LinksBean retrievedBean = new LinksBean();
retrievedBean.restore(parsedLinkJson);
MLink targetLink1 = retrievedBean.getLinks().get(0);
MLink targetLink2 = retrievedBean.getLinks().get(1);
// Check id and name
assertEquals(666L, targetLink1.getPersistenceId());
assertEquals(888L, targetLink2.getPersistenceId());
assertEquals("link1", targetLink1.getName());
assertEquals("link2", targetLink2.getName());
assertEquals("admin", targetLink1.getCreationUser());
assertEquals(created, targetLink1.getCreationDate());
assertEquals("user", targetLink1.getLastUpdateUser());
assertEquals(updated, targetLink1.getLastUpdateDate());
assertEquals(false, targetLink1.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) targetLink1.getConnectorLinkConfig().getConfigs()
.get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
}
}

4
common/src/test/java/org/apache/sqoop/json/TestSubmissionBean.java
View File

@ -36,9 +36,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
/**
*
*/
// NOTE: This tests both the submission and submissions list bean
public class TestSubmissionBean {
private static final double EPSILON = 0.01;

83
common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java Normal file
View File

@ -0,0 +1,83 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json.util;
import java.util.Date;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MToConfig;
public class BeanTestUtil {
public static MLink createLink(String connectorName, String linkName, Long linkId, Date created,
Date updated) {
MLink link1 = getLink(connectorName);
link1.setName(linkName);
link1.setPersistenceId(linkId);
link1.setCreationUser("admin");
link1.setCreationDate(created);
link1.setLastUpdateUser("user");
link1.setLastUpdateDate(updated);
link1.setEnabled(false);
return link1;
}
public static MJob createJob(String connectorName, String jobName, Long jobId, Date created, Date updated) {
MJob job = BeanTestUtil.getJob(connectorName);
job.setName(jobName);
job.setPersistenceId(jobId);
job.setCreationDate(created);
job.setLastUpdateDate(updated);
job.setEnabled(false);
return job;
}
public static MLink getLink(String connectorName) {
return new MLink(1, getConnector(1L, connectorName).getLinkConfig());
}
public static MConnector getConnector(Long connectorId, String connectorName) {
return getConnector(connectorId, connectorName, true, true);
}
public static MConnector getConnector(Long id, String name, boolean from, boolean to) {
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (from) {
fromConfig = ConfigTestUtil.getFromConfig();
}
if (to) {
toConfig = ConfigTestUtil.getToConfig();
}
MConnector connector = new MConnector(name, name + ".class", "1.0-test",
ConfigTestUtil.getLinkConfig(), fromConfig, toConfig);
// simulate a persistence id
connector.setPersistenceId(id);
return connector;
}
public static MJob getJob(String connectorName) {
return new MJob(1, 2, 1, 2, getConnector(1L, connectorName).getFromConfig(), getConnector(1L, connectorName)
.getToConfig(), ConfigTestUtil.getDriverConfig());
}
}

38
common/src/test/java/org/apache/sqoop/json/ConfigTestUtil.java → common/src/test/java/org/apache/sqoop/json/util/ConfigTestUtil.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
package org.apache.sqoop.json.util;
import java.util.ArrayList;
import java.util.HashMap;
@ -24,52 +24,16 @@
import java.util.ResourceBundle;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.utils.MapResourceBundle;
/**
*
*/
public class ConfigTestUtil {
public static MConnector getConnector(Long id, String name) {
return getConnector(id, name, true, true);
}
public static MConnector getConnector(Long id, String name, boolean from, boolean to) {
MFromConfig fromConfig = null;
MToConfig toConfig = null;
if (from) {
fromConfig = getFromConfig();
}
if (to) {
toConfig = getToConfig();
}
MConnector connector = new MConnector(name, name + ".class", "1.0-test",
getLinkConfig(), fromConfig, toConfig);
// simulate a persistence id
connector.setPersistenceId(id);
return connector;
}
public static MLink getLink(String name) {
return new MLink(1, getConnector(1L, name).getLinkConfig());
}
public static MJob getJob(String name) {
return new MJob(1, 2, 1, 2, getConnector(1L, name).getFromConfig(), getConnector(1L, name)
.getToConfig(), getDriverConfig());
}
public static MDriverConfig getDriverConfig() {
List<MInput<?>> inputs;