github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-07 20:03:16 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1686: Sqoop2: Fix the JSON names for config and config values

Browse Source 
(Veena Basavaraj via Abraham Elmahrek)
This commit is contained in:
Abraham Elmahrek 2014-11-06 10:18:42 -08:00
parent 500fab287b
commit d741c9d20a
6 changed files with 33 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
View File

@ -44,8 +44,8 @@
public class ConnectorBean extends ConfigurableBean { public class ConnectorBean extends ConfigurableBean {
// to represent the config and inputs with values // to represent the config and inputs with values
public static final String CONNECTOR_LINK_CONFIG_VALUES = "link-config-values"; public static final String CONNECTOR_LINK_CONFIG = "link-config";
public static final String CONNECTOR_JOB_CONFIG_VALUES = "job-config-values"; public static final String CONNECTOR_JOB_CONFIG = "job-config";
private static final String CONNECTOR = "connector"; private static final String CONNECTOR = "connector";
private List<MConnector> connectors; private List<MConnector> connectors;
@ -88,28 +88,28 @@ protected JSONArray extractConnectors(boolean skipSensitive) {
connectorJsonObject.put(CLASS, connector.getClassName()); connectorJsonObject.put(CLASS, connector.getClassName());
connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion()); connectorJsonObject.put(CONFIGURABLE_VERSION, connector.getVersion());
connectorJsonObject.put( connectorJsonObject.put(
CONNECTOR_LINK_CONFIG_VALUES, CONNECTOR_LINK_CONFIG,
extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig() extractConfigList(connector.getLinkConfig().getConfigs(), connector.getLinkConfig()
.getType(), skipSensitive)); .getType(), skipSensitive));
connectorJsonObject.put(CONNECTOR_JOB_CONFIG_VALUES, new JSONObject()); connectorJsonObject.put(CONNECTOR_JOB_CONFIG, new JSONObject());
// add sub fields to the job config for from and to // add sub fields to the job config for from and to
if (connector.getFromConfig() != null) { if (connector.getFromConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG_VALUES)).put( ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.FROM, Direction.FROM,
extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig() extractConfigList(connector.getFromConfig().getConfigs(), connector.getFromConfig()
.getType(), skipSensitive)); .getType(), skipSensitive));
} }
if (connector.getToConfig() != null) { if (connector.getToConfig() != null) {
((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG_VALUES)).put( ((JSONObject) connectorJsonObject.get(CONNECTOR_JOB_CONFIG)).put(
Direction.TO, Direction.TO,
extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig() extractConfigList(connector.getToConfig().getConfigs(), connector.getToConfig()
.getType(), skipSensitive)); .getType(), skipSensitive));
} }
// add the config-param inside each connector // add the config-param inside each connector
connectorJsonObject.put(ALL_CONFIGS, new JSONObject()); connectorJsonObject.put(ALL_CONFIG_RESOURCES, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) { if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIGS, connectorJsonObject.put(ALL_CONFIG_RESOURCES,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId()))); extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId())));
} }
connectorArray.add(connectorJsonObject); connectorArray.add(connectorJsonObject);
@ -136,10 +136,10 @@ protected void restoreConnectors(JSONArray array) {
String version = (String) object.get(CONFIGURABLE_VERSION); String version = (String) object.get(CONFIGURABLE_VERSION);
List<MConfig> linkConfigs = restoreConfigList((JSONArray) object List<MConfig> linkConfigs = restoreConfigList((JSONArray) object
.get(CONNECTOR_LINK_CONFIG_VALUES)); .get(CONNECTOR_LINK_CONFIG));
// parent that encapsulates both the from/to configs // parent that encapsulates both the from/to configs
JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG_VALUES); JSONObject jobConfigJson = (JSONObject) object.get(CONNECTOR_JOB_CONFIG);
JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name()); JSONArray fromJobConfigJson = (JSONArray) jobConfigJson.get(Direction.FROM.name());
JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name()); JSONArray toJobConfigJson = (JSONArray) jobConfigJson.get(Direction.TO.name());
@ -161,9 +161,9 @@ protected void restoreConnectors(JSONArray array) {
toConfig); toConfig);
connector.setPersistenceId(connectorId); connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIGS)) { if (object.containsKey(ALL_CONFIG_RESOURCES)) {
JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIGS); JSONObject jsonConfigBundle = (JSONObject) object.get(ALL_CONFIG_RESOURCES);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle)); connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle));
} }
connectors.add(connector); connectors.add(connector);

10
common/src/main/java/org/apache/sqoop/json/DriverBean.java
View File

@ -36,7 +36,7 @@
public class DriverBean extends ConfigurableBean { public class DriverBean extends ConfigurableBean {
public static final String CURRENT_DRIVER_VERSION = "1"; public static final String CURRENT_DRIVER_VERSION = "1";
static final String DRIVER_JOB_CONFIG_VALUES = "job-config-values"; static final String DRIVER_JOB_CONFIG = "job-config";
private MDriver driver; private MDriver driver;
private ResourceBundle driverConfigBundle; private ResourceBundle driverConfigBundle;
@ -68,8 +68,8 @@ public JSONObject extract(boolean skipSensitive) {
JSONObject result = new JSONObject(); JSONObject result = new JSONObject();
result.put(ID, driver.getPersistenceId()); result.put(ID, driver.getPersistenceId());
result.put(CONFIGURABLE_VERSION, driver.getVersion()); result.put(CONFIGURABLE_VERSION, driver.getVersion());
result.put(DRIVER_JOB_CONFIG_VALUES, configs); result.put(DRIVER_JOB_CONFIG, configs);
result.put(ALL_CONFIGS, extractConfigParamBundle(driverConfigBundle)); result.put(ALL_CONFIG_RESOURCES, extractConfigParamBundle(driverConfigBundle));
return result; return result;
} }
@ -77,9 +77,9 @@ public JSONObject extract(boolean skipSensitive) {
public void restore(JSONObject jsonObject) { public void restore(JSONObject jsonObject) {
long id = (Long) jsonObject.get(ID); long id = (Long) jsonObject.get(ID);
String driverVersion = (String) jsonObject.get(CONFIGURABLE_VERSION); String driverVersion = (String) jsonObject.get(CONFIGURABLE_VERSION);
List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_JOB_CONFIG_VALUES)); List<MConfig> driverConfig = restoreConfigList((JSONArray) jsonObject.get(DRIVER_JOB_CONFIG));
driver = new MDriver(new MDriverConfig(driverConfig), driverVersion); driver = new MDriver(new MDriverConfig(driverConfig), driverVersion);
driver.setPersistenceId(id); driver.setPersistenceId(id);
driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIGS)); driverConfigBundle = restoreConfigParamBundle((JSONObject) jsonObject.get(ALL_CONFIG_RESOURCES));
} }
} }

18
common/src/main/java/org/apache/sqoop/json/JobBean.java
View File

@ -45,9 +45,9 @@ public class JobBean implements JsonBean {
static final String TO_LINK_ID = "to-link-id"; static final String TO_LINK_ID = "to-link-id";
static final String FROM_CONNECTOR_ID = "from-connector-id"; static final String FROM_CONNECTOR_ID = "from-connector-id";
static final String TO_CONNECTOR_ID = "to-connector-id"; static final String TO_CONNECTOR_ID = "to-connector-id";
static final String FROM_CONFIG = "from-config"; static final String FROM_CONFIG_VALUES = "from-config-values";
static final String TO_CONFIG = "to-config"; static final String TO_CONFIG_VALUES = "to-config-values";
static final String DRIVER_CONFIG = "driver-config"; static final String DRIVER_CONFIG_VALUES = "driver-config-values";
private static final String JOB = "job"; private static final String JOB = "job";
// Required // Required
@ -129,14 +129,14 @@ protected JSONArray extractJobs(boolean skipSensitive) {
object.put(TO_LINK_ID, job.getLinkId(Direction.TO)); object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
// job configs // job configs
MFromConfig fromConfigList = job.getFromJobConfig(); MFromConfig fromConfigList = job.getFromJobConfig();
object.put(FROM_CONFIG, object.put(FROM_CONFIG_VALUES,
extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive)); extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive));
MToConfig toConfigList = job.getToJobConfig(); MToConfig toConfigList = job.getToJobConfig();
object.put(TO_CONFIG, object.put(TO_CONFIG_VALUES,
extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive)); extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive));
MDriverConfig driverConfigList = job.getDriverConfig(); MDriverConfig driverConfigList = job.getDriverConfig();
object.put( object.put(
DRIVER_CONFIG, DRIVER_CONFIG_VALUES,
extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(), extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(),
skipSensitive)); skipSensitive));
@ -161,9 +161,9 @@ protected void restoreJobs(JSONArray array) {
long toConnectorId = (Long) object.get(TO_CONNECTOR_ID); long toConnectorId = (Long) object.get(TO_CONNECTOR_ID);
long fromConnectionId = (Long) object.get(FROM_LINK_ID); long fromConnectionId = (Long) object.get(FROM_LINK_ID);
long toConnectionId = (Long) object.get(TO_LINK_ID); long toConnectionId = (Long) object.get(TO_LINK_ID);
JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG); JSONArray fromConfigJson = (JSONArray) object.get(FROM_CONFIG_VALUES);
JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG); JSONArray toConfigJson = (JSONArray) object.get(TO_CONFIG_VALUES);
JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG); JSONArray driverConfigJson = (JSONArray) object.get(DRIVER_CONFIG_VALUES);
List<MConfig> fromConfig = restoreConfigList(fromConfigJson); List<MConfig> fromConfig = restoreConfigList(fromConfigJson);
List<MConfig> toConfig = restoreConfigList(toConfigJson); List<MConfig> toConfig = restoreConfigList(toConfigJson);

2
common/src/main/java/org/apache/sqoop/json/JsonBean.java
View File

@ -23,7 +23,7 @@ public interface JsonBean {
// common JSON constants for the rest-api response // common JSON constants for the rest-api response
static final String CONFIGURABLE_VERSION = "version"; static final String CONFIGURABLE_VERSION = "version";
static final String ALL_CONFIGS = "all-configs"; static final String ALL_CONFIG_RESOURCES= "all-config-resources";
@Deprecated // should not be used anymore in the rest api @Deprecated // should not be used anymore in the rest api
static final String ALL = "all"; static final String ALL = "all";

6
common/src/main/java/org/apache/sqoop/json/LinkBean.java
View File

@ -42,7 +42,7 @@
public class LinkBean implements JsonBean { public class LinkBean implements JsonBean {
static final String CONNECTOR_ID = "connector-id"; static final String CONNECTOR_ID = "connector-id";
static final String LINK_CONFIG = "link-config"; static final String LINK_CONFIG_VALUES = "link-config-values";
static final String LINK = "link"; static final String LINK = "link";
@ -108,7 +108,7 @@ protected JSONArray extractLinks(boolean skipSensitive) {
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser()); linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime()); linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId()); linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(LINK_CONFIG, linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive)); extractConfigList(link.getConnectorLinkConfig().getConfigs(), link.getConnectorLinkConfig().getType(), skipSensitive));
linkArray.add(linkJsonObject); linkArray.add(linkJsonObject);
} }
@ -126,7 +126,7 @@ protected void restoreLinks(JSONArray array) {
for (Object obj : array) { for (Object obj : array) {
JSONObject object = (JSONObject) obj; JSONObject object = (JSONObject) obj;
long connectorId = (Long) object.get(CONNECTOR_ID); long connectorId = (Long) object.get(CONNECTOR_ID);
JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG); JSONArray connectorLinkConfig = (JSONArray) object.get(LINK_CONFIG_VALUES);
List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig); List<MConfig> linkConfig = restoreConfigList(connectorLinkConfig);
MLink link = new MLink(connectorId, new MLinkConfig(linkConfig)); MLink link = new MLink(connectorId, new MLinkConfig(linkConfig));
link.setPersistenceId((Long) object.get(ID)); link.setPersistenceId((Long) object.get(ID));

6
common/src/test/java/org/apache/sqoop/json/TestLinkBean.java
View File

@ -61,7 +61,7 @@ public void testSerialization() {
// Check for sensitivity // Check for sensitivity
JSONArray linkArray = (JSONArray)json.get(LinkBean.LINK); JSONArray linkArray = (JSONArray)json.get(LinkBean.LINK);
JSONObject linkItem = (JSONObject)linkArray.get(0); JSONObject linkItem = (JSONObject)linkArray.get(0);
JSONArray connectors = (JSONArray)linkItem.get(LinkBean.LINK_CONFIG); JSONArray connectors = (JSONArray)linkItem.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject connector = (JSONObject)connectors.get(0); JSONObject connector = (JSONObject)connectors.get(0);
JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
for (Object input1 : inputs) { for (Object input1 : inputs) {
@ -118,7 +118,7 @@ public void testSensitivityFilter() {
// Sensitive values should exist // Sensitive values should exist
JSONArray all = (JSONArray)json.get(LinkBean.LINK); JSONArray all = (JSONArray)json.get(LinkBean.LINK);
JSONObject allItem = (JSONObject)all.get(0); JSONObject allItem = (JSONObject)all.get(0);
JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG); JSONArray connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG_VALUES);
JSONObject connector = (JSONObject)connectors.get(0); JSONObject connector = (JSONObject)connectors.get(0);
JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); JSONArray inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
assertEquals(3, inputs.size()); assertEquals(3, inputs.size());
@ -129,7 +129,7 @@ public void testSensitivityFilter() {
// Sensitive values should not exist // Sensitive values should not exist
all = (JSONArray)jsonFiltered.get(LinkBean.LINK); all = (JSONArray)jsonFiltered.get(LinkBean.LINK);
allItem = (JSONObject)all.get(0); allItem = (JSONObject)all.get(0);
connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG); connectors = (JSONArray)allItem.get(LinkBean.LINK_CONFIG_VALUES);
connector = (JSONObject)connectors.get(0); connector = (JSONObject)connectors.get(0);
inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS); inputs = (JSONArray)connector.get(ConfigInputConstants.CONFIG_INPUTS);
assertEquals(3, inputs.size()); assertEquals(3, inputs.size());