From 794adbcf3d4c73bae2ee428041efedebf2e06ab5 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Mon, 7 Dec 2015 11:18:12 +0100 Subject: [PATCH] SQOOP-2690: Sqoop2: Use connector name in MLink. (Colin Ma via Jarek Jarcec Cecho) --- .../org/apache/sqoop/client/SqoopClient.java | 85 ++++++------------- .../request/ConnectorResourceRequest.java | 8 +- .../client/request/SqoopResourceRequests.java | 4 +- .../apache/sqoop/client/TestSqoopClient.java | 51 +++++------ .../org/apache/sqoop/json/ConnectorBean.java | 14 +-- .../org/apache/sqoop/json/ConnectorsBean.java | 2 +- .../java/org/apache/sqoop/json/LinkBean.java | 24 +++--- .../java/org/apache/sqoop/model/MLink.java | 18 ++-- .../apache/sqoop/json/TestConnectorBean.java | 6 +- .../apache/sqoop/json/TestConnectorsBean.java | 20 ++--- .../apache/sqoop/json/util/BeanTestUtil.java | 2 +- .../sqoop/model/TestMAccountableEntity.java | 2 +- .../org/apache/sqoop/model/TestMLink.java | 6 +- .../sqoop/connector/ConnectorManager.java | 8 +- .../org/apache/sqoop/driver/JobManager.java | 18 ++-- .../apache/sqoop/driver/TestJobManager.java | 4 +- .../sqoop/repository/TestJdbcRepository.java | 18 ++-- .../common/CommonRepositoryHandler.java | 12 ++- .../repository/derby/TestInputTypes.java | 2 +- .../repository/derby/TestLinkHandling.java | 2 +- .../repository/mysql/MySqlTestCase.java | 2 +- .../postgresql/PostgresqlTestCase.java | 2 +- .../handler/ConnectorRequestHandler.java | 4 +- .../sqoop/handler/LinkRequestHandler.java | 14 +-- .../apache/sqoop/shell/CloneJobFunction.java | 8 +- .../apache/sqoop/shell/CloneLinkFunction.java | 2 +- .../apache/sqoop/shell/CreateJobFunction.java | 4 +- .../sqoop/shell/CreateLinkFunction.java | 7 +- .../sqoop/shell/ShowConnectorFunction.java | 2 +- .../apache/sqoop/shell/ShowJobFunction.java | 9 +- .../apache/sqoop/shell/ShowLinkFunction.java | 40 +-------- .../apache/sqoop/shell/UpdateJobFunction.java | 9 +- .../sqoop/shell/UpdateLinkFunction.java | 3 +- .../apache/sqoop/shell/TestCloneCommand.java | 33 ++----- .../apache/sqoop/shell/TestCreateCommand.java | 9 +- .../apache/sqoop/shell/TestShowCommand.java | 3 +- .../apache/sqoop/shell/TestUpdateCommand.java | 20 +++-- .../sqoop/test/testcases/JettyTestCase.java | 1 - .../integration/connector/hdfs/S3Test.java | 6 +- .../sqoop/tools/tool/RepositoryLoadTool.java | 4 +- 40 files changed, 212 insertions(+), 276 deletions(-) diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java index 3d3425d1..9b9359b2 100644 --- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java +++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java @@ -61,11 +61,11 @@ public class SqoopClient { /** * All cached connectors. */ - private Map connectors; + private Map connectors; /** * All cached config params for every registered connector in the sqoop system. */ - private Map connectorConfigBundles; + private Map connectorConfigBundles; /** * Cached driver. @@ -122,25 +122,22 @@ public void setSqoopRequests(SqoopResourceRequests requests) { * Clear internal cache. */ public void clearCache() { - connectorConfigBundles = new HashMap(); + connectorConfigBundles = new HashMap(); driverConfigBundle = null; - connectors = new HashMap(); + connectors = new HashMap(); mDriver = null; isAllConnectors = false; } /** * Get connector with given id. + * TODO: This method should be removed when MJob link with Connector by name. * * @param cid Connector id. * @return */ public MConnector getConnector(long cid) { - if(connectors.containsKey(cid)) { - return connectors.get(cid).clone(false); - } - retrieveConnector(cid); - return connectors.get(cid).clone(false); + return retrieveConnector(Long.toString(cid)); } /** @@ -183,13 +180,17 @@ private MConnector getConnectorFromCache(String connectorName) { /** * Retrieve connector structure from server and cache it. + * TODO: The method support both connector name/id, this should support name only when MJob link with Connector by name. * - * @param cid Connector id + * @param connectorIdentify Connector name/id */ - private void retrieveConnector(long cid) { - ConnectorBean request = resourceRequests.readConnector(cid); - connectors.put(cid, request.getConnectors().get(0)); - connectorConfigBundles.put(cid, request.getResourceBundles().get(cid)); + private MConnector retrieveConnector(String connectorIdentify) { + ConnectorBean request = resourceRequests.readConnector(connectorIdentify); + MConnector mConnector = request.getConnectors().get(0); + String connectorName = mConnector.getUniqueName(); + connectors.put(connectorName, mConnector); + connectorConfigBundles.put(connectorName, request.getResourceBundles().get(connectorName)); + return mConnector; } /** @@ -205,7 +206,7 @@ public Collection getConnectors() { ConnectorBean bean = resourceRequests.readConnector(null); isAllConnectors = true; for(MConnector connector : bean.getConnectors()) { - connectors.put(connector.getPersistenceId(), connector); + connectors.put(connector.getUniqueName(), connector); } connectorConfigBundles = bean.getResourceBundles(); @@ -218,12 +219,12 @@ public Collection getConnectors() { * @param connectorId Connector id. * @return */ - public ResourceBundle getConnectorConfigBundle(long connectorId) { - if(connectorConfigBundles.containsKey(connectorId)) { - return connectorConfigBundles.get(connectorId); + public ResourceBundle getConnectorConfigBundle(String connectorName) { + if(connectorConfigBundles.containsKey(connectorName)) { + return connectorConfigBundles.get(connectorName); } - retrieveConnector(connectorId); - return connectorConfigBundles.get(connectorId); + retrieveConnector(connectorName); + return connectorConfigBundles.get(connectorName); } /** @@ -275,16 +276,6 @@ public ResourceBundle getDriverConfigBundle() { return driverConfigBundle; } - /** - * Create new link object for given connector id - * - * @param connectorId Connector id - * @return - */ - public MLink createLink(long connectorId) { - return new MLink(connectorId, getConnector(connectorId).getLinkConfig()); - } - /** * Create new link object for given connector name * @@ -296,7 +287,7 @@ public MLink createLink(String connectorName) { if (connector == null) { throw new SqoopException(ClientError.CLIENT_0003, connectorName); } - return createLink(connector.getPersistenceId()); + return new MLink(connectorName, getConnector(connectorName).getLinkConfig()); } @@ -398,40 +389,20 @@ public void deleteLink(long linkId) { public MJob createJob(String fromLinkName, String toLinkName) { MLink fromLink = getLink(fromLinkName); MLink toLink = getLink(toLinkName); + MConnector connectorForFromLink = getConnector(fromLink.getConnectorName()); + MConnector connectorForToLink = getConnector(toLink.getConnectorName()); return new MJob( - fromLink.getConnectorId(), - toLink.getConnectorId(), + connectorForFromLink.getPersistenceId(), + connectorForToLink.getPersistenceId(), fromLink.getPersistenceId(), toLink.getPersistenceId(), - getConnector(fromLink.getConnectorId()).getFromConfig(), - getConnector(toLink.getConnectorId()).getToConfig(), + connectorForFromLink.getFromConfig().clone(false), + connectorForToLink.getToConfig().clone(false), getDriverConfig() ); } - /** - * Create new job the for given links. - * - * @param fromLinkId From link id - * @param toLinkId To link id - * @return - */ - public MJob createJob(long fromLinkId, long toLinkId) { - MLink fromLink = getLink(fromLinkId); - MLink toLink = getLink(toLinkId); - - return new MJob( - fromLink.getConnectorId(), - toLink.getConnectorId(), - fromLink.getPersistenceId(), - toLink.getPersistenceId(), - getConnector(fromLink.getConnectorId()).getFromConfig(), - getConnector(toLink.getConnectorId()).getToConfig(), - getDriverConfig() - ); - } - /** * Retrieve job for given id. * diff --git a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java index f23c6e4d..953b23f5 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java @@ -39,17 +39,17 @@ public ConnectorResourceRequest(DelegationTokenAuthenticatedURL.Token token){ super(token); } - public ConnectorBean read(String serverUrl, Long cid) { + public ConnectorBean read(String serverUrl, String connectorName) { String response; - if (cid == null) { + if (connectorName == null) { response = super.get(serverUrl + RESOURCE + "all"); } else { - response = super.get(serverUrl + RESOURCE + cid); + response = super.get(serverUrl + RESOURCE + connectorName); } JSONObject jsonObject = JSONUtils.parse(response); // defaults to all ConnectorBean bean = new ConnectorsBean(); - if (cid != null) { + if (connectorName != null) { bean = new ConnectorBean(); } bean.restore(jsonObject); diff --git a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java index 09877033..810b9133 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java +++ b/client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java @@ -114,8 +114,8 @@ public DriverBean readDriver() { return getDriverResourceRequest().read(serverUrl); } - public ConnectorBean readConnector(Long cid) { - return getConnectorResourceRequest().read(serverUrl, cid); + public ConnectorBean readConnector(String connectorName) { + return getConnectorResourceRequest().read(serverUrl, connectorName); } public ValidationResultBean saveLink(MLink link) { diff --git a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java index c962358a..adf4eb5a 100644 --- a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java +++ b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java @@ -65,17 +65,6 @@ public void setUp() { * Retrieve connector information, request to bundle for same connector should * not require additional HTTP request. */ - @Test - public void testGetConnector() { - when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1))); - MConnector connector = client.getConnector(1); - assertEquals(1, connector.getPersistenceId()); - - client.getConnectorConfigBundle(1L); - - verify(resourceRequests, times(1)).readConnector(1L); - } - @Test public void testGetConnectorByString() { when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1))); @@ -83,9 +72,9 @@ public void testGetConnectorByString() { assertEquals(1, connector.getPersistenceId()); assertEquals("A1", connector.getUniqueName()); - client.getConnectorConfigBundle(1L); + client.getConnectorConfigBundle("A1"); - verify(resourceRequests, times(0)).readConnector(1L); + verify(resourceRequests, times(0)).readConnector("A1"); verify(resourceRequests, times(1)).readConnector(null); } @@ -95,13 +84,13 @@ public void testGetConnectorByString() { */ @Test public void testGetConnectorBundle() { - when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1))); - client.getConnectorConfigBundle(1L); + when(resourceRequests.readConnector("A1")).thenReturn(connectorBean(connector(1))); + client.getConnectorConfigBundle("A1"); - MConnector connector = client.getConnector(1); + MConnector connector = client.getConnector("A1"); assertEquals(1, connector.getPersistenceId()); - verify(resourceRequests, times(1)).readConnector(1L); + verify(resourceRequests, times(1)).readConnector("A1"); } /** @@ -144,12 +133,12 @@ public void testGetConnectors() { Collection connectors = client.getConnectors(); assertEquals(2, connectors.size()); - client.getConnectorConfigBundle(1); - connector = client.getConnector(1); + client.getConnectorConfigBundle("A1"); + connector = client.getConnector("A1"); assertEquals(1, connector.getPersistenceId()); - connector = client.getConnector(2); - client.getConnectorConfigBundle(2); + connector = client.getConnector("A2"); + client.getConnectorConfigBundle("A2"); assertEquals(2, connector.getPersistenceId()); connectors = client.getConnectors(); @@ -179,21 +168,21 @@ public void testGetConnectors() { public void testGetConnectorOneByOne() { ConnectorBean bean = connectorBean(connector(1), connector(2)); when(resourceRequests.readConnector(null)).thenReturn(bean); - when(resourceRequests.readConnector(1L)).thenReturn(bean); - when(resourceRequests.readConnector(2L)).thenReturn(bean); + when(resourceRequests.readConnector("A1")).thenReturn(bean); + when(resourceRequests.readConnector("A2")).thenReturn(bean); - client.getConnectorConfigBundle(1); - client.getConnector(1); + client.getConnectorConfigBundle("A1"); + client.getConnector("A1"); - client.getConnector(2); - client.getConnectorConfigBundle(2); + client.getConnector("A2"); + client.getConnectorConfigBundle("A2"); Collection connectors = client.getConnectors(); assertEquals(2, connectors.size()); verify(resourceRequests, times(1)).readConnector(null); - verify(resourceRequests, times(1)).readConnector(1L); - verify(resourceRequests, times(1)).readConnector(2L); + verify(resourceRequests, times(1)).readConnector("A1"); + verify(resourceRequests, times(0)).readConnector("A2"); verifyNoMoreInteractions(resourceRequests); } @@ -208,11 +197,11 @@ public void testCreateLink() { private ConnectorBean connectorBean(MConnector...connectors) { List connectorList = new ArrayList(); - Map bundles = new HashMap(); + Map bundles = new HashMap(); for(MConnector connector : connectors) { connectorList.add(connector); - bundles.put(connector.getPersistenceId(), null); + bundles.put(connector.getUniqueName(), null); } return new ConnectorBean(connectorList, bundles); } diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java index 89ae20cd..b50fa697 100644 --- a/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java +++ b/common/src/main/java/org/apache/sqoop/json/ConnectorBean.java @@ -57,10 +57,10 @@ public class ConnectorBean extends ConfigurableBean { private static final String CONNECTOR = "connector"; private List connectors; - private Map connectorConfigBundles; + private Map connectorConfigBundles; // for "extract" - public ConnectorBean(List connectors, Map bundles) { + public ConnectorBean(List connectors, Map bundles) { this.connectors = connectors; this.connectorConfigBundles = bundles; } @@ -73,7 +73,7 @@ public List getConnectors() { return connectors; } - public Map getResourceBundles() { + public Map getResourceBundles() { return connectorConfigBundles; } @@ -121,7 +121,7 @@ private JSONObject extractConnector(boolean skipSensitive, MConnector connector) connectorJsonObject.put(ALL_CONFIGS, new JSONObject()); if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) { connectorJsonObject.put(ALL_CONFIGS, - extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId()))); + extractConfigParamBundle(connectorConfigBundles.get(connector.getUniqueName()))); } return connectorJsonObject; } @@ -129,14 +129,14 @@ private JSONObject extractConnector(boolean skipSensitive, MConnector connector) @Override public void restore(JSONObject jsonObject) { connectors = new ArrayList(); - connectorConfigBundles = new HashMap(); + connectorConfigBundles = new HashMap(); JSONObject obj = JSONUtils.getJSONObject(jsonObject, CONNECTOR); connectors.add(restoreConnector(obj)); } protected void restoreConnectors(JSONArray array) { connectors = new ArrayList(); - connectorConfigBundles = new HashMap(); + connectorConfigBundles = new HashMap(); for (Object obj : array) { connectors.add(restoreConnector(obj)); } @@ -178,7 +178,7 @@ private MConnector restoreConnector(Object obj) { connector.setPersistenceId(connectorId); if (object.containsKey(ALL_CONFIGS)) { JSONObject jsonConfigBundle = JSONUtils.getJSONObject(object, ALL_CONFIGS); - connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle)); + connectorConfigBundles.put(uniqueName, restoreConfigParamBundle(jsonConfigBundle)); } return connector; } diff --git a/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java b/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java index 9762ffad..c31e52bf 100644 --- a/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java +++ b/common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java @@ -39,7 +39,7 @@ public class ConnectorsBean extends ConnectorBean { private static final String CONNECTORS = "connectors"; // for "extract" - public ConnectorsBean(List connectors, Map bundles) { + public ConnectorsBean(List connectors, Map bundles) { super(connectors, bundles); } diff --git a/common/src/main/java/org/apache/sqoop/json/LinkBean.java b/common/src/main/java/org/apache/sqoop/json/LinkBean.java index c9a8ab3f..8ce8b2be 100644 --- a/common/src/main/java/org/apache/sqoop/json/LinkBean.java +++ b/common/src/main/java/org/apache/sqoop/json/LinkBean.java @@ -47,7 +47,7 @@ @InterfaceStability.Unstable public class LinkBean implements JsonBean { - static final String CONNECTOR_ID = "connector-id"; + static final String CONNECTOR_NAME = "connector-name"; static final String LINK_CONFIG_VALUES = "link-config-values"; static final String LINK = "link"; @@ -57,7 +57,7 @@ public class LinkBean implements JsonBean { private List links; // Optional - private Map linkConfigBundles; + private Map linkConfigBundles; // For "extract" public LinkBean(MLink link) { @@ -73,23 +73,23 @@ public LinkBean(List links) { // For "restore" public LinkBean() { - linkConfigBundles = new HashMap(); + linkConfigBundles = new HashMap(); } - public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) { - linkConfigBundles.put(id, connectorConfigBundle); + public void addConnectorConfigBundle(String connectorName, ResourceBundle connectorConfigBundle) { + linkConfigBundles.put(connectorName, connectorConfigBundle); } - public boolean hasConnectorConfigBundle(Long id) { - return linkConfigBundles.containsKey(id); + public boolean hasConnectorConfigBundle(String connectorName) { + return linkConfigBundles.containsKey(connectorName); } public List getLinks() { return links; } - public ResourceBundle getConnectorConfigBundle(Long id) { - return linkConfigBundles.get(id); + public ResourceBundle getConnectorConfigBundle(String connectorName) { + return linkConfigBundles.get(connectorName); } @SuppressWarnings("unchecked") @@ -119,7 +119,7 @@ private JSONObject extractLink(boolean skipSensitive, MLink link) { linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime()); linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser()); linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime()); - linkJsonObject.put(CONNECTOR_ID, link.getConnectorId()); + linkJsonObject.put(CONNECTOR_NAME, link.getConnectorName()); linkJsonObject.put(LINK_CONFIG_VALUES, extractConfigList(link.getConnectorLinkConfig(), skipSensitive)); return linkJsonObject; @@ -141,11 +141,11 @@ protected void restoreLinks(JSONArray array) { private MLink restoreLink(Object obj) { JSONObject object = (JSONObject) obj; - long connectorId = JSONUtils.getLong(object, CONNECTOR_ID); + String connectorName = (String) object.get(CONNECTOR_NAME); JSONObject connectorLinkConfig = JSONUtils.getJSONObject(object, LINK_CONFIG_VALUES); List linkConfigs = restoreConfigs(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIGS)); List linkValidators = restoreValidator(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIG_VALIDATORS)); - MLink link = new MLink(connectorId, new MLinkConfig(linkConfigs, linkValidators)); + MLink link = new MLink(connectorName, new MLinkConfig(linkConfigs, linkValidators)); link.setPersistenceId(JSONUtils.getLong(object, ID)); link.setName(JSONUtils.getString(object, NAME)); link.setEnabled(JSONUtils.getBoolean(object, ENABLED)); diff --git a/common/src/main/java/org/apache/sqoop/model/MLink.java b/common/src/main/java/org/apache/sqoop/model/MLink.java index 51a40081..4275242c 100644 --- a/common/src/main/java/org/apache/sqoop/model/MLink.java +++ b/common/src/main/java/org/apache/sqoop/model/MLink.java @@ -26,7 +26,7 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public class MLink extends MAccountableEntity implements MClonable { - private long connectorId; + private String connectorName; // NOTE: we hold this in the model for easy access to the link config object, it might as well be retrieved on the fly using the connectorId private final MLinkConfig connectorLinkConfig; @@ -36,8 +36,8 @@ public class MLink extends MAccountableEntity implements MClonable { * @param connectorId Connector id * @param linkConfig Connector forms */ - public MLink(long connectorId, MLinkConfig linkConfig) { - this.connectorId = connectorId; + public MLink(String connectorName, MLinkConfig linkConfig) { + this.connectorName = connectorName; this.connectorLinkConfig = linkConfig; } @@ -61,7 +61,7 @@ public MLink(MLink other) { */ public MLink(MLink other, MLinkConfig linkConfig) { super(other); - this.connectorId = other.connectorId; + this.connectorName = other.connectorName; this.connectorLinkConfig = linkConfig; this.setPersistenceId(other.getPersistenceId()); } @@ -74,8 +74,8 @@ public String toString() { return sb.toString(); } - public long getConnectorId() { - return connectorId; + public String getConnectorName() { + return connectorName; } public MLinkConfig getConnectorLinkConfig() { @@ -90,7 +90,7 @@ public MLink clone(boolean cloneWithValue) { if(cloneWithValue) { return new MLink(this); } else { - return new MLink(connectorId, connectorLinkConfig.clone(false)); + return new MLink(connectorName, connectorLinkConfig.clone(false)); } } @@ -105,14 +105,14 @@ public boolean equals(Object object) { } MLink mLink = (MLink)object; - return (mLink.connectorId == this.connectorId) + return (mLink.connectorName.equals(this.connectorName)) && (mLink.getPersistenceId() == this.getPersistenceId()) && (mLink.connectorLinkConfig.equals(this.connectorLinkConfig)); } @Override public int hashCode() { - int result = (int) (connectorId ^ (connectorId >>> 32)); + int result = connectorName != null ? connectorName.hashCode() : 0; result = 31 * result + (connectorLinkConfig != null ? connectorLinkConfig.hashCode() : 0); return result; } diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java index a978d62a..f487bba6 100644 --- a/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java +++ b/common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java @@ -45,8 +45,8 @@ public void testConnectorSerialization() { connectors.add(BeanTestUtil.getConnector(1L, "jdbc")); // Create testing bundles - Map configBundles = new HashMap(); - configBundles.put(1L, ConfigTestUtil.getResourceBundle()); + Map configBundles = new HashMap(); + configBundles.put("jdbc", ConfigTestUtil.getResourceBundle()); // Serialize it to JSON object ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles); @@ -62,7 +62,7 @@ public void testConnectorSerialization() { assertEquals(connectors.size(), 1); assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size()); assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0)); - ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L); + ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get("jdbc"); assertNotNull(retrievedBundle); assertEquals("a", retrievedBundle.getString("a")); assertEquals("b", retrievedBundle.getString("b")); diff --git a/common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java b/common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java index 15377c94..ed036fb5 100644 --- a/common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java +++ b/common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java @@ -42,9 +42,9 @@ public void testConnectorsSerialization() { connectors.add(BeanTestUtil.getConnector(2L, "mysql")); // Create testing bundles - Map configBundles = new HashMap(); - configBundles.put(1L, ConfigTestUtil.getResourceBundle()); - configBundles.put(2L, ConfigTestUtil.getResourceBundle()); + Map configBundles = new HashMap(); + configBundles.put("jdbc", ConfigTestUtil.getResourceBundle()); + configBundles.put("mysql", ConfigTestUtil.getResourceBundle()); // Serialize it to JSON object ConnectorsBean connectorsBean = new ConnectorsBean(connectors, configBundles); @@ -62,7 +62,7 @@ public void testConnectorsSerialization() { assertEquals(connectors.get(0), parsedConnectorsBean.getConnectors().get(0)); assertEquals(connectors.get(1), parsedConnectorsBean.getConnectors().get(1)); - ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get(1L); + ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get("jdbc"); assertNotNull(retrievedBundle); assertEquals("a", retrievedBundle.getString("a")); assertEquals("b", retrievedBundle.getString("b")); @@ -76,9 +76,9 @@ public void testSingleDirection() { connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, true)); // Create testing bundles - Map bundles = new HashMap(); - bundles.put(1L, ConfigTestUtil.getResourceBundle()); - bundles.put(2L, ConfigTestUtil.getResourceBundle()); + Map bundles = new HashMap(); + bundles.put("jdbc", ConfigTestUtil.getResourceBundle()); + bundles.put("mysql", ConfigTestUtil.getResourceBundle()); // Serialize it to JSON object ConnectorsBean bean = new ConnectorsBean(connectors, bundles); @@ -105,9 +105,9 @@ public void testNoDirection() { connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, false)); // Create testing bundles - Map bundles = new HashMap(); - bundles.put(1L, ConfigTestUtil.getResourceBundle()); - bundles.put(2L, ConfigTestUtil.getResourceBundle()); + Map bundles = new HashMap(); + bundles.put("jdbc", ConfigTestUtil.getResourceBundle()); + bundles.put("mysql", ConfigTestUtil.getResourceBundle()); // Serialize it to JSON object ConnectorsBean bean = new ConnectorsBean(connectors, bundles); diff --git a/common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java b/common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java index 51ab0e5c..13774c82 100644 --- a/common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java +++ b/common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java @@ -52,7 +52,7 @@ public static MJob createJob(String connectorName, String jobName, Long jobId, D } public static MLink getLink(String connectorName) { - return new MLink(1, getConnector(1L, connectorName).getLinkConfig()); + return new MLink(connectorName, getConnector(1L, connectorName).getLinkConfig()); } public static MConnector getConnector(Long connectorId, String connectorName) { diff --git a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java index 5bf2465f..ae54ff5d 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java @@ -51,7 +51,7 @@ public void testInitialization() { MValidator validator = new MValidator("test", ""); validators.add(validator); - MAccountableEntity link = new MLink(123l, new MLinkConfig(configs, validators)); + MAccountableEntity link = new MLink("connector_test", new MLinkConfig(configs, validators)); // Initially creation date and last update date is same assertEquals(link.getCreationDate(), link.getLastUpdateDate()); Date testCreationDate = new Date(); diff --git a/common/src/test/java/org/apache/sqoop/model/TestMLink.java b/common/src/test/java/org/apache/sqoop/model/TestMLink.java index b5df18a8..aace78a1 100644 --- a/common/src/test/java/org/apache/sqoop/model/TestMLink.java +++ b/common/src/test/java/org/apache/sqoop/model/TestMLink.java @@ -35,14 +35,14 @@ public class TestMLink { public void testInitialization() { // Test default constructor MLink link = link(); - assertEquals(123l, link.getConnectorId()); + assertEquals("connector_test", link.getConnectorName()); assertEquals("Vampire", link.getName()); assertEquals("Buffy", link.getCreationUser()); assertEquals(linkConfig(), link.getConnectorLinkConfig()); // Test copy constructor MLink copy = new MLink(link); - assertEquals(123l, copy.getConnectorId()); + assertEquals("connector_test", copy.getConnectorName()); assertEquals("Vampire", copy.getName()); assertEquals("Buffy", copy.getCreationUser()); assertEquals(link.getCreationDate(), copy.getCreationDate()); @@ -75,7 +75,7 @@ public void testClone() { } private MLink link() { - MLink link = new MLink(123l, linkConfig()); + MLink link = new MLink("connector_test", linkConfig()); link.setName("Vampire"); link.setCreationUser("Buffy"); return link; diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java index db626a1c..f19f391d 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java @@ -117,12 +117,12 @@ public Set getConnectorIds() { return idToNameMap.keySet(); } - public Map getResourceBundles(Locale locale) { - Map bundles = new HashMap(); + public Map getResourceBundles(Locale locale) { + Map bundles = new HashMap(); for (ConnectorHandler handler : handlerMap.values()) { - long id = handler.getConnectorConfigurable().getPersistenceId(); + String connectorName = handler.getConnectorConfigurable().getUniqueName(); ResourceBundle bundle = handler.getSqoopConnector().getBundle(locale); - bundles.put(id, bundle); + bundles.put(connectorName, bundle); } return bundles; } diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java index 90ee541a..35c44512 100644 --- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java +++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java @@ -339,22 +339,22 @@ public MSubmission start(String jobName, HttpEventContext ctx) { private JobRequest createJobRequest(MSubmission submission, MJob job) { // get from/to connections for the job - MLink fromConnection = getLink(job.getFromLinkId()); - MLink toConnection = getLink(job.getToLinkId()); + MLink fromLink = getLink(job.getFromLinkId()); + MLink toLink = getLink(job.getToLinkId()); // get from/to connectors for the connection - SqoopConnector fromConnector = getSqoopConnector(fromConnection.getConnectorId()); + SqoopConnector fromConnector = getSqoopConnector(fromLink.getConnectorName()); validateSupportedDirection(fromConnector, Direction.FROM); - SqoopConnector toConnector = getSqoopConnector(toConnection.getConnectorId()); + SqoopConnector toConnector = getSqoopConnector(toLink.getConnectorName()); validateSupportedDirection(toConnector, Direction.TO); // link config for the FROM part of the job Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass()); - ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig); + ConfigUtils.fromConfigs(fromLink.getConnectorLinkConfig().getConfigs(), fromLinkConfig); // link config for the TO part of the job Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass()); - ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig); + ConfigUtils.fromConfigs(toLink.getConnectorLinkConfig().getConfigs(), toLinkConfig); // from config for the job Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM)); @@ -470,7 +470,11 @@ MSubmission createJobSubmission(HttpEventContext ctx, long jobId) { return summary; } - SqoopConnector getSqoopConnector(long connnectorId) { + SqoopConnector getSqoopConnector(String connnectorName) { + return ConnectorManager.getInstance().getSqoopConnector(connnectorName); + } + + SqoopConnector getSqoopConnector(Long connnectorId) { return ConnectorManager.getInstance().getSqoopConnector(connnectorId); } diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java index f137eefe..3b200e91 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java @@ -109,7 +109,7 @@ public void testUnsupportedDirectionForConnector() { @Test public void testGetLink() { - MLink testLink = new MLink(123l, null); + MLink testLink = new MLink("connector_test", null); testLink.setEnabled(true); MLink mConnectionSpy = org.mockito.Mockito.spy(testLink); when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); @@ -121,7 +121,7 @@ public void testGetLink() { @Test public void testDisabledLink() { - MLink testConnection = new MLink(123l, null); + MLink testConnection = new MLink("connector_test", null); testConnection.setPersistenceId(1234); testConnection.setEnabled(false); SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection: " diff --git a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java index 00b0511f..a665fd1c 100644 --- a/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java +++ b/core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java @@ -228,7 +228,7 @@ public void testConnectorConfigUpgradeWithValidLinksAndJobs() { // prepare the links and jobs // the connector Id for both are the same - List linkList = links(link(1, "LA", 1), link(2, "LB", 1)); + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName())); List jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 2)); // mock necessary methods for upgradeConnector() procedure @@ -390,7 +390,7 @@ public void testConnectorConfigUpgradeHandlerWithFindJobsForConnectorError() { when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); - List linkList = links(link(1, "LA", 1), link(2, "LB", 1)); + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName())); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000, @@ -424,7 +424,7 @@ public void testConnectorConfigUpgradeHandlerWithDeleteJobInputsError() { when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); - List linkList = links(link(1, "LA", 1), link(2, "LB", 1)); + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName())); List jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade @@ -462,7 +462,7 @@ public void testConnectorConfigUpgradeHandlerWithDeleteLinkInputsError() { when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); - List linkList = links(link(1, "LA", 1), link(2, "LB", 1)); + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName())); List jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); @@ -500,7 +500,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateConnectorError() { when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); - List linkList = links(link(1, "LA", 1), link(2, "LB", 1)); + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName())); List jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); @@ -542,7 +542,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateLinkError() { when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); - List linkList = links(link(1, "LA", 1), link(2, "LB", 1)); + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName())); List jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); @@ -588,7 +588,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateJobError() { when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); - List linkList = links(link(1, "LA", 1), link(2, "LB", 1));; + List linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));; List jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); @@ -775,8 +775,8 @@ private MDriver anotherDriver() { return driver; } - private MLink link(long linkId, String linkName, long connectorId) { - MLink link = new MLink(connectorId, new MLinkConfig(new LinkedList(), new LinkedList())); + private MLink link(long linkId, String linkName, String connectorName) { + MLink link = new MLink(connectorName, new MLinkConfig(new LinkedList(), new LinkedList())); link.setPersistenceId(linkId); link.setName(linkName); return link; diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java index d1ee3204..586ef12c 100644 --- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -371,11 +371,13 @@ public void registerDriver(MDriver mDriver, Connection conn) { @Override public void createLink(MLink link, Connection conn) { int result; + MConnector mConnector; try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertLink(), Statement.RETURN_GENERATED_KEYS)) { stmt.setString(1, link.getName()); - stmt.setLong(2, link.getConnectorId()); + mConnector = findConnector(link.getConnectorName(), conn); + stmt.setLong(2, mConnector.getPersistenceId()); stmt.setBoolean(3, link.getEnabled()); stmt.setString(4, link.getCreationUser()); stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime())); @@ -1515,7 +1517,6 @@ private List loadLinksForUpgrade(PreparedStatement stmt, try (ResultSet rsConnection = stmt.executeQuery(); PreparedStatement connectorConfigFetchStatement = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable()); PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());) { - while(rsConnection.next()) { long id = rsConnection.getLong(1); String name = rsConnection.getString(2); @@ -1525,6 +1526,7 @@ private List loadLinksForUpgrade(PreparedStatement stmt, Date creationDate = rsConnection.getTimestamp(6); String updateUser = rsConnection.getString(7); Date lastUpdateDate = rsConnection.getTimestamp(8); + String connectorName = rsConnection.getString(9); connectorConfigFetchStatement.setLong(1, connectorId); connectorConfigInputStatement.setLong(1, id); @@ -1535,7 +1537,8 @@ private List loadLinksForUpgrade(PreparedStatement stmt, loadConnectorConfigs(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement, connectorConfigInputStatement, 2, conn); - MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig, Collections.EMPTY_LIST)); + + MLink link = new MLink(connectorName, new MLinkConfig(connectorLinkConfig, Collections.EMPTY_LIST)); link.setPersistenceId(id); link.setName(name); @@ -1576,7 +1579,8 @@ private List loadLinks(PreparedStatement stmt, configStmt.setLong(1, connectorId); inputStmt.setLong(1, id); loadInputsForConfigs(connectorLinkConfig, configStmt, inputStmt); - MLink link = new MLink(connectorId, connectorLinkConfig); + + MLink link = new MLink(connectorName, connectorLinkConfig); link.setPersistenceId(id); link.setName(name); diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java index 193ee5fe..5168bb8e 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java @@ -97,7 +97,7 @@ public void testEntityDataSerialization() throws Exception { map.put("A", "B"); // Connection object with all various values - MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig()); + MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig()); MLinkConfig linkConfig = link.getConnectorLinkConfig(); assertEquals(linkConfig.getStringInput("LINK1.I1").getEditable(), InputEditable.USER_ONLY); assertEquals(linkConfig.getStringInput("LINK1.I1").getOverrides(), "LINK1.I2"); diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java index 92d1faee..81724115 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java @@ -304,6 +304,6 @@ public void testDeleteLink() throws Exception { } public MLink getLink() { - return new MLink(1, handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig()); + return new MLink("A", handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig()); } } diff --git a/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java b/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java index 232ef4c2..53e4da5e 100644 --- a/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java +++ b/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java @@ -94,7 +94,7 @@ protected MDriver getDriver() { } protected MLink getLink(String name, MConnector connector) { - MLink link = new MLink(connector.getPersistenceId(), + MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig()); link.setName(name); fillLink(link); diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java index b8b0f526..18e33fce 100644 --- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java +++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java @@ -83,7 +83,7 @@ protected MDriver getDriver() { } protected MLink getLink(String name, MConnector connector) { - MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig()); + MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig()); link.setName(name); fillLink(link); return link; diff --git a/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java index 52abe72b..efe9bef1 100644 --- a/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java +++ b/server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java @@ -48,7 +48,7 @@ public ConnectorRequestHandler() { @Override public JsonBean handleEvent(RequestContext ctx) { List connectors; - Map configParamBundles; + Map configParamBundles; Locale locale = ctx.getAcceptLanguageHeader(); String cIdentifier = ctx.getLastURLElement(); @@ -73,7 +73,7 @@ public JsonBean handleEvent(RequestContext ctx) { configParamBundles = new HashMap<>(); MConnector connector = ConnectorManager.getInstance().getConnectorConfigurable(mConnector.getUniqueName()); - configParamBundles.put(connector.getPersistenceId(), + configParamBundles.put(connector.getUniqueName(), ConnectorManager.getInstance().getResourceBundle(mConnector.getUniqueName(), locale)); AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(), diff --git a/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java b/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java index 9e752583..cc07fc85 100644 --- a/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java +++ b/server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java @@ -130,7 +130,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) { } MLink postedLink = links.get(0); - MConnector mConnector = HandlerUtils.getConnectorFromConnectorId(postedLink.getConnectorId()); + MConnector mConnector = HandlerUtils.getConnectorFromConnectorName(postedLink.getConnectorName()); // Authorization check if (create) { @@ -142,7 +142,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) { } MLinkConfig linkConfig = ConnectorManager.getInstance() - .getConnectorConfigurable(postedLink.getConnectorId()).getLinkConfig(); + .getConnectorConfigurable(postedLink.getConnectorName()).getLinkConfig(); if (!linkConfig.equals(postedLink.getConnectorLinkConfig())) { throw new SqoopException(ServerError.SERVER_0003, "Detected incorrect link config structure"); } @@ -156,7 +156,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) { } // Associated connector for this link SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector( - postedLink.getConnectorId()); + postedLink.getConnectorName()); // Validate user supplied config data ConfigValidationResult connectorLinkConfigValidation = ConfigUtils.validateConfigs(postedLink @@ -240,10 +240,10 @@ private LinksBean createLinksBean(List links, Locale locale) { private void addConnectorConfigBundle(Locale locale, LinkBean bean) { // Add associated resources into the bean for (MLink link : bean.getLinks()) { - long connectorId = link.getConnectorId(); - if (!bean.hasConnectorConfigBundle(connectorId)) { - bean.addConnectorConfigBundle(connectorId, ConnectorManager.getInstance() - .getResourceBundle(connectorId, locale)); + String connectorName = link.getConnectorName(); + if (!bean.hasConnectorConfigBundle(connectorName)) { + bean.addConnectorConfigBundle(connectorName, ConnectorManager.getInstance() + .getResourceBundle(connectorName, locale)); } } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java index 15cfad76..5e8391c5 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java @@ -20,6 +20,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; import org.apache.sqoop.common.Direction; +import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.shell.core.Constants; @@ -66,10 +67,13 @@ private Status cloneJob(String jobArg, List args, boolean isInteractive) MJob job = client.getJob(jobArg); job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT); + // TODO: the job should be related with connector by connectorName + MConnector fromConnector = getClient().getConnector(job.getFromConnectorId()); + MConnector toConnector = getClient().getConnector(job.getToConnectorId()); ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle( - job.getFromConnectorId()); + fromConnector.getUniqueName()); ResourceBundle toConnectorBundle = client.getConnectorConfigBundle( - job.getToConnectorId()); + toConnector.getUniqueName()); ResourceBundle driverConfigBundle = client.getDriverConfigBundle(); Status status = Status.OK; diff --git a/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java index a679be87..73928a5f 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java @@ -70,7 +70,7 @@ private Status cloneLink(String linkArg, List args, boolean isInteractiv Status status = Status.OK; - ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId()); + ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(link.getConnectorName()); if (isInteractive) { printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG); diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java index 79caa0d1..b0adb3d7 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java @@ -90,9 +90,9 @@ private Status createJob(String fromLinkArg, String toLinkArg, List args } ResourceBundle fromConfigBundle = getClient().getConnectorConfigBundle( - job.getFromConnectorId()); + fromConnector.getUniqueName()); ResourceBundle toConfigBundle = getClient().getConnectorConfigBundle( - job.getToConnectorId()); + toConnector.getUniqueName()); ResourceBundle driverConfigBundle = getClient().getDriverConfigBundle(); Status status = Status.OK; diff --git a/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java index 289c3c3c..e5af3f18 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java @@ -72,24 +72,23 @@ private Status createLink(CommandLine line, List args, boolean isInterac cid = getLong(line, Constants.OPT_CID); getClient().getConnector(cid); - //Would have thrown an exception before this if input was neither a valid name nor an id + //Would have thrown an exception before this if input was an invalid connector name //This will do an extra getConnector() call again inside createLink() //but should not matter as connectors are cached - link = getClient().createLink(cid); + link = getClient().createLink(connectorName); printlnResource(Constants.RES_CREATE_CREATING_LINK, cid); } else { //Command line had connector name //This will do an extra getConnector() call again inside createLink() but //should not matter as connectors are cached - cid = connector.getPersistenceId(); link = getClient().createLink(connectorName); printlnResource(Constants.RES_CREATE_CREATING_LINK, connectorName); } ConsoleReader reader = getConsoleReader(); - ResourceBundle connectorConfigBundle = getClient().getConnectorConfigBundle(cid); + ResourceBundle connectorConfigBundle = getClient().getConnectorConfigBundle(connectorName); Status status = Status.OK; if (isInteractive) { diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java index 6efb51c0..07cccf20 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java @@ -124,6 +124,6 @@ private void displayConnector(MConnector connector) { connector.getVersion(), connector.getSupportedDirections().toString() ); - displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getPersistenceId())); + displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getUniqueName())); } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java index 21873cc9..faad5052 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java @@ -159,15 +159,20 @@ private void displayJob(MJob job) { job.getLastUpdateUser(), formatter.format(job.getLastUpdateDate()) ); + + // TODO: should be removed when MJob link with Connector by name. + MConnector fromConnector = getClient().getConnector(job.getFromConnectorId()); + MConnector toConnector = getClient().getConnector(job.getToConnectorId()); + displayConfig(job.getDriverConfig().getConfigs(), client.getDriverConfigBundle()); printlnResource(Constants.RES_SHOW_PROMPT_JOB_FROM_LID_INFO, job.getFromLinkId()); displayConfig(job.getFromJobConfig().getConfigs(), - client.getConnectorConfigBundle(job.getFromConnectorId())); + client.getConnectorConfigBundle(fromConnector.getUniqueName())); printlnResource(Constants.RES_SHOW_PROMPT_JOB_TO_LID_INFO, job.getToLinkId()); displayConfig(job.getToJobConfig().getConfigs(), - client.getConnectorConfigBundle(job.getToConnectorId())); + client.getConnectorConfigBundle(toConnector.getUniqueName())); } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java index 04dd2286..663a3779 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java @@ -81,17 +81,16 @@ private void showSummary() { List ids = new LinkedList(); List names = new LinkedList(); List connectorIds = new LinkedList(); + List connectorNames = new LinkedList(); List availabilities = new LinkedList(); for (MLink link : links) { ids.add(String.valueOf(link.getPersistenceId())); names.add(link.getName()); - connectorIds.add(String.valueOf(link.getConnectorId())); + connectorNames.add(link.getConnectorName()); availabilities.add(String.valueOf(link.getEnabled())); } - List connectorNames = getConnectorNames(connectorIds); - TableDisplayer.display(header, ids, names, connectorIds, connectorNames, availabilities); } @@ -126,41 +125,10 @@ private void displayLink(MLink link) { formatter.format(link.getLastUpdateDate()) ); - long connectorId = link.getConnectorId(); - MConnector connector = client.getConnector(connectorId); - String connectorName = ""; - if (connector != null) { - connectorName = connector.getUniqueName(); - } - printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, connectorName, connectorId); + printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, link.getConnectorName()); // Display link config displayConfig(link.getConnectorLinkConfig().getConfigs(), - client.getConnectorConfigBundle(connectorId)); - } - - private List getConnectorNames(List connectorIds) { - Map connectorIdToName = new HashMap(); - for (String connectorId : connectorIds) { - if (!connectorIdToName.containsKey(connectorId)) { - try { - MConnector connector = client.getConnector(Long.parseLong(connectorId)); - if (connector != null) { - connectorIdToName.put(connectorId, connector.getUniqueName()); - } - } catch (SqoopException ex) { - connectorIdToName.put(connectorId, "Access Denied"); - } - } - } - List connectorNames = new ArrayList(); - for (String connectorId : connectorIds) { - if (connectorIdToName.get(connectorId) != null) { - connectorNames.add(connectorIdToName.get(connectorId)); - } else { - connectorNames.add(""); - } - } - return connectorNames; + client.getConnectorConfigBundle(link.getConnectorName())); } } diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java index 49cfd0b2..95f044ff 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java @@ -20,6 +20,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; import org.apache.sqoop.common.Direction; +import org.apache.sqoop.model.MConnector; import org.apache.sqoop.model.MJob; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.utils.ConfigDisplayer; @@ -67,10 +68,14 @@ private Status updateJob(String jobArg, List args, boolean isInteractive // TODO(SQOOP-1634): using from/to and driver config id, this call can be avoided MJob job = client.getJob(jobArg); + // TODO: should be removed when MJob link with Connector by name. + MConnector fromConnector = getClient().getConnector(job.getFromConnectorId()); + MConnector toConnector = getClient().getConnector(job.getToConnectorId()); + ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle( - job.getFromConnectorId()); + fromConnector.getUniqueName()); ResourceBundle toConnectorBundle = client.getConnectorConfigBundle( - job.getToConnectorId()); + toConnector.getUniqueName()); ResourceBundle driverConfigBundle = client.getDriverConfigBundle(); Status status = Status.OK; diff --git a/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java index 1bb7cd5b..aef500cd 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java @@ -65,8 +65,7 @@ private Status updateLink(String linkArg, List args, boolean isInteracti // TODO(SQOOP-1634): using link config id, this call can be avoided MLink link = client.getLink(linkArg); - - ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId()); + ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorName()); Status status = Status.OK; diff --git a/shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java b/shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java index eed27d75..14daa879 100644 --- a/shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java +++ b/shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java @@ -41,24 +41,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.model.InputEditable; -import org.apache.sqoop.model.MBooleanInput; -import org.apache.sqoop.model.MConfig; -import org.apache.sqoop.model.MDateTimeInput; -import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MEnumInput; -import org.apache.sqoop.model.MFromConfig; -import org.apache.sqoop.model.MInput; -import org.apache.sqoop.model.MIntegerInput; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MLinkConfig; -import org.apache.sqoop.model.MListInput; -import org.apache.sqoop.model.MLongInput; -import org.apache.sqoop.model.MMapInput; -import org.apache.sqoop.model.MStringInput; -import org.apache.sqoop.model.MToConfig; -import org.apache.sqoop.model.MValidator; +import org.apache.sqoop.model.*; import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.ShellError; import org.apache.sqoop.utils.MapResourceBundle; @@ -104,9 +87,9 @@ public Enumeration getKeys() { @Test public void testCloneLink() { ShellEnvironment.setInteractive(false); - MLink link = new MLink(1L, new MLinkConfig(new ArrayList(), new ArrayList())); + MLink link = new MLink("connector_name_test", new MLinkConfig(new ArrayList(), new ArrayList())); when(client.getLink("link_test")).thenReturn(link); - when(client.getConnectorConfigBundle(1L)).thenReturn(new MapResourceBundle(new HashMap())); + when(client.getConnectorConfigBundle("connector_name_test")).thenReturn(new MapResourceBundle(new HashMap())); when(client.saveLink(link)).thenReturn(Status.OK); // clone link -lid link_test @@ -136,10 +119,11 @@ public void testCloneLink() { public void testCloneLinkInteractive() { ShellEnvironment.setInteractive(true); initEnv(); - MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList())); + MLink link = new MLink("connector_name_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList())); when(client.getLink("link_test")).thenReturn(link); - when(client.getConnectorConfigBundle(1L)).thenReturn(resourceBundle); + when(client.getConnectorConfigBundle("connector_name_test")).thenReturn(resourceBundle); when(client.saveLink(link)).thenReturn(Status.OK); + when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null)); // clone link -lid link_test initData("linkname\r" + // link name @@ -176,9 +160,10 @@ public void testCloneJob() { new MToConfig(new ArrayList(), new ArrayList()), new MDriverConfig(new ArrayList(), new ArrayList())); when(client.getJob("job_test")).thenReturn(job); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(new MapResourceBundle(new HashMap())); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(new MapResourceBundle(new HashMap())); when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap())); when(client.saveJob(job)).thenReturn(Status.OK); + when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null)); // clone job -jid job_test Status status = (Status) cloneCmd.execute(Arrays.asList(Constants.FN_JOB, "-jid", "job_test")); @@ -211,7 +196,7 @@ public void testCloneJobInteractive() { new MToConfig(getConfig("toJobConfig"), new ArrayList()), new MDriverConfig(getConfig("driverConfig"), new ArrayList())); when(client.getJob("job_test")).thenReturn(job); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle); when(client.getDriverConfigBundle()).thenReturn(resourceBundle); when(client.saveJob(job)).thenReturn(Status.OK); diff --git a/shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java b/shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java index 48f646b9..89f4fe80 100644 --- a/shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java +++ b/shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java @@ -104,7 +104,8 @@ public Enumeration getKeys() { public void testCreateLink() { ShellEnvironment.setInteractive(false); when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null)); - when(client.createLink("connector_test")).thenReturn(new MLink(1, new MLinkConfig(new ArrayList(), new ArrayList()))); + when(client.createLink("connector_test")).thenReturn( + new MLink("connector_test", new MLinkConfig(new ArrayList(), new ArrayList()))); when(client.saveLink(any(MLink.class))).thenReturn(Status.OK); // create link -c connector_test @@ -144,10 +145,10 @@ public void testCreateLinkInteractive() { ShellEnvironment.setInteractive(true); initEnv(); when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null)); - MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList())); + MLink link = new MLink("connector_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList())); when(client.createLink("connector_test")).thenReturn(link); when(client.saveLink(any(MLink.class))).thenReturn(Status.OK); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle); // create link -c connector_test initData("linkname\r" + // link name @@ -228,7 +229,7 @@ public void testCreateJobInteractive() { when(client.getConnector(1)).thenReturn(fromConnector); when(client.getConnector(2)).thenReturn(toConnector); when(client.saveJob(any(MJob.class))).thenReturn(Status.OK); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle); when(client.getDriverConfigBundle()).thenReturn(resourceBundle); // create job -f link_from -to link_to diff --git a/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java b/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java index 42723867..870bee33 100644 --- a/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java +++ b/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java @@ -203,7 +203,7 @@ public void testShowDriver() { @Test public void testShowLink() { when(client.getLinks()).thenReturn(new ArrayList()); - when(client.getLink(any(String.class))).thenReturn(new MLink(1L, new MLinkConfig(new ArrayList(), new ArrayList()))); + when(client.getLink(any(String.class))).thenReturn(new MLink("connector_test", new MLinkConfig(new ArrayList(), new ArrayList()))); // show link summary out.reset(); @@ -234,6 +234,7 @@ public void testShowLink() { @Test public void testShowJob() { when(client.getJobs()).thenReturn(new ArrayList()); + when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null)); when(client.getJob("1")).thenReturn(new MJob(1L, 2L, 1L, 2L, new MFromConfig(new ArrayList(), new ArrayList()), new MToConfig(new ArrayList(), new ArrayList()), diff --git a/shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java b/shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java index 9b3e87ad..2a6670d1 100644 --- a/shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java +++ b/shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java @@ -105,10 +105,10 @@ public Enumeration getKeys() { @Test public void testUpdateLink() throws InterruptedException { ShellEnvironment.setInteractive(false); - MLink link = new MLink(1L, new MLinkConfig(new ArrayList(), new ArrayList())); + MLink link = new MLink("connector_test", new MLinkConfig(new ArrayList(), new ArrayList())); when(client.getLink("link_test")).thenReturn(link); - when(client.getConnectorConfigBundle(1L)).thenReturn(new MapResourceBundle(new HashMap())); - when(client.updateLink(link)).thenReturn(Status.OK); + when(client.getConnectorConfigBundle("connector_test")).thenReturn(new MapResourceBundle(new HashMap())); + when(client.updateLink(any(MLink.class))).thenReturn(Status.OK); // update link -lid link_test Status status = (Status) updateCmd.execute(Arrays.asList(Constants.FN_LINK, "-lid", "link_test")); @@ -134,14 +134,14 @@ public void testUpdateLink() throws InterruptedException { } @Test - public void testUpdateLinkInteractive() { + public void testUpdateLinkInteractive() throws Exception { ShellEnvironment.setInteractive(true); initEnv(); when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null)); - MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList())); + MLink link = new MLink("connector_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList())); when(client.getLink("link_test")).thenReturn(link); - when(client.updateLink(link)).thenReturn(Status.OK); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); + when(client.updateLink(any(MLink.class))).thenReturn(Status.OK); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle); // update link -lid link_test initData("linkname\r" + // link name @@ -178,7 +178,8 @@ public void testUpdateJob() throws InterruptedException { new MToConfig(new ArrayList(), new ArrayList()), new MDriverConfig(new ArrayList(), new ArrayList())); when(client.getJob("job_test")).thenReturn(job); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(new MapResourceBundle(new HashMap())); + when(client.getConnector(any(Long.class))).thenReturn(new MConnector("connect_test", "", "", null, null, null)); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(new MapResourceBundle(new HashMap())); when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap())); when(client.updateJob(job)).thenReturn(Status.OK); @@ -213,7 +214,8 @@ public void testUpdateJobInteractive() { new MToConfig(getConfig("toJobConfig"), new ArrayList()), new MDriverConfig(getConfig("driverConfig"), new ArrayList())); when(client.getJob("job_test")).thenReturn(job); - when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); + when(client.getConnector(any(Long.class))).thenReturn(new MConnector("connect_test", "", "", null, null, null)); + when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle); when(client.getDriverConfigBundle()).thenReturn(resourceBundle); when(client.updateJob(job)).thenReturn(Status.OK); diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java index bd4ba6a2..e78adc0e 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java @@ -22,7 +22,6 @@ import java.lang.reflect.Method; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.JobConf; import org.apache.log4j.Logger; diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/S3Test.java b/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/S3Test.java index 7fec3106..f98870db 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/S3Test.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hdfs/S3Test.java @@ -90,7 +90,7 @@ public void testImportExport() throws Exception { saveLink(hdfsLink); // DB -> S3 - MJob db2aws = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); + MJob db2aws = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName()); fillRdbmsFromConfig(db2aws, "id"); fillHdfsToConfig(db2aws, ToFormat.TEXT_FILE); @@ -110,7 +110,7 @@ public void testImportExport() throws Exception { assertEquals(provider.rowCount(getTableName()), 0); // S3 -> DB - MJob aws2db = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); + MJob aws2db = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName()); fillHdfsFromConfig(aws2db); fillRdbmsToConfig(aws2db); @@ -145,7 +145,7 @@ public void testIncrementalRead() throws Exception { saveLink(hdfsLink); // S3 -> HDFS - MJob aws2hdfs = getClient().createJob(s3Link.getPersistenceId(), hdfsLink.getPersistenceId()); + MJob aws2hdfs = getClient().createJob(s3Link.getName(), hdfsLink.getName()); fillHdfsFromConfig(aws2hdfs); aws2hdfs.getFromJobConfig().getEnumInput("incremental.incrementalType").setValue(IncrementalType.NEW_FILES); diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java index 75d21820..2afb965d 100644 --- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java +++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java @@ -278,7 +278,7 @@ private long loadLink(MLink link) { Repository repository = RepositoryManager.getInstance().getRepository(); - MConnector mConnector = ConnectorManager.getInstance().getConnectorConfigurable(link.getConnectorId()); + MConnector mConnector = ConnectorManager.getInstance().getConnectorConfigurable(link.getConnectorName()); ConnectorConfigurableUpgrader connectorConfigUpgrader = ConnectorManager.getInstance().getSqoopConnector(mConnector.getUniqueName()).getConfigurableUpgrader(null); List connectorConfigs = mConnector.getLinkConfig().clone(false).getConfigs(); @@ -291,7 +291,7 @@ private long loadLink(MLink link) { // Transform config structures to objects for validations SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector( - link.getConnectorId()); + link.getConnectorName()); Object connectorConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass());