github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-05 05:59:07 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2690: Sqoop2: Use connector name in MLink.

Browse Source 
(Colin Ma via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2015-12-07 11:18:12 +01:00
parent 1b228f308c
commit 794adbcf3d
40 changed files with 212 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
client/src/main/java/org/apache/sqoop/client/SqoopClient.java
View File

@ -61,11 +61,11 @@ public class SqoopClient {
/** /**
* All cached connectors. * All cached connectors.
*/ */
private Map<Long, MConnector> connectors; private Map<String, MConnector> connectors;
/** /**
* All cached config params for every registered connector in the sqoop system. * All cached config params for every registered connector in the sqoop system.
*/ */
private Map<Long, ResourceBundle> connectorConfigBundles; private Map<String, ResourceBundle> connectorConfigBundles;
/** /**
* Cached driver. * Cached driver.
@ -122,25 +122,22 @@ public void setSqoopRequests(SqoopResourceRequests requests) {
* Clear internal cache. * Clear internal cache.
*/ */
public void clearCache() { public void clearCache() {
connectorConfigBundles = new HashMap<Long, ResourceBundle>(); connectorConfigBundles = new HashMap<String, ResourceBundle>();
driverConfigBundle = null; driverConfigBundle = null;
connectors = new HashMap<Long, MConnector>(); connectors = new HashMap<String, MConnector>();
mDriver = null; mDriver = null;
isAllConnectors = false; isAllConnectors = false;
} }
/** /**
* Get connector with given id. * Get connector with given id.
* TODO: This method should be removed when MJob link with Connector by name.
* *
* @param cid Connector id. * @param cid Connector id.
* @return * @return
*/ */
public MConnector getConnector(long cid) { public MConnector getConnector(long cid) {
if(connectors.containsKey(cid)) { return retrieveConnector(Long.toString(cid));
return connectors.get(cid).clone(false);
}
retrieveConnector(cid);
return connectors.get(cid).clone(false);
} }
/** /**
@ -183,13 +180,17 @@ private MConnector getConnectorFromCache(String connectorName) {
/** /**
* Retrieve connector structure from server and cache it. * Retrieve connector structure from server and cache it.
* TODO: The method support both connector name/id, this should support name only when MJob link with Connector by name.
* *
* @param cid Connector id * @param connectorIdentify Connector name/id
*/ */
private void retrieveConnector(long cid) { private MConnector retrieveConnector(String connectorIdentify) {
ConnectorBean request = resourceRequests.readConnector(cid); ConnectorBean request = resourceRequests.readConnector(connectorIdentify);
connectors.put(cid, request.getConnectors().get(0)); MConnector mConnector = request.getConnectors().get(0);
connectorConfigBundles.put(cid, request.getResourceBundles().get(cid)); String connectorName = mConnector.getUniqueName();
connectors.put(connectorName, mConnector);
connectorConfigBundles.put(connectorName, request.getResourceBundles().get(connectorName));
return mConnector;
} }
/** /**
@ -205,7 +206,7 @@ public Collection<MConnector> getConnectors() {
ConnectorBean bean = resourceRequests.readConnector(null); ConnectorBean bean = resourceRequests.readConnector(null);
isAllConnectors = true; isAllConnectors = true;
for(MConnector connector : bean.getConnectors()) { for(MConnector connector : bean.getConnectors()) {
connectors.put(connector.getPersistenceId(), connector); connectors.put(connector.getUniqueName(), connector);
} }
connectorConfigBundles = bean.getResourceBundles(); connectorConfigBundles = bean.getResourceBundles();
@ -218,12 +219,12 @@ public Collection<MConnector> getConnectors() {
* @param connectorId Connector id. * @param connectorId Connector id.
* @return * @return
*/ */
public ResourceBundle getConnectorConfigBundle(long connectorId) { public ResourceBundle getConnectorConfigBundle(String connectorName) {
if(connectorConfigBundles.containsKey(connectorId)) { if(connectorConfigBundles.containsKey(connectorName)) {
return connectorConfigBundles.get(connectorId); return connectorConfigBundles.get(connectorName);
} }
retrieveConnector(connectorId); retrieveConnector(connectorName);
return connectorConfigBundles.get(connectorId); return connectorConfigBundles.get(connectorName);
} }
/** /**
@ -275,16 +276,6 @@ public ResourceBundle getDriverConfigBundle() {
return driverConfigBundle; return driverConfigBundle;
} }
/**
* Create new link object for given connector id
*
* @param connectorId Connector id
* @return
*/
public MLink createLink(long connectorId) {
return new MLink(connectorId, getConnector(connectorId).getLinkConfig());
}
/** /**
* Create new link object for given connector name * Create new link object for given connector name
* *
@ -296,7 +287,7 @@ public MLink createLink(String connectorName) {
if (connector == null) { if (connector == null) {
throw new SqoopException(ClientError.CLIENT_0003, connectorName); throw new SqoopException(ClientError.CLIENT_0003, connectorName);
} }
return createLink(connector.getPersistenceId()); return new MLink(connectorName, getConnector(connectorName).getLinkConfig());
} }
@ -398,40 +389,20 @@ public void deleteLink(long linkId) {
public MJob createJob(String fromLinkName, String toLinkName) { public MJob createJob(String fromLinkName, String toLinkName) {
MLink fromLink = getLink(fromLinkName); MLink fromLink = getLink(fromLinkName);
MLink toLink = getLink(toLinkName); MLink toLink = getLink(toLinkName);
MConnector connectorForFromLink = getConnector(fromLink.getConnectorName());
MConnector connectorForToLink = getConnector(toLink.getConnectorName());
return new MJob( return new MJob(
fromLink.getConnectorId(), connectorForFromLink.getPersistenceId(),
toLink.getConnectorId(), connectorForToLink.getPersistenceId(),
fromLink.getPersistenceId(), fromLink.getPersistenceId(),
toLink.getPersistenceId(), toLink.getPersistenceId(),
getConnector(fromLink.getConnectorId()).getFromConfig(), connectorForFromLink.getFromConfig().clone(false),
getConnector(toLink.getConnectorId()).getToConfig(), connectorForToLink.getToConfig().clone(false),
getDriverConfig() getDriverConfig()
); );
} }
/**
* Create new job the for given links.
*
* @param fromLinkId From link id
* @param toLinkId To link id
* @return
*/
public MJob createJob(long fromLinkId, long toLinkId) {
MLink fromLink = getLink(fromLinkId);
MLink toLink = getLink(toLinkId);
return new MJob(
fromLink.getConnectorId(),
toLink.getConnectorId(),
fromLink.getPersistenceId(),
toLink.getPersistenceId(),
getConnector(fromLink.getConnectorId()).getFromConfig(),
getConnector(toLink.getConnectorId()).getToConfig(),
getDriverConfig()
);
}
/** /**
* Retrieve job for given id. * Retrieve job for given id.
* *

8
client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
View File

@ -39,17 +39,17 @@ public ConnectorResourceRequest(DelegationTokenAuthenticatedURL.Token token){
super(token); super(token);
} }
public ConnectorBean read(String serverUrl, Long cid) { public ConnectorBean read(String serverUrl, String connectorName) {
String response; String response;
if (cid == null) { if (connectorName == null) {
response = super.get(serverUrl + RESOURCE + "all"); response = super.get(serverUrl + RESOURCE + "all");
} else { } else {
response = super.get(serverUrl + RESOURCE + cid); response = super.get(serverUrl + RESOURCE + connectorName);
} }
JSONObject jsonObject = JSONUtils.parse(response); JSONObject jsonObject = JSONUtils.parse(response);
// defaults to all // defaults to all
ConnectorBean bean = new ConnectorsBean(); ConnectorBean bean = new ConnectorsBean();
if (cid != null) { if (connectorName != null) {
bean = new ConnectorBean(); bean = new ConnectorBean();
} }
bean.restore(jsonObject); bean.restore(jsonObject);

4
client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
View File

@ -114,8 +114,8 @@ public DriverBean readDriver() {
return getDriverResourceRequest().read(serverUrl); return getDriverResourceRequest().read(serverUrl);
} }
public ConnectorBean readConnector(Long cid) { public ConnectorBean readConnector(String connectorName) {
return getConnectorResourceRequest().read(serverUrl, cid); return getConnectorResourceRequest().read(serverUrl, connectorName);
} }
public ValidationResultBean saveLink(MLink link) { public ValidationResultBean saveLink(MLink link) {

51
client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
View File

@ -65,17 +65,6 @@ public void setUp() {
* Retrieve connector information, request to bundle for same connector should * Retrieve connector information, request to bundle for same connector should
* not require additional HTTP request. * not require additional HTTP request.
*/ */
@Test
public void testGetConnector() {
when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
client.getConnectorConfigBundle(1L);
verify(resourceRequests, times(1)).readConnector(1L);
}
@Test @Test
public void testGetConnectorByString() { public void testGetConnectorByString() {
when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1))); when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1)));
@ -83,9 +72,9 @@ public void testGetConnectorByString() {
assertEquals(1, connector.getPersistenceId()); assertEquals(1, connector.getPersistenceId());
assertEquals("A1", connector.getUniqueName()); assertEquals("A1", connector.getUniqueName());
client.getConnectorConfigBundle(1L); client.getConnectorConfigBundle("A1");
verify(resourceRequests, times(0)).readConnector(1L); verify(resourceRequests, times(0)).readConnector("A1");
verify(resourceRequests, times(1)).readConnector(null); verify(resourceRequests, times(1)).readConnector(null);
} }
@ -95,13 +84,13 @@ public void testGetConnectorByString() {
*/ */
@Test @Test
public void testGetConnectorBundle() { public void testGetConnectorBundle() {
when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1))); when(resourceRequests.readConnector("A1")).thenReturn(connectorBean(connector(1)));
client.getConnectorConfigBundle(1L); client.getConnectorConfigBundle("A1");
MConnector connector = client.getConnector(1); MConnector connector = client.getConnector("A1");
assertEquals(1, connector.getPersistenceId()); assertEquals(1, connector.getPersistenceId());
verify(resourceRequests, times(1)).readConnector(1L); verify(resourceRequests, times(1)).readConnector("A1");
} }
/** /**
@ -144,12 +133,12 @@ public void testGetConnectors() {
Collection<MConnector> connectors = client.getConnectors(); Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size()); assertEquals(2, connectors.size());
client.getConnectorConfigBundle(1); client.getConnectorConfigBundle("A1");
connector = client.getConnector(1); connector = client.getConnector("A1");
assertEquals(1, connector.getPersistenceId()); assertEquals(1, connector.getPersistenceId());
connector = client.getConnector(2); connector = client.getConnector("A2");
client.getConnectorConfigBundle(2); client.getConnectorConfigBundle("A2");
assertEquals(2, connector.getPersistenceId()); assertEquals(2, connector.getPersistenceId());
connectors = client.getConnectors(); connectors = client.getConnectors();
@ -179,21 +168,21 @@ public void testGetConnectors() {
public void testGetConnectorOneByOne() { public void testGetConnectorOneByOne() {
ConnectorBean bean = connectorBean(connector(1), connector(2)); ConnectorBean bean = connectorBean(connector(1), connector(2));
when(resourceRequests.readConnector(null)).thenReturn(bean); when(resourceRequests.readConnector(null)).thenReturn(bean);
when(resourceRequests.readConnector(1L)).thenReturn(bean); when(resourceRequests.readConnector("A1")).thenReturn(bean);
when(resourceRequests.readConnector(2L)).thenReturn(bean); when(resourceRequests.readConnector("A2")).thenReturn(bean);
client.getConnectorConfigBundle(1); client.getConnectorConfigBundle("A1");
client.getConnector(1); client.getConnector("A1");
client.getConnector(2); client.getConnector("A2");
client.getConnectorConfigBundle(2); client.getConnectorConfigBundle("A2");
Collection<MConnector> connectors = client.getConnectors(); Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size()); assertEquals(2, connectors.size());
verify(resourceRequests, times(1)).readConnector(null); verify(resourceRequests, times(1)).readConnector(null);
verify(resourceRequests, times(1)).readConnector(1L); verify(resourceRequests, times(1)).readConnector("A1");
verify(resourceRequests, times(1)).readConnector(2L); verify(resourceRequests, times(0)).readConnector("A2");
verifyNoMoreInteractions(resourceRequests); verifyNoMoreInteractions(resourceRequests);
} }
@ -208,11 +197,11 @@ public void testCreateLink() {
private ConnectorBean connectorBean(MConnector...connectors) { private ConnectorBean connectorBean(MConnector...connectors) {
List<MConnector> connectorList = new ArrayList<MConnector>(); List<MConnector> connectorList = new ArrayList<MConnector>();
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
for(MConnector connector : connectors) { for(MConnector connector : connectors) {
connectorList.add(connector); connectorList.add(connector);
bundles.put(connector.getPersistenceId(), null); bundles.put(connector.getUniqueName(), null);
} }
return new ConnectorBean(connectorList, bundles); return new ConnectorBean(connectorList, bundles);
} }

14
common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
View File

@ -57,10 +57,10 @@ public class ConnectorBean extends ConfigurableBean {
private static final String CONNECTOR = "connector"; private static final String CONNECTOR = "connector";
private List<MConnector> connectors; private List<MConnector> connectors;
private Map<Long, ResourceBundle> connectorConfigBundles; private Map<String, ResourceBundle> connectorConfigBundles;
// for "extract" // for "extract"
public ConnectorBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) { public ConnectorBean(List<MConnector> connectors, Map<String, ResourceBundle> bundles) {
this.connectors = connectors; this.connectors = connectors;
this.connectorConfigBundles = bundles; this.connectorConfigBundles = bundles;
} }
@ -73,7 +73,7 @@ public List<MConnector> getConnectors() {
return connectors; return connectors;
} }
public Map<Long, ResourceBundle> getResourceBundles() { public Map<String, ResourceBundle> getResourceBundles() {
return connectorConfigBundles; return connectorConfigBundles;
} }
@ -121,7 +121,7 @@ private JSONObject extractConnector(boolean skipSensitive, MConnector connector)
connectorJsonObject.put(ALL_CONFIGS, new JSONObject()); connectorJsonObject.put(ALL_CONFIGS, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) { if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIGS, connectorJsonObject.put(ALL_CONFIGS,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId()))); extractConfigParamBundle(connectorConfigBundles.get(connector.getUniqueName())));
} }
return connectorJsonObject; return connectorJsonObject;
} }
@ -129,14 +129,14 @@ private JSONObject extractConnector(boolean skipSensitive, MConnector connector)
@Override @Override
public void restore(JSONObject jsonObject) { public void restore(JSONObject jsonObject) {
connectors = new ArrayList<MConnector>(); connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>(); connectorConfigBundles = new HashMap<String, ResourceBundle>();
JSONObject obj = JSONUtils.getJSONObject(jsonObject, CONNECTOR); JSONObject obj = JSONUtils.getJSONObject(jsonObject, CONNECTOR);
connectors.add(restoreConnector(obj)); connectors.add(restoreConnector(obj));
} }
protected void restoreConnectors(JSONArray array) { protected void restoreConnectors(JSONArray array) {
connectors = new ArrayList<MConnector>(); connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>(); connectorConfigBundles = new HashMap<String, ResourceBundle>();
for (Object obj : array) { for (Object obj : array) {
connectors.add(restoreConnector(obj)); connectors.add(restoreConnector(obj));
} }
@ -178,7 +178,7 @@ private MConnector restoreConnector(Object obj) {
connector.setPersistenceId(connectorId); connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIGS)) { if (object.containsKey(ALL_CONFIGS)) {
JSONObject jsonConfigBundle = JSONUtils.getJSONObject(object, ALL_CONFIGS); JSONObject jsonConfigBundle = JSONUtils.getJSONObject(object, ALL_CONFIGS);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle)); connectorConfigBundles.put(uniqueName, restoreConfigParamBundle(jsonConfigBundle));
} }
return connector; return connector;
} }

2
common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java
View File

@ -39,7 +39,7 @@ public class ConnectorsBean extends ConnectorBean {
private static final String CONNECTORS = "connectors"; private static final String CONNECTORS = "connectors";
// for "extract" // for "extract"
public ConnectorsBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) { public ConnectorsBean(List<MConnector> connectors, Map<String, ResourceBundle> bundles) {
super(connectors, bundles); super(connectors, bundles);
} }

24
common/src/main/java/org/apache/sqoop/json/LinkBean.java
View File

@ -47,7 +47,7 @@
@InterfaceStability.Unstable @InterfaceStability.Unstable
public class LinkBean implements JsonBean { public class LinkBean implements JsonBean {
static final String CONNECTOR_ID = "connector-id"; static final String CONNECTOR_NAME = "connector-name";
static final String LINK_CONFIG_VALUES = "link-config-values"; static final String LINK_CONFIG_VALUES = "link-config-values";
static final String LINK = "link"; static final String LINK = "link";
@ -57,7 +57,7 @@ public class LinkBean implements JsonBean {
private List<MLink> links; private List<MLink> links;
// Optional // Optional
private Map<Long, ResourceBundle> linkConfigBundles; private Map<String, ResourceBundle> linkConfigBundles;
// For "extract" // For "extract"
public LinkBean(MLink link) { public LinkBean(MLink link) {
@ -73,23 +73,23 @@ public LinkBean(List<MLink> links) {
// For "restore" // For "restore"
public LinkBean() { public LinkBean() {
linkConfigBundles = new HashMap<Long, ResourceBundle>(); linkConfigBundles = new HashMap<String, ResourceBundle>();
} }
public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) { public void addConnectorConfigBundle(String connectorName, ResourceBundle connectorConfigBundle) {
linkConfigBundles.put(id, connectorConfigBundle); linkConfigBundles.put(connectorName, connectorConfigBundle);
} }
public boolean hasConnectorConfigBundle(Long id) { public boolean hasConnectorConfigBundle(String connectorName) {
return linkConfigBundles.containsKey(id); return linkConfigBundles.containsKey(connectorName);
} }
public List<MLink> getLinks() { public List<MLink> getLinks() {
return links; return links;
} }
public ResourceBundle getConnectorConfigBundle(Long id) { public ResourceBundle getConnectorConfigBundle(String connectorName) {
return linkConfigBundles.get(id); return linkConfigBundles.get(connectorName);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -119,7 +119,7 @@ private JSONObject extractLink(boolean skipSensitive, MLink link) {
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime()); linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser()); linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime()); linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId()); linkJsonObject.put(CONNECTOR_NAME, link.getConnectorName());
linkJsonObject.put(LINK_CONFIG_VALUES, linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig(), skipSensitive)); extractConfigList(link.getConnectorLinkConfig(), skipSensitive));
return linkJsonObject; return linkJsonObject;
@ -141,11 +141,11 @@ protected void restoreLinks(JSONArray array) {
private MLink restoreLink(Object obj) { private MLink restoreLink(Object obj) {
JSONObject object = (JSONObject) obj; JSONObject object = (JSONObject) obj;
long connectorId = JSONUtils.getLong(object, CONNECTOR_ID); String connectorName = (String) object.get(CONNECTOR_NAME);
JSONObject connectorLinkConfig = JSONUtils.getJSONObject(object, LINK_CONFIG_VALUES); JSONObject connectorLinkConfig = JSONUtils.getJSONObject(object, LINK_CONFIG_VALUES);
List<MConfig> linkConfigs = restoreConfigs(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIGS)); List<MConfig> linkConfigs = restoreConfigs(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIGS));
List<MValidator> linkValidators = restoreValidator(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIG_VALIDATORS)); List<MValidator> linkValidators = restoreValidator(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIG_VALIDATORS));
MLink link = new MLink(connectorId, new MLinkConfig(linkConfigs, linkValidators)); MLink link = new MLink(connectorName, new MLinkConfig(linkConfigs, linkValidators));
link.setPersistenceId(JSONUtils.getLong(object, ID)); link.setPersistenceId(JSONUtils.getLong(object, ID));
link.setName(JSONUtils.getString(object, NAME)); link.setName(JSONUtils.getString(object, NAME));
link.setEnabled(JSONUtils.getBoolean(object, ENABLED)); link.setEnabled(JSONUtils.getBoolean(object, ENABLED));

18
common/src/main/java/org/apache/sqoop/model/MLink.java
View File

@ -26,7 +26,7 @@
@InterfaceAudience.Public @InterfaceAudience.Public
@InterfaceStability.Unstable @InterfaceStability.Unstable
public class MLink extends MAccountableEntity implements MClonable { public class MLink extends MAccountableEntity implements MClonable {
private long connectorId; private String connectorName;
// NOTE: we hold this in the model for easy access to the link config object, it might as well be retrieved on the fly using the connectorId // NOTE: we hold this in the model for easy access to the link config object, it might as well be retrieved on the fly using the connectorId
private final MLinkConfig connectorLinkConfig; private final MLinkConfig connectorLinkConfig;
@ -36,8 +36,8 @@ public class MLink extends MAccountableEntity implements MClonable {
* @param connectorId Connector id * @param connectorId Connector id
* @param linkConfig Connector forms * @param linkConfig Connector forms
*/ */
public MLink(long connectorId, MLinkConfig linkConfig) { public MLink(String connectorName, MLinkConfig linkConfig) {
this.connectorId = connectorId; this.connectorName = connectorName;
this.connectorLinkConfig = linkConfig; this.connectorLinkConfig = linkConfig;
} }
@ -61,7 +61,7 @@ public MLink(MLink other) {
*/ */
public MLink(MLink other, MLinkConfig linkConfig) { public MLink(MLink other, MLinkConfig linkConfig) {
super(other); super(other);
this.connectorId = other.connectorId; this.connectorName = other.connectorName;
this.connectorLinkConfig = linkConfig; this.connectorLinkConfig = linkConfig;
this.setPersistenceId(other.getPersistenceId()); this.setPersistenceId(other.getPersistenceId());
} }
@ -74,8 +74,8 @@ public String toString() {
return sb.toString(); return sb.toString();
} }
public long getConnectorId() { public String getConnectorName() {
return connectorId; return connectorName;
} }
public MLinkConfig getConnectorLinkConfig() { public MLinkConfig getConnectorLinkConfig() {
@ -90,7 +90,7 @@ public MLink clone(boolean cloneWithValue) {
if(cloneWithValue) { if(cloneWithValue) {
return new MLink(this); return new MLink(this);
} else { } else {
return new MLink(connectorId, connectorLinkConfig.clone(false)); return new MLink(connectorName, connectorLinkConfig.clone(false));
} }
} }
@ -105,14 +105,14 @@ public boolean equals(Object object) {
} }
MLink mLink = (MLink)object; MLink mLink = (MLink)object;
return (mLink.connectorId == this.connectorId) return (mLink.connectorName.equals(this.connectorName))
&& (mLink.getPersistenceId() == this.getPersistenceId()) && (mLink.getPersistenceId() == this.getPersistenceId())
&& (mLink.connectorLinkConfig.equals(this.connectorLinkConfig)); && (mLink.connectorLinkConfig.equals(this.connectorLinkConfig));
} }
@Override @Override
public int hashCode() { public int hashCode() {
int result = (int) (connectorId ^ (connectorId >>> 32)); int result = connectorName != null ? connectorName.hashCode() : 0;
result = 31 * result + (connectorLinkConfig != null ? connectorLinkConfig.hashCode() : 0); result = 31 * result + (connectorLinkConfig != null ? connectorLinkConfig.hashCode() : 0);
return result; return result;
} }

6
common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
View File

@ -45,8 +45,8 @@ public void testConnectorSerialization() {
connectors.add(BeanTestUtil.getConnector(1L, "jdbc")); connectors.add(BeanTestUtil.getConnector(1L, "jdbc"));
// Create testing bundles // Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>(); Map<String, ResourceBundle> configBundles = new HashMap<String, ResourceBundle>();
configBundles.put(1L, ConfigTestUtil.getResourceBundle()); configBundles.put("jdbc", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object // Serialize it to JSON object
ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles); ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles);
@ -62,7 +62,7 @@ public void testConnectorSerialization() {
assertEquals(connectors.size(), 1); assertEquals(connectors.size(), 1);
assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size()); assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0)); assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0));
ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L); ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get("jdbc");
assertNotNull(retrievedBundle); assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a")); assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b")); assertEquals("b", retrievedBundle.getString("b"));

20
common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java
View File

@ -42,9 +42,9 @@ public void testConnectorsSerialization() {
connectors.add(BeanTestUtil.getConnector(2L, "mysql")); connectors.add(BeanTestUtil.getConnector(2L, "mysql"));
// Create testing bundles // Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>(); Map<String, ResourceBundle> configBundles = new HashMap<String, ResourceBundle>();
configBundles.put(1L, ConfigTestUtil.getResourceBundle()); configBundles.put("jdbc", ConfigTestUtil.getResourceBundle());
configBundles.put(2L, ConfigTestUtil.getResourceBundle()); configBundles.put("mysql", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object // Serialize it to JSON object
ConnectorsBean connectorsBean = new ConnectorsBean(connectors, configBundles); ConnectorsBean connectorsBean = new ConnectorsBean(connectors, configBundles);
@ -62,7 +62,7 @@ public void testConnectorsSerialization() {
assertEquals(connectors.get(0), parsedConnectorsBean.getConnectors().get(0)); assertEquals(connectors.get(0), parsedConnectorsBean.getConnectors().get(0));
assertEquals(connectors.get(1), parsedConnectorsBean.getConnectors().get(1)); assertEquals(connectors.get(1), parsedConnectorsBean.getConnectors().get(1));
ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get(1L); ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get("jdbc");
assertNotNull(retrievedBundle); assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a")); assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b")); assertEquals("b", retrievedBundle.getString("b"));
@ -76,9 +76,9 @@ public void testSingleDirection() {
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, true)); connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, true));
// Create testing bundles // Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle()); bundles.put("jdbc", ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle()); bundles.put("mysql", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object // Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles); ConnectorsBean bean = new ConnectorsBean(connectors, bundles);
@ -105,9 +105,9 @@ public void testNoDirection() {
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, false)); connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, false));
// Create testing bundles // Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle()); bundles.put("jdbc", ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle()); bundles.put("mysql", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object // Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles); ConnectorsBean bean = new ConnectorsBean(connectors, bundles);

2
common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java
View File

@ -52,7 +52,7 @@ public static MJob createJob(String connectorName, String jobName, Long jobId, D
} }
public static MLink getLink(String connectorName) { public static MLink getLink(String connectorName) {
return new MLink(1, getConnector(1L, connectorName).getLinkConfig()); return new MLink(connectorName, getConnector(1L, connectorName).getLinkConfig());
} }
public static MConnector getConnector(Long connectorId, String connectorName) { public static MConnector getConnector(Long connectorId, String connectorName) {

2
common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
View File

@ -51,7 +51,7 @@ public void testInitialization() {
MValidator validator = new MValidator("test", ""); MValidator validator = new MValidator("test", "");
validators.add(validator); validators.add(validator);
MAccountableEntity link = new MLink(123l, new MLinkConfig(configs, validators)); MAccountableEntity link = new MLink("connector_test", new MLinkConfig(configs, validators));
// Initially creation date and last update date is same // Initially creation date and last update date is same
assertEquals(link.getCreationDate(), link.getLastUpdateDate()); assertEquals(link.getCreationDate(), link.getLastUpdateDate());
Date testCreationDate = new Date(); Date testCreationDate = new Date();

6
common/src/test/java/org/apache/sqoop/model/TestMLink.java
View File

@ -35,14 +35,14 @@ public class TestMLink {
public void testInitialization() { public void testInitialization() {
// Test default constructor // Test default constructor
MLink link = link(); MLink link = link();
assertEquals(123l, link.getConnectorId()); assertEquals("connector_test", link.getConnectorName());
assertEquals("Vampire", link.getName()); assertEquals("Vampire", link.getName());
assertEquals("Buffy", link.getCreationUser()); assertEquals("Buffy", link.getCreationUser());
assertEquals(linkConfig(), link.getConnectorLinkConfig()); assertEquals(linkConfig(), link.getConnectorLinkConfig());
// Test copy constructor // Test copy constructor
MLink copy = new MLink(link); MLink copy = new MLink(link);
assertEquals(123l, copy.getConnectorId()); assertEquals("connector_test", copy.getConnectorName());
assertEquals("Vampire", copy.getName()); assertEquals("Vampire", copy.getName());
assertEquals("Buffy", copy.getCreationUser()); assertEquals("Buffy", copy.getCreationUser());
assertEquals(link.getCreationDate(), copy.getCreationDate()); assertEquals(link.getCreationDate(), copy.getCreationDate());
@ -75,7 +75,7 @@ public void testClone() {
} }
private MLink link() { private MLink link() {
MLink link = new MLink(123l, linkConfig()); MLink link = new MLink("connector_test", linkConfig());
link.setName("Vampire"); link.setName("Vampire");
link.setCreationUser("Buffy"); link.setCreationUser("Buffy");
return link; return link;

8
core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
View File

@ -117,12 +117,12 @@ public Set<Long> getConnectorIds() {
return idToNameMap.keySet(); return idToNameMap.keySet();
} }
public Map<Long, ResourceBundle> getResourceBundles(Locale locale) { public Map<String, ResourceBundle> getResourceBundles(Locale locale) {
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
for (ConnectorHandler handler : handlerMap.values()) { for (ConnectorHandler handler : handlerMap.values()) {
long id = handler.getConnectorConfigurable().getPersistenceId(); String connectorName = handler.getConnectorConfigurable().getUniqueName();
ResourceBundle bundle = handler.getSqoopConnector().getBundle(locale); ResourceBundle bundle = handler.getSqoopConnector().getBundle(locale);
bundles.put(id, bundle); bundles.put(connectorName, bundle);
} }
return bundles; return bundles;
} }

18
core/src/main/java/org/apache/sqoop/driver/JobManager.java
View File

@ -339,22 +339,22 @@ public MSubmission start(String jobName, HttpEventContext ctx) {
private JobRequest createJobRequest(MSubmission submission, MJob job) { private JobRequest createJobRequest(MSubmission submission, MJob job) {
// get from/to connections for the job // get from/to connections for the job
MLink fromConnection = getLink(job.getFromLinkId()); MLink fromLink = getLink(job.getFromLinkId());
MLink toConnection = getLink(job.getToLinkId()); MLink toLink = getLink(job.getToLinkId());
// get from/to connectors for the connection // get from/to connectors for the connection
SqoopConnector fromConnector = getSqoopConnector(fromConnection.getConnectorId()); SqoopConnector fromConnector = getSqoopConnector(fromLink.getConnectorName());
validateSupportedDirection(fromConnector, Direction.FROM); validateSupportedDirection(fromConnector, Direction.FROM);
SqoopConnector toConnector = getSqoopConnector(toConnection.getConnectorId()); SqoopConnector toConnector = getSqoopConnector(toLink.getConnectorName());
validateSupportedDirection(toConnector, Direction.TO); validateSupportedDirection(toConnector, Direction.TO);
// link config for the FROM part of the job // link config for the FROM part of the job
Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass()); Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass());
ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig); ConfigUtils.fromConfigs(fromLink.getConnectorLinkConfig().getConfigs(), fromLinkConfig);
// link config for the TO part of the job // link config for the TO part of the job
Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass()); Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass());
ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig); ConfigUtils.fromConfigs(toLink.getConnectorLinkConfig().getConfigs(), toLinkConfig);
// from config for the job // from config for the job
Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM)); Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
@ -470,7 +470,11 @@ MSubmission createJobSubmission(HttpEventContext ctx, long jobId) {
return summary; return summary;
} }
SqoopConnector getSqoopConnector(long connnectorId) { SqoopConnector getSqoopConnector(String connnectorName) {
return ConnectorManager.getInstance().getSqoopConnector(connnectorName);
}
SqoopConnector getSqoopConnector(Long connnectorId) {
return ConnectorManager.getInstance().getSqoopConnector(connnectorId); return ConnectorManager.getInstance().getSqoopConnector(connnectorId);
} }

4
core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
View File

@ -109,7 +109,7 @@ public void testUnsupportedDirectionForConnector() {
@Test @Test
public void testGetLink() { public void testGetLink() {
MLink testLink = new MLink(123l, null); MLink testLink = new MLink("connector_test", null);
testLink.setEnabled(true); testLink.setEnabled(true);
MLink mConnectionSpy = org.mockito.Mockito.spy(testLink); MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
@ -121,7 +121,7 @@ public void testGetLink() {
@Test @Test
public void testDisabledLink() { public void testDisabledLink() {
MLink testConnection = new MLink(123l, null); MLink testConnection = new MLink("connector_test", null);
testConnection.setPersistenceId(1234); testConnection.setPersistenceId(1234);
testConnection.setEnabled(false); testConnection.setEnabled(false);
SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection: " SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection: "

18
core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
View File

@ -228,7 +228,7 @@ public void testConnectorConfigUpgradeWithValidLinksAndJobs() {
// prepare the links and jobs // prepare the links and jobs
// the connector Id for both are the same // the connector Id for both are the same
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1)); List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 2)); List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 2));
// mock necessary methods for upgradeConnector() procedure // mock necessary methods for upgradeConnector() procedure
@ -390,7 +390,7 @@ public void testConnectorConfigUpgradeHandlerWithFindJobsForConnectorError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1)); List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000, SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@ -424,7 +424,7 @@ public void testConnectorConfigUpgradeHandlerWithDeleteJobInputsError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1)); List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade
@ -462,7 +462,7 @@ public void testConnectorConfigUpgradeHandlerWithDeleteLinkInputsError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1)); List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -500,7 +500,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateConnectorError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock); when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1)); List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -542,7 +542,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateLinkError() {
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class); when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1)); List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -588,7 +588,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateJobError() {
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class); when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector); when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));; List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));;
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1)); List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class)); doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class)); doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -775,8 +775,8 @@ private MDriver anotherDriver() {
return driver; return driver;
} }
private MLink link(long linkId, String linkName, long connectorId) { private MLink link(long linkId, String linkName, String connectorName) {
MLink link = new MLink(connectorId, new MLinkConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>())); MLink link = new MLink(connectorName, new MLinkConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>()));
link.setPersistenceId(linkId); link.setPersistenceId(linkId);
link.setName(linkName); link.setName(linkName);
return link; return link;

12
repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
View File

@ -371,11 +371,13 @@ public void registerDriver(MDriver mDriver, Connection conn) {
@Override @Override
public void createLink(MLink link, Connection conn) { public void createLink(MLink link, Connection conn) {
int result; int result;
MConnector mConnector;
try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertLink(), try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertLink(),
Statement.RETURN_GENERATED_KEYS)) { Statement.RETURN_GENERATED_KEYS)) {
stmt.setString(1, link.getName()); stmt.setString(1, link.getName());
stmt.setLong(2, link.getConnectorId()); mConnector = findConnector(link.getConnectorName(), conn);
stmt.setLong(2, mConnector.getPersistenceId());
stmt.setBoolean(3, link.getEnabled()); stmt.setBoolean(3, link.getEnabled());
stmt.setString(4, link.getCreationUser()); stmt.setString(4, link.getCreationUser());
stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime())); stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime()));
@ -1515,7 +1517,6 @@ private List<MLink> loadLinksForUpgrade(PreparedStatement stmt,
try (ResultSet rsConnection = stmt.executeQuery(); try (ResultSet rsConnection = stmt.executeQuery();
PreparedStatement connectorConfigFetchStatement = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable()); PreparedStatement connectorConfigFetchStatement = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable());
PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());) { PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());) {
while(rsConnection.next()) { while(rsConnection.next()) {
long id = rsConnection.getLong(1); long id = rsConnection.getLong(1);
String name = rsConnection.getString(2); String name = rsConnection.getString(2);
@ -1525,6 +1526,7 @@ private List<MLink> loadLinksForUpgrade(PreparedStatement stmt,
Date creationDate = rsConnection.getTimestamp(6); Date creationDate = rsConnection.getTimestamp(6);
String updateUser = rsConnection.getString(7); String updateUser = rsConnection.getString(7);
Date lastUpdateDate = rsConnection.getTimestamp(8); Date lastUpdateDate = rsConnection.getTimestamp(8);
String connectorName = rsConnection.getString(9);
connectorConfigFetchStatement.setLong(1, connectorId); connectorConfigFetchStatement.setLong(1, connectorId);
connectorConfigInputStatement.setLong(1, id); connectorConfigInputStatement.setLong(1, id);
@ -1535,7 +1537,8 @@ private List<MLink> loadLinksForUpgrade(PreparedStatement stmt,
loadConnectorConfigs(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement, loadConnectorConfigs(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement,
connectorConfigInputStatement, 2, conn); connectorConfigInputStatement, 2, conn);
MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig, Collections.EMPTY_LIST));
MLink link = new MLink(connectorName, new MLinkConfig(connectorLinkConfig, Collections.EMPTY_LIST));
link.setPersistenceId(id); link.setPersistenceId(id);
link.setName(name); link.setName(name);
@ -1576,7 +1579,8 @@ private List<MLink> loadLinks(PreparedStatement stmt,
configStmt.setLong(1, connectorId); configStmt.setLong(1, connectorId);
inputStmt.setLong(1, id); inputStmt.setLong(1, id);
loadInputsForConfigs(connectorLinkConfig, configStmt, inputStmt); loadInputsForConfigs(connectorLinkConfig, configStmt, inputStmt);
MLink link = new MLink(connectorId, connectorLinkConfig);
MLink link = new MLink(connectorName, connectorLinkConfig);
link.setPersistenceId(id); link.setPersistenceId(id);
link.setName(name); link.setName(name);

2
repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
View File

@ -97,7 +97,7 @@ public void testEntityDataSerialization() throws Exception {
map.put("A", "B"); map.put("A", "B");
// Connection object with all various values // Connection object with all various values
MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig()); MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig());
MLinkConfig linkConfig = link.getConnectorLinkConfig(); MLinkConfig linkConfig = link.getConnectorLinkConfig();
assertEquals(linkConfig.getStringInput("LINK1.I1").getEditable(), InputEditable.USER_ONLY); assertEquals(linkConfig.getStringInput("LINK1.I1").getEditable(), InputEditable.USER_ONLY);
assertEquals(linkConfig.getStringInput("LINK1.I1").getOverrides(), "LINK1.I2"); assertEquals(linkConfig.getStringInput("LINK1.I1").getOverrides(), "LINK1.I2");

2
repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
View File

@ -304,6 +304,6 @@ public void testDeleteLink() throws Exception {
} }
public MLink getLink() { public MLink getLink() {
return new MLink(1, handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig()); return new MLink("A", handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig());
} }
} }

2
repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
View File

@ -94,7 +94,7 @@ protected MDriver getDriver() {
} }
protected MLink getLink(String name, MConnector connector) { protected MLink getLink(String name, MConnector connector) {
MLink link = new MLink(connector.getPersistenceId(), MLink link = new MLink(connector.getUniqueName(),
connector.getLinkConfig()); connector.getLinkConfig());
link.setName(name); link.setName(name);
fillLink(link); fillLink(link);

2
repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
View File

@ -83,7 +83,7 @@ protected MDriver getDriver() {
} }
protected MLink getLink(String name, MConnector connector) { protected MLink getLink(String name, MConnector connector) {
MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig()); MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig());
link.setName(name); link.setName(name);
fillLink(link); fillLink(link);
return link; return link;

4
server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java
View File

@ -48,7 +48,7 @@ public ConnectorRequestHandler() {
@Override @Override
public JsonBean handleEvent(RequestContext ctx) { public JsonBean handleEvent(RequestContext ctx) {
List<MConnector> connectors; List<MConnector> connectors;
Map<Long, ResourceBundle> configParamBundles; Map<String, ResourceBundle> configParamBundles;
Locale locale = ctx.getAcceptLanguageHeader(); Locale locale = ctx.getAcceptLanguageHeader();
String cIdentifier = ctx.getLastURLElement(); String cIdentifier = ctx.getLastURLElement();
@ -73,7 +73,7 @@ public JsonBean handleEvent(RequestContext ctx) {
configParamBundles = new HashMap<>(); configParamBundles = new HashMap<>();
MConnector connector = ConnectorManager.getInstance().getConnectorConfigurable(mConnector.getUniqueName()); MConnector connector = ConnectorManager.getInstance().getConnectorConfigurable(mConnector.getUniqueName());
configParamBundles.put(connector.getPersistenceId(), configParamBundles.put(connector.getUniqueName(),
ConnectorManager.getInstance().getResourceBundle(mConnector.getUniqueName(), locale)); ConnectorManager.getInstance().getResourceBundle(mConnector.getUniqueName(), locale));
AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(), AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(),

14
server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
View File

@ -130,7 +130,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
} }
MLink postedLink = links.get(0); MLink postedLink = links.get(0);
MConnector mConnector = HandlerUtils.getConnectorFromConnectorId(postedLink.getConnectorId()); MConnector mConnector = HandlerUtils.getConnectorFromConnectorName(postedLink.getConnectorName());
// Authorization check // Authorization check
if (create) { if (create) {
@ -142,7 +142,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
} }
MLinkConfig linkConfig = ConnectorManager.getInstance() MLinkConfig linkConfig = ConnectorManager.getInstance()
.getConnectorConfigurable(postedLink.getConnectorId()).getLinkConfig(); .getConnectorConfigurable(postedLink.getConnectorName()).getLinkConfig();
if (!linkConfig.equals(postedLink.getConnectorLinkConfig())) { if (!linkConfig.equals(postedLink.getConnectorLinkConfig())) {
throw new SqoopException(ServerError.SERVER_0003, "Detected incorrect link config structure"); throw new SqoopException(ServerError.SERVER_0003, "Detected incorrect link config structure");
} }
@ -156,7 +156,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
} }
// Associated connector for this link // Associated connector for this link
SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector( SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector(
postedLink.getConnectorId()); postedLink.getConnectorName());
// Validate user supplied config data // Validate user supplied config data
ConfigValidationResult connectorLinkConfigValidation = ConfigUtils.validateConfigs(postedLink ConfigValidationResult connectorLinkConfigValidation = ConfigUtils.validateConfigs(postedLink
@ -240,10 +240,10 @@ private LinksBean createLinksBean(List<MLink> links, Locale locale) {
private void addConnectorConfigBundle(Locale locale, LinkBean bean) { private void addConnectorConfigBundle(Locale locale, LinkBean bean) {
// Add associated resources into the bean // Add associated resources into the bean
for (MLink link : bean.getLinks()) { for (MLink link : bean.getLinks()) {
long connectorId = link.getConnectorId(); String connectorName = link.getConnectorName();
if (!bean.hasConnectorConfigBundle(connectorId)) { if (!bean.hasConnectorConfigBundle(connectorName)) {
bean.addConnectorConfigBundle(connectorId, ConnectorManager.getInstance() bean.addConnectorConfigBundle(connectorName, ConnectorManager.getInstance()
.getResourceBundle(connectorId, locale)); .getResourceBundle(connectorName, locale));
} }
} }
} }

8
shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
View File

@ -20,6 +20,7 @@
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.Constants;
@ -66,10 +67,13 @@ private Status cloneJob(String jobArg, List<String> args, boolean isInteractive)
MJob job = client.getJob(jobArg); MJob job = client.getJob(jobArg);
job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT); job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
// TODO: the job should be related with connector by connectorName
MConnector fromConnector = getClient().getConnector(job.getFromConnectorId());
MConnector toConnector = getClient().getConnector(job.getToConnectorId());
ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle( ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle(
job.getFromConnectorId()); fromConnector.getUniqueName());
ResourceBundle toConnectorBundle = client.getConnectorConfigBundle( ResourceBundle toConnectorBundle = client.getConnectorConfigBundle(
job.getToConnectorId()); toConnector.getUniqueName());
ResourceBundle driverConfigBundle = client.getDriverConfigBundle(); ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
Status status = Status.OK; Status status = Status.OK;

2
shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
View File

@ -70,7 +70,7 @@ private Status cloneLink(String linkArg, List<String> args, boolean isInteractiv
Status status = Status.OK; Status status = Status.OK;
ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId()); ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(link.getConnectorName());
if (isInteractive) { if (isInteractive) {
printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG); printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG);

4
shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
View File

@ -90,9 +90,9 @@ private Status createJob(String fromLinkArg, String toLinkArg, List<String> args
} }
ResourceBundle fromConfigBundle = getClient().getConnectorConfigBundle( ResourceBundle fromConfigBundle = getClient().getConnectorConfigBundle(
job.getFromConnectorId()); fromConnector.getUniqueName());
ResourceBundle toConfigBundle = getClient().getConnectorConfigBundle( ResourceBundle toConfigBundle = getClient().getConnectorConfigBundle(
job.getToConnectorId()); toConnector.getUniqueName());
ResourceBundle driverConfigBundle = getClient().getDriverConfigBundle(); ResourceBundle driverConfigBundle = getClient().getDriverConfigBundle();
Status status = Status.OK; Status status = Status.OK;

7
shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
View File

@ -72,24 +72,23 @@ private Status createLink(CommandLine line, List<String> args, boolean isInterac
cid = getLong(line, Constants.OPT_CID); cid = getLong(line, Constants.OPT_CID);
getClient().getConnector(cid); getClient().getConnector(cid);
//Would have thrown an exception before this if input was neither a valid name nor an id //Would have thrown an exception before this if input was an invalid connector name
//This will do an extra getConnector() call again inside createLink() //This will do an extra getConnector() call again inside createLink()
//but should not matter as connectors are cached //but should not matter as connectors are cached
link = getClient().createLink(cid); link = getClient().createLink(connectorName);
printlnResource(Constants.RES_CREATE_CREATING_LINK, cid); printlnResource(Constants.RES_CREATE_CREATING_LINK, cid);
} }
else { else {
//Command line had connector name //Command line had connector name
//This will do an extra getConnector() call again inside createLink() but //This will do an extra getConnector() call again inside createLink() but
//should not matter as connectors are cached //should not matter as connectors are cached
cid = connector.getPersistenceId();
link = getClient().createLink(connectorName); link = getClient().createLink(connectorName);
printlnResource(Constants.RES_CREATE_CREATING_LINK, connectorName); printlnResource(Constants.RES_CREATE_CREATING_LINK, connectorName);
} }
ConsoleReader reader = getConsoleReader(); ConsoleReader reader = getConsoleReader();
ResourceBundle connectorConfigBundle = getClient().getConnectorConfigBundle(cid); ResourceBundle connectorConfigBundle = getClient().getConnectorConfigBundle(connectorName);
Status status = Status.OK; Status status = Status.OK;
if (isInteractive) { if (isInteractive) {

2
shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
View File

@ -124,6 +124,6 @@ private void displayConnector(MConnector connector) {
connector.getVersion(), connector.getVersion(),
connector.getSupportedDirections().toString() connector.getSupportedDirections().toString()
); );
displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getPersistenceId())); displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getUniqueName()));
} }
} }

9
shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
View File

@ -159,15 +159,20 @@ private void displayJob(MJob job) {
job.getLastUpdateUser(), job.getLastUpdateUser(),
formatter.format(job.getLastUpdateDate()) formatter.format(job.getLastUpdateDate())
); );
// TODO: should be removed when MJob link with Connector by name.
MConnector fromConnector = getClient().getConnector(job.getFromConnectorId());
MConnector toConnector = getClient().getConnector(job.getToConnectorId());
displayConfig(job.getDriverConfig().getConfigs(), displayConfig(job.getDriverConfig().getConfigs(),
client.getDriverConfigBundle()); client.getDriverConfigBundle());
printlnResource(Constants.RES_SHOW_PROMPT_JOB_FROM_LID_INFO, printlnResource(Constants.RES_SHOW_PROMPT_JOB_FROM_LID_INFO,
job.getFromLinkId()); job.getFromLinkId());
displayConfig(job.getFromJobConfig().getConfigs(), displayConfig(job.getFromJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getFromConnectorId())); client.getConnectorConfigBundle(fromConnector.getUniqueName()));
printlnResource(Constants.RES_SHOW_PROMPT_JOB_TO_LID_INFO, printlnResource(Constants.RES_SHOW_PROMPT_JOB_TO_LID_INFO,
job.getToLinkId()); job.getToLinkId());
displayConfig(job.getToJobConfig().getConfigs(), displayConfig(job.getToJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getToConnectorId())); client.getConnectorConfigBundle(toConnector.getUniqueName()));
} }
} }

40
shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
View File

@ -81,17 +81,16 @@ private void showSummary() {
List<String> ids = new LinkedList<String>(); List<String> ids = new LinkedList<String>();
List<String> names = new LinkedList<String>(); List<String> names = new LinkedList<String>();
List<String> connectorIds = new LinkedList<String>(); List<String> connectorIds = new LinkedList<String>();
List<String> connectorNames = new LinkedList<String>();
List<String> availabilities = new LinkedList<String>(); List<String> availabilities = new LinkedList<String>();
for (MLink link : links) { for (MLink link : links) {
ids.add(String.valueOf(link.getPersistenceId())); ids.add(String.valueOf(link.getPersistenceId()));
names.add(link.getName()); names.add(link.getName());
connectorIds.add(String.valueOf(link.getConnectorId())); connectorNames.add(link.getConnectorName());
availabilities.add(String.valueOf(link.getEnabled())); availabilities.add(String.valueOf(link.getEnabled()));
} }
List<String> connectorNames = getConnectorNames(connectorIds);
TableDisplayer.display(header, ids, names, connectorIds, connectorNames, availabilities); TableDisplayer.display(header, ids, names, connectorIds, connectorNames, availabilities);
} }
@ -126,41 +125,10 @@ private void displayLink(MLink link) {
formatter.format(link.getLastUpdateDate()) formatter.format(link.getLastUpdateDate())
); );
long connectorId = link.getConnectorId(); printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, link.getConnectorName());
MConnector connector = client.getConnector(connectorId);
String connectorName = "";
if (connector != null) {
connectorName = connector.getUniqueName();
}
printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, connectorName, connectorId);
// Display link config // Display link config
displayConfig(link.getConnectorLinkConfig().getConfigs(), displayConfig(link.getConnectorLinkConfig().getConfigs(),
client.getConnectorConfigBundle(connectorId)); client.getConnectorConfigBundle(link.getConnectorName()));
}
private List<String> getConnectorNames(List<String> connectorIds) {
Map<String, String> connectorIdToName = new HashMap<String, String>();
for (String connectorId : connectorIds) {
if (!connectorIdToName.containsKey(connectorId)) {
try {
MConnector connector = client.getConnector(Long.parseLong(connectorId));
if (connector != null) {
connectorIdToName.put(connectorId, connector.getUniqueName());
}
} catch (SqoopException ex) {
connectorIdToName.put(connectorId, "Access Denied");
}
}
}
List<String> connectorNames = new ArrayList<String>();
for (String connectorId : connectorIds) {
if (connectorIdToName.get(connectorId) != null) {
connectorNames.add(connectorIdToName.get(connectorId));
} else {
connectorNames.add("");
}
}
return connectorNames;
} }
} }

9
shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
View File

@ -20,6 +20,7 @@
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MJob;
import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.utils.ConfigDisplayer; import org.apache.sqoop.shell.utils.ConfigDisplayer;
@ -67,10 +68,14 @@ private Status updateJob(String jobArg, List<String> args, boolean isInteractive
// TODO(SQOOP-1634): using from/to and driver config id, this call can be avoided // TODO(SQOOP-1634): using from/to and driver config id, this call can be avoided
MJob job = client.getJob(jobArg); MJob job = client.getJob(jobArg);
// TODO: should be removed when MJob link with Connector by name.
MConnector fromConnector = getClient().getConnector(job.getFromConnectorId());
MConnector toConnector = getClient().getConnector(job.getToConnectorId());
ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle( ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle(
job.getFromConnectorId()); fromConnector.getUniqueName());
ResourceBundle toConnectorBundle = client.getConnectorConfigBundle( ResourceBundle toConnectorBundle = client.getConnectorConfigBundle(
job.getToConnectorId()); toConnector.getUniqueName());
ResourceBundle driverConfigBundle = client.getDriverConfigBundle(); ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
Status status = Status.OK; Status status = Status.OK;

3
shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
View File

@ -65,8 +65,7 @@ private Status updateLink(String linkArg, List<String> args, boolean isInteracti
// TODO(SQOOP-1634): using link config id, this call can be avoided // TODO(SQOOP-1634): using link config id, this call can be avoided
MLink link = client.getLink(linkArg); MLink link = client.getLink(linkArg);
ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorName());
ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId());
Status status = Status.OK; Status status = Status.OK;

33
shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java
View File

@ -41,24 +41,7 @@
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.InputEditable; import org.apache.sqoop.model.*;
import org.apache.sqoop.model.MBooleanInput;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MDateTimeInput;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MListInput;
import org.apache.sqoop.model.MLongInput;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.model.MValidator;
import org.apache.sqoop.shell.core.Constants; import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.core.ShellError; import org.apache.sqoop.shell.core.ShellError;
import org.apache.sqoop.utils.MapResourceBundle; import org.apache.sqoop.utils.MapResourceBundle;
@ -104,9 +87,9 @@ public Enumeration<String> getKeys() {
@Test @Test
public void testCloneLink() { public void testCloneLink() {
ShellEnvironment.setInteractive(false); ShellEnvironment.setInteractive(false);
MLink link = new MLink(1L, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())); MLink link = new MLink("connector_name_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link); when(client.getLink("link_test")).thenReturn(link);
when(client.getConnectorConfigBundle(1L)).thenReturn(new MapResourceBundle(new HashMap())); when(client.getConnectorConfigBundle("connector_name_test")).thenReturn(new MapResourceBundle(new HashMap()));
when(client.saveLink(link)).thenReturn(Status.OK); when(client.saveLink(link)).thenReturn(Status.OK);
// clone link -lid link_test // clone link -lid link_test
@ -136,10 +119,11 @@ public void testCloneLink() {
public void testCloneLinkInteractive() { public void testCloneLinkInteractive() {
ShellEnvironment.setInteractive(true); ShellEnvironment.setInteractive(true);
initEnv(); initEnv();
MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>())); MLink link = new MLink("connector_name_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link); when(client.getLink("link_test")).thenReturn(link);
when(client.getConnectorConfigBundle(1L)).thenReturn(resourceBundle); when(client.getConnectorConfigBundle("connector_name_test")).thenReturn(resourceBundle);
when(client.saveLink(link)).thenReturn(Status.OK); when(client.saveLink(link)).thenReturn(Status.OK);
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
// clone link -lid link_test // clone link -lid link_test
initData("linkname\r" + // link name initData("linkname\r" + // link name
@ -176,9 +160,10 @@ public void testCloneJob() {
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())); new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job); when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(new MapResourceBundle(new HashMap())); when(client.getConnectorConfigBundle(any(String.class))).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap())); when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap()));
when(client.saveJob(job)).thenReturn(Status.OK); when(client.saveJob(job)).thenReturn(Status.OK);
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
// clone job -jid job_test // clone job -jid job_test
Status status = (Status) cloneCmd.execute(Arrays.asList(Constants.FN_JOB, "-jid", "job_test")); Status status = (Status) cloneCmd.execute(Arrays.asList(Constants.FN_JOB, "-jid", "job_test"));
@ -211,7 +196,7 @@ public void testCloneJobInteractive() {
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()), new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>())); new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job); when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
when(client.getDriverConfigBundle()).thenReturn(resourceBundle); when(client.getDriverConfigBundle()).thenReturn(resourceBundle);
when(client.saveJob(job)).thenReturn(Status.OK); when(client.saveJob(job)).thenReturn(Status.OK);

9
shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java
View File

@ -104,7 +104,8 @@ public Enumeration<String> getKeys() {
public void testCreateLink() { public void testCreateLink() {
ShellEnvironment.setInteractive(false); ShellEnvironment.setInteractive(false);
when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null)); when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null));
when(client.createLink("connector_test")).thenReturn(new MLink(1, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()))); when(client.createLink("connector_test")).thenReturn(
new MLink("connector_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
when(client.saveLink(any(MLink.class))).thenReturn(Status.OK); when(client.saveLink(any(MLink.class))).thenReturn(Status.OK);
// create link -c connector_test // create link -c connector_test
@ -144,10 +145,10 @@ public void testCreateLinkInteractive() {
ShellEnvironment.setInteractive(true); ShellEnvironment.setInteractive(true);
initEnv(); initEnv();
when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null)); when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null));
MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>())); MLink link = new MLink("connector_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
when(client.createLink("connector_test")).thenReturn(link); when(client.createLink("connector_test")).thenReturn(link);
when(client.saveLink(any(MLink.class))).thenReturn(Status.OK); when(client.saveLink(any(MLink.class))).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
// create link -c connector_test // create link -c connector_test
initData("linkname\r" + // link name initData("linkname\r" + // link name
@ -228,7 +229,7 @@ public void testCreateJobInteractive() {
when(client.getConnector(1)).thenReturn(fromConnector); when(client.getConnector(1)).thenReturn(fromConnector);
when(client.getConnector(2)).thenReturn(toConnector); when(client.getConnector(2)).thenReturn(toConnector);
when(client.saveJob(any(MJob.class))).thenReturn(Status.OK); when(client.saveJob(any(MJob.class))).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
when(client.getDriverConfigBundle()).thenReturn(resourceBundle); when(client.getDriverConfigBundle()).thenReturn(resourceBundle);
// create job -f link_from -to link_to // create job -f link_from -to link_to

3
shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java
View File

@ -203,7 +203,7 @@ public void testShowDriver() {
@Test @Test
public void testShowLink() { public void testShowLink() {
when(client.getLinks()).thenReturn(new ArrayList<MLink>()); when(client.getLinks()).thenReturn(new ArrayList<MLink>());
when(client.getLink(any(String.class))).thenReturn(new MLink(1L, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()))); when(client.getLink(any(String.class))).thenReturn(new MLink("connector_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
// show link summary // show link summary
out.reset(); out.reset();
@ -234,6 +234,7 @@ public void testShowLink() {
@Test @Test
public void testShowJob() { public void testShowJob() {
when(client.getJobs()).thenReturn(new ArrayList<MJob>()); when(client.getJobs()).thenReturn(new ArrayList<MJob>());
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
when(client.getJob("1")).thenReturn(new MJob(1L, 2L, 1L, 2L, when(client.getJob("1")).thenReturn(new MJob(1L, 2L, 1L, 2L,
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),

20
shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java
View File

@ -105,10 +105,10 @@ public Enumeration<String> getKeys() {
@Test @Test
public void testUpdateLink() throws InterruptedException { public void testUpdateLink() throws InterruptedException {
ShellEnvironment.setInteractive(false); ShellEnvironment.setInteractive(false);
MLink link = new MLink(1L, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())); MLink link = new MLink("connector_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link); when(client.getLink("link_test")).thenReturn(link);
when(client.getConnectorConfigBundle(1L)).thenReturn(new MapResourceBundle(new HashMap())); when(client.getConnectorConfigBundle("connector_test")).thenReturn(new MapResourceBundle(new HashMap()));
when(client.updateLink(link)).thenReturn(Status.OK); when(client.updateLink(any(MLink.class))).thenReturn(Status.OK);
// update link -lid link_test // update link -lid link_test
Status status = (Status) updateCmd.execute(Arrays.asList(Constants.FN_LINK, "-lid", "link_test")); Status status = (Status) updateCmd.execute(Arrays.asList(Constants.FN_LINK, "-lid", "link_test"));
@ -134,14 +134,14 @@ public void testUpdateLink() throws InterruptedException {
} }
@Test @Test
public void testUpdateLinkInteractive() { public void testUpdateLinkInteractive() throws Exception {
ShellEnvironment.setInteractive(true); ShellEnvironment.setInteractive(true);
initEnv(); initEnv();
when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null)); when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null));
MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>())); MLink link = new MLink("connector_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link); when(client.getLink("link_test")).thenReturn(link);
when(client.updateLink(link)).thenReturn(Status.OK); when(client.updateLink(any(MLink.class))).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
// update link -lid link_test // update link -lid link_test
initData("linkname\r" + // link name initData("linkname\r" + // link name
@ -178,7 +178,8 @@ public void testUpdateJob() throws InterruptedException {
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())); new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job); when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(new MapResourceBundle(new HashMap())); when(client.getConnector(any(Long.class))).thenReturn(new MConnector("connect_test", "", "", null, null, null));
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap())); when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap()));
when(client.updateJob(job)).thenReturn(Status.OK); when(client.updateJob(job)).thenReturn(Status.OK);
@ -213,7 +214,8 @@ public void testUpdateJobInteractive() {
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()), new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>())); new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job); when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle); when(client.getConnector(any(Long.class))).thenReturn(new MConnector("connect_test", "", "", null, null, null));
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
when(client.getDriverConfigBundle()).thenReturn(resourceBundle); when(client.getDriverConfigBundle()).thenReturn(resourceBundle);
when(client.updateJob(job)).thenReturn(Status.OK); when(client.updateJob(job)).thenReturn(Status.OK);

1
test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
View File

@ -22,7 +22,6 @@
import java.lang.reflect.Method; import java.lang.reflect.Method;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;

6
test/src/test/java/org/apache/sqoop/integration/connector/hdfs/S3Test.java
View File

@ -90,7 +90,7 @@ public void testImportExport() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// DB -> S3 // DB -> S3
MJob db2aws = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob db2aws = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
fillRdbmsFromConfig(db2aws, "id"); fillRdbmsFromConfig(db2aws, "id");
fillHdfsToConfig(db2aws, ToFormat.TEXT_FILE); fillHdfsToConfig(db2aws, ToFormat.TEXT_FILE);
@ -110,7 +110,7 @@ public void testImportExport() throws Exception {
assertEquals(provider.rowCount(getTableName()), 0); assertEquals(provider.rowCount(getTableName()), 0);
// S3 -> DB // S3 -> DB
MJob aws2db = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); MJob aws2db = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
fillHdfsFromConfig(aws2db); fillHdfsFromConfig(aws2db);
fillRdbmsToConfig(aws2db); fillRdbmsToConfig(aws2db);
@ -145,7 +145,7 @@ public void testIncrementalRead() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// S3 -> HDFS // S3 -> HDFS
MJob aws2hdfs = getClient().createJob(s3Link.getPersistenceId(), hdfsLink.getPersistenceId()); MJob aws2hdfs = getClient().createJob(s3Link.getName(), hdfsLink.getName());
fillHdfsFromConfig(aws2hdfs); fillHdfsFromConfig(aws2hdfs);
aws2hdfs.getFromJobConfig().getEnumInput("incremental.incrementalType").setValue(IncrementalType.NEW_FILES); aws2hdfs.getFromJobConfig().getEnumInput("incremental.incrementalType").setValue(IncrementalType.NEW_FILES);

4
tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
View File

@ -278,7 +278,7 @@ private long loadLink(MLink link) {
Repository repository = RepositoryManager.getInstance().getRepository(); Repository repository = RepositoryManager.getInstance().getRepository();
MConnector mConnector = ConnectorManager.getInstance().getConnectorConfigurable(link.getConnectorId()); MConnector mConnector = ConnectorManager.getInstance().getConnectorConfigurable(link.getConnectorName());
ConnectorConfigurableUpgrader connectorConfigUpgrader = ConnectorManager.getInstance().getSqoopConnector(mConnector.getUniqueName()).getConfigurableUpgrader(null); ConnectorConfigurableUpgrader connectorConfigUpgrader = ConnectorManager.getInstance().getSqoopConnector(mConnector.getUniqueName()).getConfigurableUpgrader(null);
List<MConfig> connectorConfigs = mConnector.getLinkConfig().clone(false).getConfigs(); List<MConfig> connectorConfigs = mConnector.getLinkConfig().clone(false).getConfigs();
@ -291,7 +291,7 @@ private long loadLink(MLink link) {
// Transform config structures to objects for validations // Transform config structures to objects for validations
SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector( SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector(
link.getConnectorId()); link.getConnectorName());
Object connectorConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass()); Object connectorConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass());