github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-05 05:59:07 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2690: Sqoop2: Use connector name in MLink.

Browse Source 
(Colin Ma via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2015-12-07 11:18:12 +01:00
parent 1b228f308c
commit 794adbcf3d
40 changed files with 212 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
client/src/main/java/org/apache/sqoop/client/SqoopClient.java
View File

@ -61,11 +61,11 @@ public class SqoopClient {
/**
* All cached connectors.
*/
private Map<Long, MConnector> connectors;
private Map<String, MConnector> connectors;
/**
* All cached config params for every registered connector in the sqoop system.
*/
private Map<Long, ResourceBundle> connectorConfigBundles;
private Map<String, ResourceBundle> connectorConfigBundles;
/**
* Cached driver.
@ -122,25 +122,22 @@ public void setSqoopRequests(SqoopResourceRequests requests) {
* Clear internal cache.
*/
public void clearCache() {
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
connectorConfigBundles = new HashMap<String, ResourceBundle>();
driverConfigBundle = null;
connectors = new HashMap<Long, MConnector>();
connectors = new HashMap<String, MConnector>();
mDriver = null;
isAllConnectors = false;
}
/**
* Get connector with given id.
* TODO: This method should be removed when MJob link with Connector by name.
*
* @param cid Connector id.
* @return
*/
public MConnector getConnector(long cid) {
if(connectors.containsKey(cid)) {
return connectors.get(cid).clone(false);
}
retrieveConnector(cid);
return connectors.get(cid).clone(false);
return retrieveConnector(Long.toString(cid));
}
/**
@ -183,13 +180,17 @@ private MConnector getConnectorFromCache(String connectorName) {
/**
* Retrieve connector structure from server and cache it.
* TODO: The method support both connector name/id, this should support name only when MJob link with Connector by name.
*
* @param cid Connector id
* @param connectorIdentify Connector name/id
*/
private void retrieveConnector(long cid) {
ConnectorBean request = resourceRequests.readConnector(cid);
connectors.put(cid, request.getConnectors().get(0));
connectorConfigBundles.put(cid, request.getResourceBundles().get(cid));
private MConnector retrieveConnector(String connectorIdentify) {
ConnectorBean request = resourceRequests.readConnector(connectorIdentify);
MConnector mConnector = request.getConnectors().get(0);
String connectorName = mConnector.getUniqueName();
connectors.put(connectorName, mConnector);
connectorConfigBundles.put(connectorName, request.getResourceBundles().get(connectorName));
return mConnector;
}
/**
@ -205,7 +206,7 @@ public Collection<MConnector> getConnectors() {
ConnectorBean bean = resourceRequests.readConnector(null);
isAllConnectors = true;
for(MConnector connector : bean.getConnectors()) {
connectors.put(connector.getPersistenceId(), connector);
connectors.put(connector.getUniqueName(), connector);
}
connectorConfigBundles = bean.getResourceBundles();
@ -218,12 +219,12 @@ public Collection<MConnector> getConnectors() {
* @param connectorId Connector id.
* @return
*/
public ResourceBundle getConnectorConfigBundle(long connectorId) {
if(connectorConfigBundles.containsKey(connectorId)) {
return connectorConfigBundles.get(connectorId);
public ResourceBundle getConnectorConfigBundle(String connectorName) {
if(connectorConfigBundles.containsKey(connectorName)) {
return connectorConfigBundles.get(connectorName);
}
retrieveConnector(connectorId);
return connectorConfigBundles.get(connectorId);
retrieveConnector(connectorName);
return connectorConfigBundles.get(connectorName);
}
/**
@ -275,16 +276,6 @@ public ResourceBundle getDriverConfigBundle() {
return driverConfigBundle;
}
/**
* Create new link object for given connector id
*
* @param connectorId Connector id
* @return
*/
public MLink createLink(long connectorId) {
return new MLink(connectorId, getConnector(connectorId).getLinkConfig());
}
/**
* Create new link object for given connector name
*
@ -296,7 +287,7 @@ public MLink createLink(String connectorName) {
if (connector == null) {
throw new SqoopException(ClientError.CLIENT_0003, connectorName);
}
return createLink(connector.getPersistenceId());
return new MLink(connectorName, getConnector(connectorName).getLinkConfig());
}
@ -398,36 +389,16 @@ public void deleteLink(long linkId) {
public MJob createJob(String fromLinkName, String toLinkName) {
MLink fromLink = getLink(fromLinkName);
MLink toLink = getLink(toLinkName);
MConnector connectorForFromLink = getConnector(fromLink.getConnectorName());
MConnector connectorForToLink = getConnector(toLink.getConnectorName());
return new MJob(
fromLink.getConnectorId(),
toLink.getConnectorId(),
connectorForFromLink.getPersistenceId(),
connectorForToLink.getPersistenceId(),
fromLink.getPersistenceId(),
toLink.getPersistenceId(),
getConnector(fromLink.getConnectorId()).getFromConfig(),
getConnector(toLink.getConnectorId()).getToConfig(),
getDriverConfig()
);
}
/**
* Create new job the for given links.
*
* @param fromLinkId From link id
* @param toLinkId To link id
* @return
*/
public MJob createJob(long fromLinkId, long toLinkId) {
MLink fromLink = getLink(fromLinkId);
MLink toLink = getLink(toLinkId);
return new MJob(
fromLink.getConnectorId(),
toLink.getConnectorId(),
fromLink.getPersistenceId(),
toLink.getPersistenceId(),
getConnector(fromLink.getConnectorId()).getFromConfig(),
getConnector(toLink.getConnectorId()).getToConfig(),
connectorForFromLink.getFromConfig().clone(false),
connectorForToLink.getToConfig().clone(false),
getDriverConfig()
);
}

8
client/src/main/java/org/apache/sqoop/client/request/ConnectorResourceRequest.java
View File

@ -39,17 +39,17 @@ public ConnectorResourceRequest(DelegationTokenAuthenticatedURL.Token token){
super(token);
}
public ConnectorBean read(String serverUrl, Long cid) {
public ConnectorBean read(String serverUrl, String connectorName) {
String response;
if (cid == null) {
if (connectorName == null) {
response = super.get(serverUrl + RESOURCE + "all");
} else {
response = super.get(serverUrl + RESOURCE + cid);
response = super.get(serverUrl + RESOURCE + connectorName);
}
JSONObject jsonObject = JSONUtils.parse(response);
// defaults to all
ConnectorBean bean = new ConnectorsBean();
if (cid != null) {
if (connectorName != null) {
bean = new ConnectorBean();
}
bean.restore(jsonObject);

4
client/src/main/java/org/apache/sqoop/client/request/SqoopResourceRequests.java
View File

@ -114,8 +114,8 @@ public DriverBean readDriver() {
return getDriverResourceRequest().read(serverUrl);
}
public ConnectorBean readConnector(Long cid) {
return getConnectorResourceRequest().read(serverUrl, cid);
public ConnectorBean readConnector(String connectorName) {
return getConnectorResourceRequest().read(serverUrl, connectorName);
}
public ValidationResultBean saveLink(MLink link) {

51
client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java
View File

@ -65,17 +65,6 @@ public void setUp() {
* Retrieve connector information, request to bundle for same connector should
* not require additional HTTP request.
*/
@Test
public void testGetConnector() {
when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
MConnector connector = client.getConnector(1);
assertEquals(1, connector.getPersistenceId());
client.getConnectorConfigBundle(1L);
verify(resourceRequests, times(1)).readConnector(1L);
}
@Test
public void testGetConnectorByString() {
when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connector(1)));
@ -83,9 +72,9 @@ public void testGetConnectorByString() {
assertEquals(1, connector.getPersistenceId());
assertEquals("A1", connector.getUniqueName());
client.getConnectorConfigBundle(1L);
client.getConnectorConfigBundle("A1");
verify(resourceRequests, times(0)).readConnector(1L);
verify(resourceRequests, times(0)).readConnector("A1");
verify(resourceRequests, times(1)).readConnector(null);
}
@ -95,13 +84,13 @@ public void testGetConnectorByString() {
*/
@Test
public void testGetConnectorBundle() {
when(resourceRequests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
client.getConnectorConfigBundle(1L);
when(resourceRequests.readConnector("A1")).thenReturn(connectorBean(connector(1)));
client.getConnectorConfigBundle("A1");
MConnector connector = client.getConnector(1);
MConnector connector = client.getConnector("A1");
assertEquals(1, connector.getPersistenceId());
verify(resourceRequests, times(1)).readConnector(1L);
verify(resourceRequests, times(1)).readConnector("A1");
}
/**
@ -144,12 +133,12 @@ public void testGetConnectors() {
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
client.getConnectorConfigBundle(1);
connector = client.getConnector(1);
client.getConnectorConfigBundle("A1");
connector = client.getConnector("A1");
assertEquals(1, connector.getPersistenceId());
connector = client.getConnector(2);
client.getConnectorConfigBundle(2);
connector = client.getConnector("A2");
client.getConnectorConfigBundle("A2");
assertEquals(2, connector.getPersistenceId());
connectors = client.getConnectors();
@ -179,21 +168,21 @@ public void testGetConnectors() {
public void testGetConnectorOneByOne() {
ConnectorBean bean = connectorBean(connector(1), connector(2));
when(resourceRequests.readConnector(null)).thenReturn(bean);
when(resourceRequests.readConnector(1L)).thenReturn(bean);
when(resourceRequests.readConnector(2L)).thenReturn(bean);
when(resourceRequests.readConnector("A1")).thenReturn(bean);
when(resourceRequests.readConnector("A2")).thenReturn(bean);
client.getConnectorConfigBundle(1);
client.getConnector(1);
client.getConnectorConfigBundle("A1");
client.getConnector("A1");
client.getConnector(2);
client.getConnectorConfigBundle(2);
client.getConnector("A2");
client.getConnectorConfigBundle("A2");
Collection<MConnector> connectors = client.getConnectors();
assertEquals(2, connectors.size());
verify(resourceRequests, times(1)).readConnector(null);
verify(resourceRequests, times(1)).readConnector(1L);
verify(resourceRequests, times(1)).readConnector(2L);
verify(resourceRequests, times(1)).readConnector("A1");
verify(resourceRequests, times(0)).readConnector("A2");
verifyNoMoreInteractions(resourceRequests);
}
@ -208,11 +197,11 @@ public void testCreateLink() {
private ConnectorBean connectorBean(MConnector...connectors) {
List<MConnector> connectorList = new ArrayList<MConnector>();
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
for(MConnector connector : connectors) {
connectorList.add(connector);
bundles.put(connector.getPersistenceId(), null);
bundles.put(connector.getUniqueName(), null);
}
return new ConnectorBean(connectorList, bundles);
}

14
common/src/main/java/org/apache/sqoop/json/ConnectorBean.java
View File

@ -57,10 +57,10 @@ public class ConnectorBean extends ConfigurableBean {
private static final String CONNECTOR = "connector";
private List<MConnector> connectors;
private Map<Long, ResourceBundle> connectorConfigBundles;
private Map<String, ResourceBundle> connectorConfigBundles;
// for "extract"
public ConnectorBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) {
public ConnectorBean(List<MConnector> connectors, Map<String, ResourceBundle> bundles) {
this.connectors = connectors;
this.connectorConfigBundles = bundles;
}
@ -73,7 +73,7 @@ public List<MConnector> getConnectors() {
return connectors;
}
public Map<Long, ResourceBundle> getResourceBundles() {
public Map<String, ResourceBundle> getResourceBundles() {
return connectorConfigBundles;
}
@ -121,7 +121,7 @@ private JSONObject extractConnector(boolean skipSensitive, MConnector connector)
connectorJsonObject.put(ALL_CONFIGS, new JSONObject());
if (connectorConfigBundles != null && !connectorConfigBundles.isEmpty()) {
connectorJsonObject.put(ALL_CONFIGS,
extractConfigParamBundle(connectorConfigBundles.get(connector.getPersistenceId())));
extractConfigParamBundle(connectorConfigBundles.get(connector.getUniqueName())));
}
return connectorJsonObject;
}
@ -129,14 +129,14 @@ private JSONObject extractConnector(boolean skipSensitive, MConnector connector)
@Override
public void restore(JSONObject jsonObject) {
connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
connectorConfigBundles = new HashMap<String, ResourceBundle>();
JSONObject obj = JSONUtils.getJSONObject(jsonObject, CONNECTOR);
connectors.add(restoreConnector(obj));
}
protected void restoreConnectors(JSONArray array) {
connectors = new ArrayList<MConnector>();
connectorConfigBundles = new HashMap<Long, ResourceBundle>();
connectorConfigBundles = new HashMap<String, ResourceBundle>();
for (Object obj : array) {
connectors.add(restoreConnector(obj));
}
@ -178,7 +178,7 @@ private MConnector restoreConnector(Object obj) {
connector.setPersistenceId(connectorId);
if (object.containsKey(ALL_CONFIGS)) {
JSONObject jsonConfigBundle = JSONUtils.getJSONObject(object, ALL_CONFIGS);
connectorConfigBundles.put(connectorId, restoreConfigParamBundle(jsonConfigBundle));
connectorConfigBundles.put(uniqueName, restoreConfigParamBundle(jsonConfigBundle));
}
return connector;
}

2
common/src/main/java/org/apache/sqoop/json/ConnectorsBean.java
View File

@ -39,7 +39,7 @@ public class ConnectorsBean extends ConnectorBean {
private static final String CONNECTORS = "connectors";
// for "extract"
public ConnectorsBean(List<MConnector> connectors, Map<Long, ResourceBundle> bundles) {
public ConnectorsBean(List<MConnector> connectors, Map<String, ResourceBundle> bundles) {
super(connectors, bundles);
}

24
common/src/main/java/org/apache/sqoop/json/LinkBean.java
View File

@ -47,7 +47,7 @@
@InterfaceStability.Unstable
public class LinkBean implements JsonBean {
static final String CONNECTOR_ID = "connector-id";
static final String CONNECTOR_NAME = "connector-name";
static final String LINK_CONFIG_VALUES = "link-config-values";
static final String LINK = "link";
@ -57,7 +57,7 @@ public class LinkBean implements JsonBean {
private List<MLink> links;
// Optional
private Map<Long, ResourceBundle> linkConfigBundles;
private Map<String, ResourceBundle> linkConfigBundles;
// For "extract"
public LinkBean(MLink link) {
@ -73,23 +73,23 @@ public LinkBean(List<MLink> links) {
// For "restore"
public LinkBean() {
linkConfigBundles = new HashMap<Long, ResourceBundle>();
linkConfigBundles = new HashMap<String, ResourceBundle>();
}
public void addConnectorConfigBundle(Long id, ResourceBundle connectorConfigBundle) {
linkConfigBundles.put(id, connectorConfigBundle);
public void addConnectorConfigBundle(String connectorName, ResourceBundle connectorConfigBundle) {
linkConfigBundles.put(connectorName, connectorConfigBundle);
}
public boolean hasConnectorConfigBundle(Long id) {
return linkConfigBundles.containsKey(id);
public boolean hasConnectorConfigBundle(String connectorName) {
return linkConfigBundles.containsKey(connectorName);
}
public List<MLink> getLinks() {
return links;
}
public ResourceBundle getConnectorConfigBundle(Long id) {
return linkConfigBundles.get(id);
public ResourceBundle getConnectorConfigBundle(String connectorName) {
return linkConfigBundles.get(connectorName);
}
@SuppressWarnings("unchecked")
@ -119,7 +119,7 @@ private JSONObject extractLink(boolean skipSensitive, MLink link) {
linkJsonObject.put(CREATION_DATE, link.getCreationDate().getTime());
linkJsonObject.put(UPDATE_USER, link.getLastUpdateUser());
linkJsonObject.put(UPDATE_DATE, link.getLastUpdateDate().getTime());
linkJsonObject.put(CONNECTOR_ID, link.getConnectorId());
linkJsonObject.put(CONNECTOR_NAME, link.getConnectorName());
linkJsonObject.put(LINK_CONFIG_VALUES,
extractConfigList(link.getConnectorLinkConfig(), skipSensitive));
return linkJsonObject;
@ -141,11 +141,11 @@ protected void restoreLinks(JSONArray array) {
private MLink restoreLink(Object obj) {
JSONObject object = (JSONObject) obj;
long connectorId = JSONUtils.getLong(object, CONNECTOR_ID);
String connectorName = (String) object.get(CONNECTOR_NAME);
JSONObject connectorLinkConfig = JSONUtils.getJSONObject(object, LINK_CONFIG_VALUES);
List<MConfig> linkConfigs = restoreConfigs(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIGS));
List<MValidator> linkValidators = restoreValidator(JSONUtils.getJSONArray(connectorLinkConfig, ConfigInputConstants.CONFIG_VALIDATORS));
MLink link = new MLink(connectorId, new MLinkConfig(linkConfigs, linkValidators));
MLink link = new MLink(connectorName, new MLinkConfig(linkConfigs, linkValidators));
link.setPersistenceId(JSONUtils.getLong(object, ID));
link.setName(JSONUtils.getString(object, NAME));
link.setEnabled(JSONUtils.getBoolean(object, ENABLED));

18
common/src/main/java/org/apache/sqoop/model/MLink.java
View File

@ -26,7 +26,7 @@
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class MLink extends MAccountableEntity implements MClonable {
private long connectorId;
private String connectorName;
// NOTE: we hold this in the model for easy access to the link config object, it might as well be retrieved on the fly using the connectorId
private final MLinkConfig connectorLinkConfig;
@ -36,8 +36,8 @@ public class MLink extends MAccountableEntity implements MClonable {
* @param connectorId Connector id
* @param linkConfig Connector forms
*/
public MLink(long connectorId, MLinkConfig linkConfig) {
this.connectorId = connectorId;
public MLink(String connectorName, MLinkConfig linkConfig) {
this.connectorName = connectorName;
this.connectorLinkConfig = linkConfig;
}
@ -61,7 +61,7 @@ public MLink(MLink other) {
*/
public MLink(MLink other, MLinkConfig linkConfig) {
super(other);
this.connectorId = other.connectorId;
this.connectorName = other.connectorName;
this.connectorLinkConfig = linkConfig;
this.setPersistenceId(other.getPersistenceId());
}
@ -74,8 +74,8 @@ public String toString() {
return sb.toString();
}
public long getConnectorId() {
return connectorId;
public String getConnectorName() {
return connectorName;
}
public MLinkConfig getConnectorLinkConfig() {
@ -90,7 +90,7 @@ public MLink clone(boolean cloneWithValue) {
if(cloneWithValue) {
return new MLink(this);
} else {
return new MLink(connectorId, connectorLinkConfig.clone(false));
return new MLink(connectorName, connectorLinkConfig.clone(false));
}
}
@ -105,14 +105,14 @@ public boolean equals(Object object) {
}
MLink mLink = (MLink)object;
return (mLink.connectorId == this.connectorId)
return (mLink.connectorName.equals(this.connectorName))
&& (mLink.getPersistenceId() == this.getPersistenceId())
&& (mLink.connectorLinkConfig.equals(this.connectorLinkConfig));
}
@Override
public int hashCode() {
int result = (int) (connectorId ^ (connectorId >>> 32));
int result = connectorName != null ? connectorName.hashCode() : 0;
result = 31 * result + (connectorLinkConfig != null ? connectorLinkConfig.hashCode() : 0);
return result;
}

6
common/src/test/java/org/apache/sqoop/json/TestConnectorBean.java
View File

@ -45,8 +45,8 @@ public void testConnectorSerialization() {
connectors.add(BeanTestUtil.getConnector(1L, "jdbc"));
// Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, ConfigTestUtil.getResourceBundle());
Map<String, ResourceBundle> configBundles = new HashMap<String, ResourceBundle>();
configBundles.put("jdbc", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorBean connectorBean = new ConnectorBean(connectors, configBundles);
@ -62,7 +62,7 @@ public void testConnectorSerialization() {
assertEquals(connectors.size(), 1);
assertEquals(connectors.size(), parsedConnectorBean.getConnectors().size());
assertEquals(connectors.get(0), parsedConnectorBean.getConnectors().get(0));
ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get(1L);
ResourceBundle retrievedBundle = parsedConnectorBean.getResourceBundles().get("jdbc");
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));

20
common/src/test/java/org/apache/sqoop/json/TestConnectorsBean.java
View File

@ -42,9 +42,9 @@ public void testConnectorsSerialization() {
connectors.add(BeanTestUtil.getConnector(2L, "mysql"));
// Create testing bundles
Map<Long, ResourceBundle> configBundles = new HashMap<Long, ResourceBundle>();
configBundles.put(1L, ConfigTestUtil.getResourceBundle());
configBundles.put(2L, ConfigTestUtil.getResourceBundle());
Map<String, ResourceBundle> configBundles = new HashMap<String, ResourceBundle>();
configBundles.put("jdbc", ConfigTestUtil.getResourceBundle());
configBundles.put("mysql", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean connectorsBean = new ConnectorsBean(connectors, configBundles);
@ -62,7 +62,7 @@ public void testConnectorsSerialization() {
assertEquals(connectors.get(0), parsedConnectorsBean.getConnectors().get(0));
assertEquals(connectors.get(1), parsedConnectorsBean.getConnectors().get(1));
ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get(1L);
ResourceBundle retrievedBundle = parsedConnectorsBean.getResourceBundles().get("jdbc");
assertNotNull(retrievedBundle);
assertEquals("a", retrievedBundle.getString("a"));
assertEquals("b", retrievedBundle.getString("b"));
@ -76,9 +76,9 @@ public void testSingleDirection() {
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, true));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle());
Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
bundles.put("jdbc", ConfigTestUtil.getResourceBundle());
bundles.put("mysql", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles);
@ -105,9 +105,9 @@ public void testNoDirection() {
connectors.add(BeanTestUtil.getConnector(2L, "mysql", false, false));
// Create testing bundles
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
bundles.put(1L, ConfigTestUtil.getResourceBundle());
bundles.put(2L, ConfigTestUtil.getResourceBundle());
Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
bundles.put("jdbc", ConfigTestUtil.getResourceBundle());
bundles.put("mysql", ConfigTestUtil.getResourceBundle());
// Serialize it to JSON object
ConnectorsBean bean = new ConnectorsBean(connectors, bundles);

2
common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java
View File

@ -52,7 +52,7 @@ public static MJob createJob(String connectorName, String jobName, Long jobId, D
}
public static MLink getLink(String connectorName) {
return new MLink(1, getConnector(1L, connectorName).getLinkConfig());
return new MLink(connectorName, getConnector(1L, connectorName).getLinkConfig());
}
public static MConnector getConnector(Long connectorId, String connectorName) {

2
common/src/test/java/org/apache/sqoop/model/TestMAccountableEntity.java
View File

@ -51,7 +51,7 @@ public void testInitialization() {
MValidator validator = new MValidator("test", "");
validators.add(validator);
MAccountableEntity link = new MLink(123l, new MLinkConfig(configs, validators));
MAccountableEntity link = new MLink("connector_test", new MLinkConfig(configs, validators));
// Initially creation date and last update date is same
assertEquals(link.getCreationDate(), link.getLastUpdateDate());
Date testCreationDate = new Date();

6
common/src/test/java/org/apache/sqoop/model/TestMLink.java
View File

@ -35,14 +35,14 @@ public class TestMLink {
public void testInitialization() {
// Test default constructor
MLink link = link();
assertEquals(123l, link.getConnectorId());
assertEquals("connector_test", link.getConnectorName());
assertEquals("Vampire", link.getName());
assertEquals("Buffy", link.getCreationUser());
assertEquals(linkConfig(), link.getConnectorLinkConfig());
// Test copy constructor
MLink copy = new MLink(link);
assertEquals(123l, copy.getConnectorId());
assertEquals("connector_test", copy.getConnectorName());
assertEquals("Vampire", copy.getName());
assertEquals("Buffy", copy.getCreationUser());
assertEquals(link.getCreationDate(), copy.getCreationDate());
@ -75,7 +75,7 @@ public void testClone() {
}
private MLink link() {
MLink link = new MLink(123l, linkConfig());
MLink link = new MLink("connector_test", linkConfig());
link.setName("Vampire");
link.setCreationUser("Buffy");
return link;

8
core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
View File

@ -117,12 +117,12 @@ public Set<Long> getConnectorIds() {
return idToNameMap.keySet();
}
public Map<Long, ResourceBundle> getResourceBundles(Locale locale) {
Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
public Map<String, ResourceBundle> getResourceBundles(Locale locale) {
Map<String, ResourceBundle> bundles = new HashMap<String, ResourceBundle>();
for (ConnectorHandler handler : handlerMap.values()) {
long id = handler.getConnectorConfigurable().getPersistenceId();
String connectorName = handler.getConnectorConfigurable().getUniqueName();
ResourceBundle bundle = handler.getSqoopConnector().getBundle(locale);
bundles.put(id, bundle);
bundles.put(connectorName, bundle);
}
return bundles;
}

18
core/src/main/java/org/apache/sqoop/driver/JobManager.java
View File

@ -339,22 +339,22 @@ public MSubmission start(String jobName, HttpEventContext ctx) {
private JobRequest createJobRequest(MSubmission submission, MJob job) {
// get from/to connections for the job
MLink fromConnection = getLink(job.getFromLinkId());
MLink toConnection = getLink(job.getToLinkId());
MLink fromLink = getLink(job.getFromLinkId());
MLink toLink = getLink(job.getToLinkId());
// get from/to connectors for the connection
SqoopConnector fromConnector = getSqoopConnector(fromConnection.getConnectorId());
SqoopConnector fromConnector = getSqoopConnector(fromLink.getConnectorName());
validateSupportedDirection(fromConnector, Direction.FROM);
SqoopConnector toConnector = getSqoopConnector(toConnection.getConnectorId());
SqoopConnector toConnector = getSqoopConnector(toLink.getConnectorName());
validateSupportedDirection(toConnector, Direction.TO);
// link config for the FROM part of the job
Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass());
ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig);
ConfigUtils.fromConfigs(fromLink.getConnectorLinkConfig().getConfigs(), fromLinkConfig);
// link config for the TO part of the job
Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass());
ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig);
ConfigUtils.fromConfigs(toLink.getConnectorLinkConfig().getConfigs(), toLinkConfig);
// from config for the job
Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
@ -470,7 +470,11 @@ MSubmission createJobSubmission(HttpEventContext ctx, long jobId) {
return summary;
}
SqoopConnector getSqoopConnector(long connnectorId) {
SqoopConnector getSqoopConnector(String connnectorName) {
return ConnectorManager.getInstance().getSqoopConnector(connnectorName);
}
SqoopConnector getSqoopConnector(Long connnectorId) {
return ConnectorManager.getInstance().getSqoopConnector(connnectorId);
}

4
core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
View File

@ -109,7 +109,7 @@ public void testUnsupportedDirectionForConnector() {
@Test
public void testGetLink() {
MLink testLink = new MLink(123l, null);
MLink testLink = new MLink("connector_test", null);
testLink.setEnabled(true);
MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
@ -121,7 +121,7 @@ public void testGetLink() {
@Test
public void testDisabledLink() {
MLink testConnection = new MLink(123l, null);
MLink testConnection = new MLink("connector_test", null);
testConnection.setPersistenceId(1234);
testConnection.setEnabled(false);
SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection: "

18
core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
View File

@ -228,7 +228,7 @@ public void testConnectorConfigUpgradeWithValidLinksAndJobs() {
// prepare the links and jobs
// the connector Id for both are the same
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 2));
// mock necessary methods for upgradeConnector() procedure
@ -390,7 +390,7 @@ public void testConnectorConfigUpgradeHandlerWithFindJobsForConnectorError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@ -424,7 +424,7 @@ public void testConnectorConfigUpgradeHandlerWithDeleteJobInputsError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade
@ -462,7 +462,7 @@ public void testConnectorConfigUpgradeHandlerWithDeleteLinkInputsError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -500,7 +500,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateConnectorError() {
when(sqconnector.getConfigurableUpgrader(oldConnector.getVersion())).thenReturn(connectorUpgraderMock);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -542,7 +542,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateLinkError() {
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -588,7 +588,7 @@ public void testConnectorConfigUpgradeHandlerWithUpdateJobError() {
when(sqconnector.getJobConfigurationClass(any(Direction.class))).thenReturn(ValidConfiguration.class);
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", 1), link(2, "LB", 1));;
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));;
List<MJob> jobList = jobs(job(1, "JA", 1, 1, 1, 1), job(2, "JB", 1, 1, 2, 1));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
@ -775,8 +775,8 @@ private MDriver anotherDriver() {
return driver;
}
private MLink link(long linkId, String linkName, long connectorId) {
MLink link = new MLink(connectorId, new MLinkConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>()));
private MLink link(long linkId, String linkName, String connectorName) {
MLink link = new MLink(connectorName, new MLinkConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>()));
link.setPersistenceId(linkId);
link.setName(linkName);
return link;

12
repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
View File

@ -371,11 +371,13 @@ public void registerDriver(MDriver mDriver, Connection conn) {
@Override
public void createLink(MLink link, Connection conn) {
int result;
MConnector mConnector;
try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertLink(),
Statement.RETURN_GENERATED_KEYS)) {
stmt.setString(1, link.getName());
stmt.setLong(2, link.getConnectorId());
mConnector = findConnector(link.getConnectorName(), conn);
stmt.setLong(2, mConnector.getPersistenceId());
stmt.setBoolean(3, link.getEnabled());
stmt.setString(4, link.getCreationUser());
stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime()));
@ -1515,7 +1517,6 @@ private List<MLink> loadLinksForUpgrade(PreparedStatement stmt,
try (ResultSet rsConnection = stmt.executeQuery();
PreparedStatement connectorConfigFetchStatement = conn.prepareStatement(crudQueries.getStmtSelectConfigForConfigurable());
PreparedStatement connectorConfigInputStatement = conn.prepareStatement(crudQueries.getStmtFetchLinkInput());) {
while(rsConnection.next()) {
long id = rsConnection.getLong(1);
String name = rsConnection.getString(2);
@ -1525,6 +1526,7 @@ private List<MLink> loadLinksForUpgrade(PreparedStatement stmt,
Date creationDate = rsConnection.getTimestamp(6);
String updateUser = rsConnection.getString(7);
Date lastUpdateDate = rsConnection.getTimestamp(8);
String connectorName = rsConnection.getString(9);
connectorConfigFetchStatement.setLong(1, connectorId);
connectorConfigInputStatement.setLong(1, id);
@ -1535,7 +1537,8 @@ private List<MLink> loadLinksForUpgrade(PreparedStatement stmt,
loadConnectorConfigs(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement,
connectorConfigInputStatement, 2, conn);
MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig, Collections.EMPTY_LIST));
MLink link = new MLink(connectorName, new MLinkConfig(connectorLinkConfig, Collections.EMPTY_LIST));
link.setPersistenceId(id);
link.setName(name);
@ -1576,7 +1579,8 @@ private List<MLink> loadLinks(PreparedStatement stmt,
configStmt.setLong(1, connectorId);
inputStmt.setLong(1, id);
loadInputsForConfigs(connectorLinkConfig, configStmt, inputStmt);
MLink link = new MLink(connectorId, connectorLinkConfig);
MLink link = new MLink(connectorName, connectorLinkConfig);
link.setPersistenceId(id);
link.setName(name);

2
repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestInputTypes.java
View File

@ -97,7 +97,7 @@ public void testEntityDataSerialization() throws Exception {
map.put("A", "B");
// Connection object with all various values
MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig());
MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig());
MLinkConfig linkConfig = link.getConnectorLinkConfig();
assertEquals(linkConfig.getStringInput("LINK1.I1").getEditable(), InputEditable.USER_ONLY);
assertEquals(linkConfig.getStringInput("LINK1.I1").getOverrides(), "LINK1.I2");

2
repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java
View File

@ -304,6 +304,6 @@ public void testDeleteLink() throws Exception {
}
public MLink getLink() {
return new MLink(1, handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig());
return new MLink("A", handler.findConnector("A", getDerbyDatabaseConnection()).getLinkConfig());
}
}

2
repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
View File

@ -94,7 +94,7 @@ protected MDriver getDriver() {
}
protected MLink getLink(String name, MConnector connector) {
MLink link = new MLink(connector.getPersistenceId(),
MLink link = new MLink(connector.getUniqueName(),
connector.getLinkConfig());
link.setName(name);
fillLink(link);

2
repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
View File

@ -83,7 +83,7 @@ protected MDriver getDriver() {
}
protected MLink getLink(String name, MConnector connector) {
MLink link = new MLink(connector.getPersistenceId(), connector.getLinkConfig());
MLink link = new MLink(connector.getUniqueName(), connector.getLinkConfig());
link.setName(name);
fillLink(link);
return link;

4
server/src/main/java/org/apache/sqoop/handler/ConnectorRequestHandler.java
View File

@ -48,7 +48,7 @@ public ConnectorRequestHandler() {
@Override
public JsonBean handleEvent(RequestContext ctx) {
List<MConnector> connectors;
Map<Long, ResourceBundle> configParamBundles;
Map<String, ResourceBundle> configParamBundles;
Locale locale = ctx.getAcceptLanguageHeader();
String cIdentifier = ctx.getLastURLElement();
@ -73,7 +73,7 @@ public JsonBean handleEvent(RequestContext ctx) {
configParamBundles = new HashMap<>();
MConnector connector = ConnectorManager.getInstance().getConnectorConfigurable(mConnector.getUniqueName());
configParamBundles.put(connector.getPersistenceId(),
configParamBundles.put(connector.getUniqueName(),
ConnectorManager.getInstance().getResourceBundle(mConnector.getUniqueName(), locale));
AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(),

14
server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
View File

@ -130,7 +130,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
}
MLink postedLink = links.get(0);
MConnector mConnector = HandlerUtils.getConnectorFromConnectorId(postedLink.getConnectorId());
MConnector mConnector = HandlerUtils.getConnectorFromConnectorName(postedLink.getConnectorName());
// Authorization check
if (create) {
@ -142,7 +142,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
}
MLinkConfig linkConfig = ConnectorManager.getInstance()
.getConnectorConfigurable(postedLink.getConnectorId()).getLinkConfig();
.getConnectorConfigurable(postedLink.getConnectorName()).getLinkConfig();
if (!linkConfig.equals(postedLink.getConnectorLinkConfig())) {
throw new SqoopException(ServerError.SERVER_0003, "Detected incorrect link config structure");
}
@ -156,7 +156,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
}
// Associated connector for this link
SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector(
postedLink.getConnectorId());
postedLink.getConnectorName());
// Validate user supplied config data
ConfigValidationResult connectorLinkConfigValidation = ConfigUtils.validateConfigs(postedLink
@ -240,10 +240,10 @@ private LinksBean createLinksBean(List<MLink> links, Locale locale) {
private void addConnectorConfigBundle(Locale locale, LinkBean bean) {
// Add associated resources into the bean
for (MLink link : bean.getLinks()) {
long connectorId = link.getConnectorId();
if (!bean.hasConnectorConfigBundle(connectorId)) {
bean.addConnectorConfigBundle(connectorId, ConnectorManager.getInstance()
.getResourceBundle(connectorId, locale));
String connectorName = link.getConnectorName();
if (!bean.hasConnectorConfigBundle(connectorName)) {
bean.addConnectorConfigBundle(connectorName, ConnectorManager.getInstance()
.getResourceBundle(connectorName, locale));
}
}
}

8
shell/src/main/java/org/apache/sqoop/shell/CloneJobFunction.java
View File

@ -20,6 +20,7 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.shell.core.Constants;
@ -66,10 +67,13 @@ private Status cloneJob(String jobArg, List<String> args, boolean isInteractive)
MJob job = client.getJob(jobArg);
job.setPersistenceId(MPersistableEntity.PERSISTANCE_ID_DEFAULT);
// TODO: the job should be related with connector by connectorName
MConnector fromConnector = getClient().getConnector(job.getFromConnectorId());
MConnector toConnector = getClient().getConnector(job.getToConnectorId());
ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle(
job.getFromConnectorId());
fromConnector.getUniqueName());
ResourceBundle toConnectorBundle = client.getConnectorConfigBundle(
job.getToConnectorId());
toConnector.getUniqueName());
ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
Status status = Status.OK;

2
shell/src/main/java/org/apache/sqoop/shell/CloneLinkFunction.java
View File

@ -70,7 +70,7 @@ private Status cloneLink(String linkArg, List<String> args, boolean isInteractiv
Status status = Status.OK;
ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId());
ResourceBundle linkConfigBundle = client.getConnectorConfigBundle(link.getConnectorName());
if (isInteractive) {
printlnResource(Constants.RES_PROMPT_UPDATE_LINK_CONFIG);

4
shell/src/main/java/org/apache/sqoop/shell/CreateJobFunction.java
View File

@ -90,9 +90,9 @@ private Status createJob(String fromLinkArg, String toLinkArg, List<String> args
}
ResourceBundle fromConfigBundle = getClient().getConnectorConfigBundle(
job.getFromConnectorId());
fromConnector.getUniqueName());
ResourceBundle toConfigBundle = getClient().getConnectorConfigBundle(
job.getToConnectorId());
toConnector.getUniqueName());
ResourceBundle driverConfigBundle = getClient().getDriverConfigBundle();
Status status = Status.OK;

7
shell/src/main/java/org/apache/sqoop/shell/CreateLinkFunction.java
View File

@ -72,24 +72,23 @@ private Status createLink(CommandLine line, List<String> args, boolean isInterac
cid = getLong(line, Constants.OPT_CID);
getClient().getConnector(cid);
//Would have thrown an exception before this if input was neither a valid name nor an id
//Would have thrown an exception before this if input was an invalid connector name
//This will do an extra getConnector() call again inside createLink()
//but should not matter as connectors are cached
link = getClient().createLink(cid);
link = getClient().createLink(connectorName);
printlnResource(Constants.RES_CREATE_CREATING_LINK, cid);
}
else {
//Command line had connector name
//This will do an extra getConnector() call again inside createLink() but
//should not matter as connectors are cached
cid = connector.getPersistenceId();
link = getClient().createLink(connectorName);
printlnResource(Constants.RES_CREATE_CREATING_LINK, connectorName);
}
ConsoleReader reader = getConsoleReader();
ResourceBundle connectorConfigBundle = getClient().getConnectorConfigBundle(cid);
ResourceBundle connectorConfigBundle = getClient().getConnectorConfigBundle(connectorName);
Status status = Status.OK;
if (isInteractive) {

2
shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
View File

@ -124,6 +124,6 @@ private void displayConnector(MConnector connector) {
connector.getVersion(),
connector.getSupportedDirections().toString()
);
displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getPersistenceId()));
displayConnectorConfigDetails(connector, client.getConnectorConfigBundle(connector.getUniqueName()));
}
}

9
shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
View File

@ -159,15 +159,20 @@ private void displayJob(MJob job) {
job.getLastUpdateUser(),
formatter.format(job.getLastUpdateDate())
);
// TODO: should be removed when MJob link with Connector by name.
MConnector fromConnector = getClient().getConnector(job.getFromConnectorId());
MConnector toConnector = getClient().getConnector(job.getToConnectorId());
displayConfig(job.getDriverConfig().getConfigs(),
client.getDriverConfigBundle());
printlnResource(Constants.RES_SHOW_PROMPT_JOB_FROM_LID_INFO,
job.getFromLinkId());
displayConfig(job.getFromJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getFromConnectorId()));
client.getConnectorConfigBundle(fromConnector.getUniqueName()));
printlnResource(Constants.RES_SHOW_PROMPT_JOB_TO_LID_INFO,
job.getToLinkId());
displayConfig(job.getToJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getToConnectorId()));
client.getConnectorConfigBundle(toConnector.getUniqueName()));
}
}

40
shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
View File

@ -81,17 +81,16 @@ private void showSummary() {
List<String> ids = new LinkedList<String>();
List<String> names = new LinkedList<String>();
List<String> connectorIds = new LinkedList<String>();
List<String> connectorNames = new LinkedList<String>();
List<String> availabilities = new LinkedList<String>();
for (MLink link : links) {
ids.add(String.valueOf(link.getPersistenceId()));
names.add(link.getName());
connectorIds.add(String.valueOf(link.getConnectorId()));
connectorNames.add(link.getConnectorName());
availabilities.add(String.valueOf(link.getEnabled()));
}
List<String> connectorNames = getConnectorNames(connectorIds);
TableDisplayer.display(header, ids, names, connectorIds, connectorNames, availabilities);
}
@ -126,41 +125,10 @@ private void displayLink(MLink link) {
formatter.format(link.getLastUpdateDate())
);
long connectorId = link.getConnectorId();
MConnector connector = client.getConnector(connectorId);
String connectorName = "";
if (connector != null) {
connectorName = connector.getUniqueName();
}
printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, connectorName, connectorId);
printlnResource(Constants.RES_SHOW_PROMPT_LINK_CID_INFO, link.getConnectorName());
// Display link config
displayConfig(link.getConnectorLinkConfig().getConfigs(),
client.getConnectorConfigBundle(connectorId));
}
private List<String> getConnectorNames(List<String> connectorIds) {
Map<String, String> connectorIdToName = new HashMap<String, String>();
for (String connectorId : connectorIds) {
if (!connectorIdToName.containsKey(connectorId)) {
try {
MConnector connector = client.getConnector(Long.parseLong(connectorId));
if (connector != null) {
connectorIdToName.put(connectorId, connector.getUniqueName());
}
} catch (SqoopException ex) {
connectorIdToName.put(connectorId, "Access Denied");
}
}
}
List<String> connectorNames = new ArrayList<String>();
for (String connectorId : connectorIds) {
if (connectorIdToName.get(connectorId) != null) {
connectorNames.add(connectorIdToName.get(connectorId));
} else {
connectorNames.add("");
}
}
return connectorNames;
client.getConnectorConfigBundle(link.getConnectorName()));
}
}

9
shell/src/main/java/org/apache/sqoop/shell/UpdateJobFunction.java
View File

@ -20,6 +20,7 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.utils.ConfigDisplayer;
@ -67,10 +68,14 @@ private Status updateJob(String jobArg, List<String> args, boolean isInteractive
// TODO(SQOOP-1634): using from/to and driver config id, this call can be avoided
MJob job = client.getJob(jobArg);
// TODO: should be removed when MJob link with Connector by name.
MConnector fromConnector = getClient().getConnector(job.getFromConnectorId());
MConnector toConnector = getClient().getConnector(job.getToConnectorId());
ResourceBundle fromConnectorBundle = client.getConnectorConfigBundle(
job.getFromConnectorId());
fromConnector.getUniqueName());
ResourceBundle toConnectorBundle = client.getConnectorConfigBundle(
job.getToConnectorId());
toConnector.getUniqueName());
ResourceBundle driverConfigBundle = client.getDriverConfigBundle();
Status status = Status.OK;

3
shell/src/main/java/org/apache/sqoop/shell/UpdateLinkFunction.java
View File

@ -65,8 +65,7 @@ private Status updateLink(String linkArg, List<String> args, boolean isInteracti
// TODO(SQOOP-1634): using link config id, this call can be avoided
MLink link = client.getLink(linkArg);
ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorId());
ResourceBundle connectorLinkConfigBundle = client.getConnectorConfigBundle(link.getConnectorName());
Status status = Status.OK;

33
shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java
View File

@ -41,24 +41,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.InputEditable;
import org.apache.sqoop.model.MBooleanInput;
import org.apache.sqoop.model.MConfig;
import org.apache.sqoop.model.MDateTimeInput;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MEnumInput;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MInput;
import org.apache.sqoop.model.MIntegerInput;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MListInput;
import org.apache.sqoop.model.MLongInput;
import org.apache.sqoop.model.MMapInput;
import org.apache.sqoop.model.MStringInput;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.model.MValidator;
import org.apache.sqoop.model.*;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.core.ShellError;
import org.apache.sqoop.utils.MapResourceBundle;
@ -104,9 +87,9 @@ public Enumeration<String> getKeys() {
@Test
public void testCloneLink() {
ShellEnvironment.setInteractive(false);
MLink link = new MLink(1L, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
MLink link = new MLink("connector_name_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link);
when(client.getConnectorConfigBundle(1L)).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getConnectorConfigBundle("connector_name_test")).thenReturn(new MapResourceBundle(new HashMap()));
when(client.saveLink(link)).thenReturn(Status.OK);
// clone link -lid link_test
@ -136,10 +119,11 @@ public void testCloneLink() {
public void testCloneLinkInteractive() {
ShellEnvironment.setInteractive(true);
initEnv();
MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
MLink link = new MLink("connector_name_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link);
when(client.getConnectorConfigBundle(1L)).thenReturn(resourceBundle);
when(client.getConnectorConfigBundle("connector_name_test")).thenReturn(resourceBundle);
when(client.saveLink(link)).thenReturn(Status.OK);
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
// clone link -lid link_test
initData("linkname\r" + // link name
@ -176,9 +160,10 @@ public void testCloneJob() {
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap()));
when(client.saveJob(job)).thenReturn(Status.OK);
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
// clone job -jid job_test
Status status = (Status) cloneCmd.execute(Arrays.asList(Constants.FN_JOB, "-jid", "job_test"));
@ -211,7 +196,7 @@ public void testCloneJobInteractive() {
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle);
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
when(client.getDriverConfigBundle()).thenReturn(resourceBundle);
when(client.saveJob(job)).thenReturn(Status.OK);

9
shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java
View File

@ -104,7 +104,8 @@ public Enumeration<String> getKeys() {
public void testCreateLink() {
ShellEnvironment.setInteractive(false);
when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null));
when(client.createLink("connector_test")).thenReturn(new MLink(1, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
when(client.createLink("connector_test")).thenReturn(
new MLink("connector_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
when(client.saveLink(any(MLink.class))).thenReturn(Status.OK);
// create link -c connector_test
@ -144,10 +145,10 @@ public void testCreateLinkInteractive() {
ShellEnvironment.setInteractive(true);
initEnv();
when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null));
MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
MLink link = new MLink("connector_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
when(client.createLink("connector_test")).thenReturn(link);
when(client.saveLink(any(MLink.class))).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle);
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
// create link -c connector_test
initData("linkname\r" + // link name
@ -228,7 +229,7 @@ public void testCreateJobInteractive() {
when(client.getConnector(1)).thenReturn(fromConnector);
when(client.getConnector(2)).thenReturn(toConnector);
when(client.saveJob(any(MJob.class))).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle);
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
when(client.getDriverConfigBundle()).thenReturn(resourceBundle);
// create job -f link_from -to link_to

3
shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java
View File

@ -203,7 +203,7 @@ public void testShowDriver() {
@Test
public void testShowLink() {
when(client.getLinks()).thenReturn(new ArrayList<MLink>());
when(client.getLink(any(String.class))).thenReturn(new MLink(1L, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
when(client.getLink(any(String.class))).thenReturn(new MLink("connector_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
// show link summary
out.reset();
@ -234,6 +234,7 @@ public void testShowLink() {
@Test
public void testShowJob() {
when(client.getJobs()).thenReturn(new ArrayList<MJob>());
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
when(client.getJob("1")).thenReturn(new MJob(1L, 2L, 1L, 2L,
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),

20
shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java
View File

@ -105,10 +105,10 @@ public Enumeration<String> getKeys() {
@Test
public void testUpdateLink() throws InterruptedException {
ShellEnvironment.setInteractive(false);
MLink link = new MLink(1L, new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
MLink link = new MLink("connector_test", new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link);
when(client.getConnectorConfigBundle(1L)).thenReturn(new MapResourceBundle(new HashMap()));
when(client.updateLink(link)).thenReturn(Status.OK);
when(client.getConnectorConfigBundle("connector_test")).thenReturn(new MapResourceBundle(new HashMap()));
when(client.updateLink(any(MLink.class))).thenReturn(Status.OK);
// update link -lid link_test
Status status = (Status) updateCmd.execute(Arrays.asList(Constants.FN_LINK, "-lid", "link_test"));
@ -134,14 +134,14 @@ public void testUpdateLink() throws InterruptedException {
}
@Test
public void testUpdateLinkInteractive() {
public void testUpdateLinkInteractive() throws Exception {
ShellEnvironment.setInteractive(true);
initEnv();
when(client.getConnector("connector_test")).thenReturn(new MConnector("", "", "", null, null, null));
MLink link = new MLink(1, new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
MLink link = new MLink("connector_test", new MLinkConfig(getConfig("CONFIGFROMNAME"), new ArrayList<MValidator>()));
when(client.getLink("link_test")).thenReturn(link);
when(client.updateLink(link)).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle);
when(client.updateLink(any(MLink.class))).thenReturn(Status.OK);
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
// update link -lid link_test
initData("linkname\r" + // link name
@ -178,7 +178,8 @@ public void testUpdateJob() throws InterruptedException {
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("connect_test", "", "", null, null, null));
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(new MapResourceBundle(new HashMap()));
when(client.getDriverConfigBundle()).thenReturn(new MapResourceBundle(new HashMap()));
when(client.updateJob(job)).thenReturn(Status.OK);
@ -213,7 +214,8 @@ public void testUpdateJobInteractive() {
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job);
when(client.getConnectorConfigBundle(any(Long.class))).thenReturn(resourceBundle);
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("connect_test", "", "", null, null, null));
when(client.getConnectorConfigBundle(any(String.class))).thenReturn(resourceBundle);
when(client.getDriverConfigBundle()).thenReturn(resourceBundle);
when(client.updateJob(job)).thenReturn(Status.OK);

1
test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
View File

@ -22,7 +22,6 @@
import java.lang.reflect.Method;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger;

6
test/src/test/java/org/apache/sqoop/integration/connector/hdfs/S3Test.java
View File

@ -90,7 +90,7 @@ public void testImportExport() throws Exception {
saveLink(hdfsLink);
// DB -> S3
MJob db2aws = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());
MJob db2aws = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
fillRdbmsFromConfig(db2aws, "id");
fillHdfsToConfig(db2aws, ToFormat.TEXT_FILE);
@ -110,7 +110,7 @@ public void testImportExport() throws Exception {
assertEquals(provider.rowCount(getTableName()), 0);
// S3 -> DB
MJob aws2db = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());
MJob aws2db = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
fillHdfsFromConfig(aws2db);
fillRdbmsToConfig(aws2db);
@ -145,7 +145,7 @@ public void testIncrementalRead() throws Exception {
saveLink(hdfsLink);
// S3 -> HDFS
MJob aws2hdfs = getClient().createJob(s3Link.getPersistenceId(), hdfsLink.getPersistenceId());
MJob aws2hdfs = getClient().createJob(s3Link.getName(), hdfsLink.getName());
fillHdfsFromConfig(aws2hdfs);
aws2hdfs.getFromJobConfig().getEnumInput("incremental.incrementalType").setValue(IncrementalType.NEW_FILES);

4
tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
View File

@ -278,7 +278,7 @@ private long loadLink(MLink link) {
Repository repository = RepositoryManager.getInstance().getRepository();
MConnector mConnector = ConnectorManager.getInstance().getConnectorConfigurable(link.getConnectorId());
MConnector mConnector = ConnectorManager.getInstance().getConnectorConfigurable(link.getConnectorName());
ConnectorConfigurableUpgrader connectorConfigUpgrader = ConnectorManager.getInstance().getSqoopConnector(mConnector.getUniqueName()).getConfigurableUpgrader(null);
List<MConfig> connectorConfigs = mConnector.getLinkConfig().clone(false).getConfigs();
@ -291,7 +291,7 @@ private long loadLink(MLink link) {
// Transform config structures to objects for validations
SqoopConnector connector = ConnectorManager.getInstance().getSqoopConnector(
link.getConnectorId());
link.getConnectorName());
Object connectorConfig = ClassUtils.instantiate(connector.getLinkConfigurationClass());