github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-12 15:01:45 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2713: Sqoop2: Use link name in MJob

Browse Source 
(Colin Ma via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2015-12-09 10:45:46 +01:00
parent e70b9c259a
commit 3f5b33d0d9
22 changed files with 144 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client/src/main/java/org/apache/sqoop/client/SqoopClient.java
View File

@ -395,8 +395,8 @@ public MJob createJob(String fromLinkName, String toLinkName) {
return new MJob(
connectorForFromLink.getUniqueName(),
connectorForToLink.getUniqueName(),
fromLink.getPersistenceId(),
toLink.getPersistenceId(),
fromLinkName,
toLinkName,
connectorForFromLink.getFromConfig().clone(false),
connectorForToLink.getToConfig().clone(false),
getDriverConfig()

16
common/src/main/java/org/apache/sqoop/json/JobBean.java
View File

@ -48,8 +48,8 @@
@InterfaceStability.Unstable
public class JobBean implements JsonBean {
static final String FROM_LINK_ID = "from-link-id";
static final String TO_LINK_ID = "to-link-id";
static final String FROM_LINK_NAME = "from-link-name";
static final String TO_LINK_NAME = "to-link-name";
static final String FROM_CONNECTOR_NAME = "from-connector-name";
static final String TO_CONNECTOR_NAME = "to-connector-name";
static final String FROM_CONFIG_VALUES = "from-config-values";
@ -137,8 +137,8 @@ private JSONObject extractJob(boolean skipSensitive, MJob job) {
object.put(FROM_CONNECTOR_NAME, job.getFromConnectorName());
object.put(TO_CONNECTOR_NAME, job.getToConnectorName());
// job associated links
object.put(FROM_LINK_ID, job.getFromLinkId());
object.put(TO_LINK_ID, job.getToLinkId());
object.put(FROM_LINK_NAME, job.getFromLinkName());
object.put(TO_LINK_NAME, job.getToLinkName());
// job configs
MFromConfig fromConfigList = job.getFromJobConfig();
object.put(FROM_CONFIG_VALUES, extractConfigList(fromConfigList, skipSensitive));
@ -170,8 +170,8 @@ private MJob restoreJob(Object obj) {
JSONObject object = (JSONObject) obj;
String fromConnectorName = JSONUtils.getString(object, FROM_CONNECTOR_NAME);
String toConnectorName = JSONUtils.getString(object, TO_CONNECTOR_NAME);
long fromConnectionId = JSONUtils.getLong(object, FROM_LINK_ID);
long toConnectionId = JSONUtils.getLong(object, TO_LINK_ID);
String fromLinkName = JSONUtils.getString(object, FROM_LINK_NAME);
String toLinkName = JSONUtils.getString(object, TO_LINK_NAME);
JSONObject fromConfigJson = JSONUtils.getJSONObject(object, FROM_CONFIG_VALUES);
JSONObject toConfigJson = JSONUtils.getJSONObject(object, TO_CONFIG_VALUES);
JSONObject driverConfigJson = JSONUtils.getJSONObject(object, DRIVER_CONFIG_VALUES);
@ -188,8 +188,8 @@ private MJob restoreJob(Object obj) {
MJob job = new MJob(
fromConnectorName,
toConnectorName,
fromConnectionId,
toConnectionId,
fromLinkName,
toLinkName,
new MFromConfig(fromConfigs, fromValidators),
new MToConfig(toConfigs, toValidators),
new MDriverConfig(driverConfigs, driverValidators)

41
common/src/main/java/org/apache/sqoop/model/MJob.java
View File

@ -37,9 +37,8 @@ public class MJob extends MAccountableEntity implements MClonable {
*/
private final String fromConnectorName;
private final String toConnectorName;
private final long fromLinkId;
private final long toLinkId;
private final String fromLinkName;
private final String toLinkName;
private final MFromConfig fromConfig;
private final MToConfig toConfig;
private final MDriverConfig driverConfig;
@ -49,23 +48,23 @@ public class MJob extends MAccountableEntity implements MClonable {
*
* @param fromConnectorId FROM Connector id
* @param toConnectorId TO Connector id
* @param fromLinkId FROM Link id
* @param toLinkId TO Link id
* @param fromLinkName FROM Link name
* @param toLinkName TO Link name
* @param fromConfig FROM job config
* @param toConfig TO job config
* @param driverConfig driver config
*/
public MJob(String fromConnectorName,
String toConnectorName,
long fromLinkId,
long toLinkId,
String fromLinkName,
String toLinkName,
MFromConfig fromConfig,
MToConfig toConfig,
MDriverConfig driverConfig) {
this.fromConnectorName = fromConnectorName;
this.toConnectorName = toConnectorName;
this.fromLinkId = fromLinkId;
this.toLinkId = toLinkId;
this.fromLinkName = fromLinkName;
this.toLinkName = toLinkName;
this.fromConfig = fromConfig;
this.toConfig = toConfig;
this.driverConfig = driverConfig;
@ -99,8 +98,8 @@ public MJob(MJob other, MFromConfig fromConfig, MToConfig toConfig, MDriverConfi
this.fromConnectorName = other.getFromConnectorName();
this.toConnectorName = other.getToConnectorName();
this.fromLinkId = other.getFromLinkId();
this.toLinkId = other.getToLinkId();
this.fromLinkName = other.getFromLinkName();
this.toLinkName = other.getToLinkName();
this.fromConfig = fromConfig;
this.toConfig = toConfig;
this.driverConfig = driverConfig;
@ -117,12 +116,12 @@ public String toString() {
return sb.toString();
}
public long getFromLinkId() {
return fromLinkId;
public String getFromLinkName() {
return fromLinkName;
}
public long getToLinkId() {
return toLinkId;
public String getToLinkName() {
return toLinkName;
}
public String getFromConnectorName() {
@ -153,8 +152,8 @@ public MJob clone(boolean cloneWithValue) {
return new MJob(
getFromConnectorName(),
getToConnectorName(),
getFromLinkId(),
getToLinkId(),
getFromLinkName(),
getToLinkName(),
getFromJobConfig().clone(false),
getToJobConfig().clone(false),
getDriverConfig().clone(false));
@ -174,8 +173,8 @@ public boolean equals(Object object) {
MJob job = (MJob)object;
return (job.getFromConnectorName().equals(this.getFromConnectorName()))
&& (job.getToConnectorName().equals(this.getToConnectorName()))
&& (job.getFromLinkId() == this.getFromLinkId())
&& (job.getToLinkId() == this.getToLinkId())
&& (job.getFromLinkName().equals(this.getFromLinkName()))
&& (job.getToLinkName().equals(this.getToLinkName()))
&& (job.getPersistenceId() == this.getPersistenceId())
&& (job.getFromJobConfig().equals(this.getFromJobConfig()))
&& (job.getToJobConfig().equals(this.getToJobConfig()))
@ -186,8 +185,8 @@ public boolean equals(Object object) {
public int hashCode() {
int result = fromConnectorName != null ? fromConnectorName.hashCode() : 0;
result = 31 * result + (toConnectorName != null ? toConnectorName.hashCode() : 0);
result = 31 * result + (int) (fromLinkId ^ (fromLinkId >>> 32));
result = 31 * result + (int) (toLinkId ^ (toLinkId >>> 32));
result = 31 * result + (fromLinkName != null ? fromLinkName.hashCode() : 0);
result = 31 * result + (toLinkName != null ? toLinkName.hashCode() : 0);
result = 31 * result + (fromConfig != null ? fromConfig.hashCode() : 0);
result = 31 * result + (toConfig != null ? toConfig.hashCode() : 0);
result = 31 * result + (driverConfig != null ? driverConfig.hashCode() : 0);

4
common/src/test/java/org/apache/sqoop/json/TestJobBean.java
View File

@ -61,8 +61,8 @@ public void testJobSerialization() throws ParseException {
assertEquals(22L, target.getPersistenceId());
assertEquals("The big Job", target.getName());
assertEquals(target.getFromLinkId(), 1);
assertEquals(target.getToLinkId(), 2);
assertEquals(target.getFromLinkName(), "fromLinkName");
assertEquals(target.getToLinkName(), "toLinkName");
assertEquals(target.getFromConnectorName(), "from_ahoj");
assertEquals(target.getToConnectorName(), "to_ahoj");
assertEquals(created, target.getCreationDate());

4
common/src/test/java/org/apache/sqoop/json/TestJobsBean.java
View File

@ -72,8 +72,8 @@ public void testJobsSerialization() throws ParseException {
assertEquals(44L, retrievedJob2.getPersistenceId());
assertEquals("The small Job", retrievedJob2.getName());
assertEquals(retrievedJob1.getFromLinkId(), 1);
assertEquals(retrievedJob1.getToLinkId(), 2);
assertEquals(retrievedJob1.getFromLinkName(), "fromLinkName");
assertEquals(retrievedJob1.getToLinkName(), "toLinkName");
assertEquals(retrievedJob1.getFromConnectorName(), "from_ahoj");
assertEquals(retrievedJob1.getToConnectorName(), "to_ahoj");
assertEquals(created, retrievedJob1.getCreationDate());

2
common/src/test/java/org/apache/sqoop/json/util/BeanTestUtil.java
View File

@ -79,7 +79,7 @@ public static MConnector getConnector(Long id, String name, boolean from, boolea
public static MJob getJob(String connectorName) {
String fromConnectorName = "from_" + connectorName;
String toConnectorName = "to_" + connectorName;
return new MJob(fromConnectorName, toConnectorName, 1, 2, getConnector(1L, fromConnectorName).getFromConfig(),
return new MJob(fromConnectorName, toConnectorName, "fromLinkName", "toLinkName", getConnector(1L, fromConnectorName).getFromConfig(),
getConnector(2L, toConnectorName).getToConfig(), ConfigTestUtil.getDriverConfig());
}

3
common/src/test/java/org/apache/sqoop/model/TestMJob.java
View File

@ -98,7 +98,8 @@ public void testClone() {
.getConfig("CONFIGFROMNAME").getInput("STRING-INPUT").getValue()); }
private MJob job() {
MJob job = new MJob("fromConnectorName", "toConnectorName", 1L, 2L, fromConfig(), toConfig(), driverConfig());
MJob job = new MJob("fromConnectorName", "toConnectorName", "fromLinkName", "toLinkName",
fromConfig(), toConfig(), driverConfig());
job.setName("Vampire");
job.setCreationUser("Buffy");
return job;

12
core/src/main/java/org/apache/sqoop/driver/JobManager.java
View File

@ -339,8 +339,8 @@ public MSubmission start(String jobName, HttpEventContext ctx) {
private JobRequest createJobRequest(MSubmission submission, MJob job) {
// get from/to connections for the job
MLink fromLink = getLink(job.getFromLinkId());
MLink toLink = getLink(job.getToLinkId());
MLink fromLink = getLink(job.getFromLinkName());
MLink toLink = getLink(job.getToLinkName());
// get from/to connectors for the connection
SqoopConnector fromConnector = getSqoopConnector(fromLink.getConnectorName());
@ -486,9 +486,9 @@ void validateSupportedDirection(SqoopConnector connector, Direction direction) {
}
}
MLink getLink(long linkId) {
MLink getLink(String linkName) {
MLink link = RepositoryManager.getInstance().getRepository()
.findLink(linkId);
.findLink(linkName);
if (!link.getEnabled()) {
throw new SqoopException(DriverError.DRIVER_0010, "Connection: "
+ link.getName());
@ -585,8 +585,8 @@ void invokeDestroyerOnJobSuccess(MSubmission submission) {
SqoopConnector fromConnector = getSqoopConnector(job.getFromConnectorName());
SqoopConnector toConnector = getSqoopConnector(job.getToConnectorName());
MLink fromConnection = getLink(job.getFromLinkId());
MLink toConnection = getLink(job.getToLinkId());
MLink fromConnection = getLink(job.getFromLinkName());
MLink toConnection = getLink(job.getToLinkName());
Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass());
ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig);

18
core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
View File

@ -109,19 +109,19 @@ public void testUnsupportedDirectionForConnector() {
@Test
public void testGetLink() {
MLink testLink = new MLink("connector_test", null);
MLink testLink = new MLink("linkName", null);
testLink.setEnabled(true);
MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
when(jdbcRepoMock.findLink(123l)).thenReturn(mConnectionSpy);
assertEquals(jobManager.getLink(123l), mConnectionSpy);
when(jdbcRepoMock.findLink("linkName")).thenReturn(mConnectionSpy);
assertEquals(jobManager.getLink("linkName"), mConnectionSpy);
verify(repositoryManagerMock, times(1)).getRepository();
verify(jdbcRepoMock, times(1)).findLink(123l);
verify(jdbcRepoMock, times(1)).findLink("linkName");
}
@Test
public void testDisabledLink() {
MLink testConnection = new MLink("connector_test", null);
MLink testConnection = new MLink("linkName", null);
testConnection.setPersistenceId(1234);
testConnection.setEnabled(false);
SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection: "
@ -129,13 +129,13 @@ public void testDisabledLink() {
MLink mConnectionSpy = org.mockito.Mockito.spy(testConnection);
when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
when(jdbcRepoMock.findLink(123l)).thenReturn(mConnectionSpy);
when(jdbcRepoMock.findLink("linkName")).thenReturn(mConnectionSpy);
try {
jobManager.getLink(123l);
jobManager.getLink("linkName");
} catch (SqoopException ex) {
assertEquals(ex.getMessage(), exception.getMessage());
verify(repositoryManagerMock, times(1)).getRepository();
verify(jdbcRepoMock, times(1)).findLink(123l);
verify(jdbcRepoMock, times(1)).findLink("linkName");
}
}
@ -188,7 +188,7 @@ public void testUnknownJob() {
}
private MJob job(String jobName, String fromConnectorName, String toConnectorName) {
MJob job = new MJob(fromConnectorName, toConnectorName, 1L, 2L, null, null, null);
MJob job = new MJob(fromConnectorName, toConnectorName, "fromLinkName", "toLinkName", null, null, null);
job.setName(jobName);
job.setCreationUser("Buffy");
return job;

38
core/src/test/java/org/apache/sqoop/repository/TestJdbcRepository.java
View File

@ -229,7 +229,8 @@ public void testConnectorConfigUpgradeWithValidLinksAndJobs() {
// prepare the links and jobs
// the connector Id for both are the same
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", "A1", "A1", 1, 1), job(2, "JB", "A1", "A1", 2, 2));
List<MJob> jobList = jobs(job(1, "JA", "A1", "A1", "linkName1", "linkName1"),
job(2, "JB", "A1", "A1", "linkName2", "linkName2"));
// mock necessary methods for upgradeConnector() procedure
doReturn(linkList).when(repoSpy).findLinksForConnectorUpgrade(anyString());
@ -277,7 +278,8 @@ public void testDriverConfigUpgradeWithValidJobs() {
when(driverMock.getConfigurableUpgrader(DriverBean.CURRENT_DRIVER_VERSION)).thenReturn(driverUpgraderMock);
when(driverMock.getDriverJobConfigurationClass()).thenReturn(ValidConfiguration.class);
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(jobList).when(repoSpy).findJobs();
doNothing().when(repoSpy).updateLink(any(MLink.class), any(RepositoryTransaction.class));
@ -315,7 +317,8 @@ public void testDriverConfigUpgradeWithInvalidJobs() {
when(driverMock.getConfigurableUpgrader(DriverBean.CURRENT_DRIVER_VERSION)).thenReturn(driverUpgraderMock);
when(driverMock.getDriverJobConfigurationClass()).thenReturn(InvalidConfiguration.class);
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(jobList).when(repoSpy).findJobs();
doNothing().when(repoSpy).updateJob(any(MJob.class), any(RepositoryTransaction.class));
@ -425,7 +428,8 @@ public void testConnectorConfigUpgradeHandlerWithDeleteJobInputsError() {
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade
(anyLong(), any(Connection.class));
@ -463,7 +467,8 @@ public void testConnectorConfigUpgradeHandlerWithDeleteLinkInputsError() {
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyString(), any(Connection.class));
@ -501,7 +506,8 @@ public void testConnectorConfigUpgradeHandlerWithUpdateConnectorError() {
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyString(), any(Connection.class));
@ -543,7 +549,8 @@ public void testConnectorConfigUpgradeHandlerWithUpdateLinkError() {
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyString(), any(Connection.class));
@ -589,7 +596,8 @@ public void testConnectorConfigUpgradeHandlerWithUpdateJobError() {
when(connectorMgrMock.getSqoopConnector(anyString())).thenReturn(sqconnector);
List<MLink> linkList = links(link(1, "LA", oldConnector.getUniqueName()), link(2, "LB", oldConnector.getUniqueName()));;
List<MJob> jobList = jobs(job(1, "JA", "A1", "A1", 1, 1), job(2, "JB", "A1", "A1", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "A1", "A1", "linkName1", "linkName1"),
job(2, "JB", "A1", "A1", "linkName2", "linkName1"));
doReturn(linkList).when(repoHandlerMock).findLinksForConnectorUpgrade(anyString(), any(Connection.class));
doReturn(jobList).when(repoHandlerMock).findJobsForConnectorUpgrade(anyLong(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyString(), any(Connection.class));
@ -659,7 +667,8 @@ public void testDriverConfigUpgradeHandlerWithDeleteJobInputsError() {
when(driverMock.getConfigurableUpgrader(DriverBean.CURRENT_DRIVER_VERSION)).thenReturn(driverUpgraderMock);
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
SqoopException exception = new SqoopException(RepositoryError.JDBCREPO_0000,
@ -689,7 +698,8 @@ public void testDriverConfigUpgradeHandlerWithUpdateDriverConfigError() {
when(driverMock.getConfigurableUpgrader(DriverBean.CURRENT_DRIVER_VERSION)).thenReturn(driverUpgraderMock);
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyString(), any(Connection.class));
doNothing().when(repoHandlerMock).deleteLinkInputs(anyString(), any(Connection.class));
@ -723,7 +733,8 @@ public void testDriverConfigUpgradeHandlerWithUpdateJobError() {
when(driverMock.getConfigurableUpgrader(DriverBean.CURRENT_DRIVER_VERSION)).thenReturn(driverUpgraderMock);
when(driverMock.getDriverJobConfigurationClass()).thenReturn(ValidConfiguration.class);
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", 1, 1), job(2, "JB", "fromConnectorName", "toConnectorName", 2, 1));
List<MJob> jobList = jobs(job(1, "JA", "fromConnectorName", "toConnectorName", "linkName1", "linkName1"),
job(2, "JB", "fromConnectorName", "toConnectorName", "linkName2", "linkName1"));
doReturn(jobList).when(repoHandlerMock).findJobs(any(Connection.class));
doNothing().when(repoHandlerMock).deleteJobInputs(anyString(), any(Connection.class));
doNothing().when(repoHandlerMock).upgradeDriverAndConfigs(any(MDriver.class), any(Connection.class));
@ -782,8 +793,9 @@ private MLink link(long linkId, String linkName, String connectorName) {
return link;
}
private MJob job(long id, String jobName, String fromConnectorName, String toConnectorName, long fromLinkId, long toLinkId) {
MJob job = new MJob(fromConnectorName, toConnectorName, fromLinkId, toLinkId,
private MJob job(long id, String jobName, String fromConnectorName, String toConnectorName,
String fromLinkName, String toLinkName) {
MJob job = new MJob(fromConnectorName, toConnectorName, fromLinkName, toLinkName,
new MFromConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>()),
new MToConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>()),
new MDriverConfig(new LinkedList<MConfig>(), new LinkedList<MValidator>()));

34
repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
View File

@ -624,15 +624,34 @@ public List<MLink> findLinksForConnector(String connectorName,
}
}
public Long findLinkIdByName(String linkName, Connection conn) {
try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtSelectLinkIdByName())) {
stmt.setString(1,linkName);
try (ResultSet rs = stmt.executeQuery()) {
if (rs.next()) {
return rs.getLong(1);
} else {
throw new SqoopException(CommonRepositoryError.COMMON_0020);
}
}
} catch (SQLException ex) {
logException(ex);
throw new SqoopException(CommonRepositoryError.COMMON_0036, ex);
}
}
/**
* {@inheritDoc}
*/
public void createJob(MJob job, Connection conn) {
int result;
// get link id to create job
Long fromLinkId = findLinkIdByName(job.getFromLinkName(), conn);
Long toLinkId = findLinkIdByName(job.getToLinkName(), conn);
try (PreparedStatement stmt = conn.prepareStatement(crudQueries.getStmtInsertJob(), Statement.RETURN_GENERATED_KEYS)) {
stmt.setString(1, job.getName());
stmt.setLong(2, job.getFromLinkId());
stmt.setLong(3, job.getToLinkId());
stmt.setLong(2, fromLinkId);
stmt.setLong(3, toLinkId);
stmt.setBoolean(4, job.getEnabled());
stmt.setString(5, job.getCreationUser());
stmt.setTimestamp(6, new Timestamp(job.getCreationDate().getTime()));
@ -1615,8 +1634,6 @@ private List<MJob> loadJobsForUpgrade(PreparedStatement stmt,
long toConnectorId = rsJob.getLong(2);
long id = rsJob.getLong(3);
String name = rsJob.getString(4);
long fromLinkId = rsJob.getLong(5);
long toLinkId = rsJob.getLong(6);
boolean enabled = rsJob.getBoolean(7);
String createBy = rsJob.getString(8);
Date creationDate = rsJob.getTimestamp(9);
@ -1624,6 +1641,8 @@ private List<MJob> loadJobsForUpgrade(PreparedStatement stmt,
Date lastUpdateDate = rsJob.getTimestamp(11);
String fromConnectorName = rsJob.getString(12);
String toConnectorName = rsJob.getString(13);
String fromLinkName = rsJob.getString(14);
String toLinkName = rsJob.getString(15);
fromConfigFetchStmt.setLong(1, fromConnectorId);
toConfigFetchStmt.setLong(1,toConnectorId);
@ -1653,7 +1672,7 @@ private List<MJob> loadJobsForUpgrade(PreparedStatement stmt,
MJob job = new MJob(
fromConnectorName, toConnectorName,
fromLinkId, toLinkId,
fromLinkName, toLinkName,
new MFromConfig(fromConnectorFromJobConfig, Collections.EMPTY_LIST),
new MToConfig(toConnectorToJobConfig, Collections.EMPTY_LIST),
new MDriverConfig(driverConfig, Collections.EMPTY_LIST));
@ -1706,7 +1725,8 @@ private List<MJob> loadJobs(PreparedStatement stmt,
Date lastUpdateDate = rsJob.getTimestamp(11);
String fromConnectorName = rsJob.getString(12);
String toConnectorName = rsJob.getString(13);
String fromLinkName = rsJob.getString(14);
String toLinkName = rsJob.getString(15);
driverConfigfetchStmt.setLong(1, driverId);
jobInputFetchStmt.setLong(1, id);
@ -1728,7 +1748,7 @@ private List<MJob> loadJobs(PreparedStatement stmt,
MJob job = new MJob(
fromConnectorName, toConnectorName,
fromLinkId, toLinkId,
fromLinkName, toLinkName,
new MFromConfig(mFromConfig.getConfigs(), Collections.EMPTY_LIST),
new MToConfig(mToConfig.getConfigs(), Collections.EMPTY_LIST),
new MDriverConfig(driverConfig, Collections.EMPTY_LIST));

13
repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
View File

@ -335,6 +335,13 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
STMT_SELECT_LINK_ALL
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_NAME) + " = ?";
// DML Select link id by name
private static final String STMT_SELECT_LINK_ID_BY_NAME =
"SELECT "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_ID)
+ " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_NAME)
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_NAME) + " = ?";
// DML: Select all links for a specific connector.
private static final String STMT_SELECT_LINK_FOR_CONNECTOR_CONFIGURABLE =
"SELECT "
@ -442,7 +449,9 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ "JOB." + CommonRepoUtils.escapeColumnName(COLUMN_SQB_UPDATE_USER) + ", "
+ "JOB." + CommonRepoUtils.escapeColumnName(COLUMN_SQB_UPDATE_DATE) + ", "
+ "FROM_CONF_NAME." + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME) + ", "
+ "TO_CONF_NAME." + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME)
+ "TO_CONF_NAME." + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME) + ", "
+ "FROM_CONNECTOR." + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_NAME) + ", "
+ "TO_CONNECTOR." + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_NAME)
+ " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME) + " JOB"
+ " LEFT JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_NAME) + " FROM_CONNECTOR"
+ " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQB_FROM_LINK) + " = FROM_CONNECTOR." + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNK_ID)
@ -800,6 +809,8 @@ public String getStmtSelectLinkSingleByName() {
return STMT_SELECT_LINK_SINGLE_BY_NAME;
}
public String getStmtSelectLinkIdByName() {return STMT_SELECT_LINK_ID_BY_NAME;}
public String getStmtSelectLinkAll() {
return STMT_SELECT_LINK_ALL;
}

2
repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java
View File

@ -343,7 +343,7 @@ public void testDeleteJob() throws Exception {
}
public MJob getJob() {
return new MJob("A", "A", 1, 1, handler.findConnector("A", derbyConnection).getFromConfig(),
return new MJob("A", "A", "CA", "CA", handler.findConnector("A", derbyConnection).getFromConfig(),
handler.findConnector("A", derbyConnection).getToConfig(), handler.findDriver(
MDriver.DRIVER_NAME, derbyConnection).getDriverConfig());
}

4
repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
View File

@ -106,8 +106,8 @@ protected MJob getJob(String name, MConnector connectorA,
MDriver driver = handler.findDriver(MDriver.DRIVER_NAME,
provider.getConnection());
MJob job = new MJob(connectorA.getUniqueName(),
connectorB.getUniqueName(), linkA.getPersistenceId(),
linkB.getPersistenceId(), connectorA.getFromConfig(),
connectorB.getUniqueName(), linkA.getName(),
linkB.getName(), connectorA.getFromConfig(),
connectorB.getToConfig(), driver.getDriverConfig());
job.setName(name);
fillJob(job);

4
repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
View File

@ -94,8 +94,8 @@ protected MJob getJob(String name, MConnector connectorA, MConnector connectorB,
MJob job = new MJob(
connectorA.getUniqueName(),
connectorB.getUniqueName(),
linkA.getPersistenceId(),
linkB.getPersistenceId(),
linkA.getName(),
linkB.getName(),
connectorA.getFromConfig(),
connectorB.getToConfig(),
driver.getDriverConfig());

7
server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
View File

@ -179,14 +179,13 @@ private JsonBean createUpdateJob(RequestContext ctx, boolean create) {
// Job object
MJob postedJob = jobs.get(0);
MLink fromLink = HandlerUtils.getLinkFromLinkId(postedJob.getFromLinkId());
MLink toLink = HandlerUtils.getLinkFromLinkId(postedJob.getToLinkId());
// Authorization check
if (create) {
AuthorizationEngine.createJob(ctx.getUserName(), fromLink.getName(), toLink.getName());
AuthorizationEngine.createJob(ctx.getUserName(), postedJob.getFromLinkName(), postedJob.getToLinkName());
} else {
AuthorizationEngine.updateJob(ctx.getUserName(), fromLink.getName(), toLink.getName(), postedJob.getName());
AuthorizationEngine.updateJob(ctx.getUserName(), postedJob.getFromLinkName(), postedJob.getToLinkName(),
postedJob.getName());
}
// Verify that user is not trying to spoof us

21
shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
View File

@ -92,22 +92,9 @@ private void showSummary() {
for(MJob job : jobs) {
ids.add(String.valueOf(job.getPersistenceId()));
names.add(job.getName());
// From link and connnector
String fromLinkName = "";
MLink fromLink = client.getLink(job.getFromLinkId());
if (fromLink != null) {
fromLinkName = fromLink.getName();
}
fromConnectors.add(fromLinkName + " (" + job.getFromConnectorName() + ")");
// To link and connector
String toLinkName = "";
MLink toLink = client.getLink(job.getToLinkId());
if (toLink != null) {
toLinkName = toLink.getName();
}
toConnectors.add(toLinkName + " (" + job.getToConnectorName() + ")");
fromConnectors.add(job.getFromLinkName() + " (" + job.getFromConnectorName() + ")");
toConnectors.add(job.getToLinkName() + " (" + job.getToConnectorName() + ")");
availabilities.add(String.valueOf(job.getEnabled()));
}
@ -153,11 +140,11 @@ private void displayJob(MJob job) {
displayConfig(job.getDriverConfig().getConfigs(),
client.getDriverConfigBundle());
printlnResource(Constants.RES_SHOW_PROMPT_JOB_FROM_LID_INFO,
job.getFromLinkId());
job.getFromLinkName());
displayConfig(job.getFromJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getFromConnectorName()));
printlnResource(Constants.RES_SHOW_PROMPT_JOB_TO_LID_INFO,
job.getToLinkId());
job.getToLinkName());
displayConfig(job.getToJobConfig().getConfigs(),
client.getConnectorConfigBundle(job.getToConnectorName()));
}

5
shell/src/test/java/org/apache/sqoop/shell/TestCloneCommand.java
View File

@ -155,7 +155,7 @@ public void testCloneLinkInteractive() {
@Test
public void testCloneJob() {
ShellEnvironment.setInteractive(false);
MJob job = new MJob("fromConnectorName", "toConnectorName", 1L, 2L,
MJob job = new MJob("fromConnectorName", "toConnectorName", "linkName1", "linkName2",
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
@ -192,7 +192,8 @@ public void testCloneJob() {
public void testCloneJobInteractive() {
ShellEnvironment.setInteractive(true);
initEnv();
MJob job = new MJob("fromConnectorName", "toConnectorName", 1, 2, new MFromConfig(getConfig("fromJobConfig"), new ArrayList<MValidator>()),
MJob job = new MJob("fromConnectorName", "toConnectorName", "linkName1", "linkName2",
new MFromConfig(getConfig("fromJobConfig"), new ArrayList<MValidator>()),
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job);

6
shell/src/test/java/org/apache/sqoop/shell/TestCreateCommand.java
View File

@ -182,7 +182,8 @@ public void testCreateJob() {
MConnector fromConnector = new MConnector("connector_from", "", "", null, new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), null);
MConnector toConnector = new MConnector("connector_to", "", "", null, null, new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
when(client.createJob("link_from", "link_to")).thenReturn(
new MJob("fromConnectorName", "toConnectorName", 1, 2, new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MJob("fromConnectorName", "toConnectorName", "link_from", "link_to",
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
when(client.getConnector("fromConnectorName")).thenReturn(fromConnector);
@ -222,7 +223,8 @@ public void testCreateJobInteractive() {
initEnv();
MConnector fromConnector = new MConnector("connector_from", "", "", null, new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), null);
MConnector toConnector = new MConnector("connector_to", "", "", null, null, new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
MJob job = new MJob("fromConnectorName", "toConnectorName", 1, 2, new MFromConfig(getConfig("fromJobConfig"), new ArrayList<MValidator>()),
MJob job = new MJob("fromConnectorName", "toConnectorName", "link_from", "link_to",
new MFromConfig(getConfig("fromJobConfig"), new ArrayList<MValidator>()),
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.createJob("link_from", "link_to")).thenReturn(job);

6
shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java
View File

@ -235,12 +235,12 @@ public void testShowLink() {
public void testShowJob() {
when(client.getJobs()).thenReturn(new ArrayList<MJob>());
when(client.getConnector(any(Long.class))).thenReturn(new MConnector("", "", "", null, null, null));
when(client.getJob("1")).thenReturn(new MJob("fromConnectorName", "toConnectorName", 1L, 2L,
when(client.getJob("1")).thenReturn(new MJob("fromConnectorName", "toConnectorName", "linkName1", "linkName2",
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())));
when(client.getJobsByConnector("2")).thenReturn(Arrays.asList(new MJob("fromConnectorName", "toConnectorName", 1L, 2L,
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
when(client.getJobsByConnector("2")).thenReturn(Arrays.asList(new MJob("fromConnectorName", "toConnectorName",
"linkName1", "linkName2", new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()))));

5
shell/src/test/java/org/apache/sqoop/shell/TestUpdateCommand.java
View File

@ -173,7 +173,7 @@ public void testUpdateLinkInteractive() throws Exception {
@Test
public void testUpdateJob() throws InterruptedException {
ShellEnvironment.setInteractive(false);
MJob job = new MJob("fromConnectorName", "toConnectorName", 1L, 2L,
MJob job = new MJob("fromConnectorName", "toConnectorName", "link_from", "link_to",
new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()),
new MDriverConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()));
@ -210,7 +210,8 @@ public void testUpdateJob() throws InterruptedException {
public void testUpdateJobInteractive() {
ShellEnvironment.setInteractive(true);
initEnv();
MJob job = new MJob("fromConnectorName", "toConnectorName", 1, 2, new MFromConfig(getConfig("fromJobConfig"), new ArrayList<MValidator>()),
MJob job = new MJob("fromConnectorName", "toConnectorName", "link_from", "link_to",
new MFromConfig(getConfig("fromJobConfig"), new ArrayList<MValidator>()),
new MToConfig(getConfig("toJobConfig"), new ArrayList<MValidator>()),
new MDriverConfig(getConfig("driverConfig"), new ArrayList<MValidator>()));
when(client.getJob("job_test")).thenReturn(job);

2
test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
View File

@ -66,8 +66,10 @@ public void testCreateLinkWithNonexistantConnector() throws Exception {
@AfterMethod
public void stopCluster() throws Exception {
if (getCluster() != null) {
getCluster().stop();
}
}
@Override
public void startSqoop() throws Exception {