github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-19 10:21:14 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2678: Sqoop2: Remove the id from public interface for Link

Browse Source 
(Colin Ma via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2015-11-13 11:54:05 -08:00
parent 7112964e75
commit aaee89dc6c
27 changed files with 104 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
server/src/main/java/org/apache/sqoop/handler/HandlerUtils.java
View File

@ -37,53 +37,24 @@ public static MJob getJobFromIdentifier(String identifier) {
return job; return job;
} }
public static long getLinkIdFromIdentifier(String identifier) { public static MLink getLinkFromLinkName(String linkName) {
// support linkName or linkId for the api
// NOTE: linkId is a fallback for older sqoop clients if any, since we want
// to primarily use unique linkNames
long linkId;
Repository repository = RepositoryManager.getInstance().getRepository(); Repository repository = RepositoryManager.getInstance().getRepository();
MLink link = repository.findLink(identifier); MLink link = repository.findLink(linkName);
if (link != null) { if (link == null) {
linkId = link.getPersistenceId(); throw new SqoopException(ServerError.SERVER_0006, "Invalid link name: " + linkName
} else { + " doesn't exist");
try {
linkId = Long.parseLong(identifier);
} catch (NumberFormatException ex) {
// this means name nor Id existed and we want to throw a user friendly
// message than a number format exception
throw new SqoopException(ServerError.SERVER_0005, "Invalid link: " + identifier
+ " requested");
}
} }
return linkId; return link;
} }
public static String getLinkNameFromIdentifier(String identifier) { public static MLink getLinkFromLinkId(Long linkId) {
// support linkName or linkId for the api
// NOTE: linkId is a fallback for older sqoop clients if any, since we want
// to primarily use unique linkNames
Repository repository = RepositoryManager.getInstance().getRepository(); Repository repository = RepositoryManager.getInstance().getRepository();
MLink link = repository.findLink(identifier); MLink link = repository.findLink(linkId);
if (link == null) { if (link == null) {
long linkId; throw new SqoopException(ServerError.SERVER_0006, "Invalid link id: " + linkId
try { + " doesn't exist");
linkId = Long.parseLong(identifier);
} catch (NumberFormatException ex) {
// this means name nor Id existed and we want to throw a user friendly
// message than a number format exception
throw new SqoopException(ServerError.SERVER_0005, "Invalid link: " + identifier
+ " requested");
}
link = repository.findLink(linkId);
if (link == null) {
throw new SqoopException(ServerError.SERVER_0006, "Link: " + identifier
+ " doesn't exist");
}
} }
return link;
return link.getName();
} }
public static long getConnectorIdFromIdentifier(String identifier) { public static long getConnectorIdFromIdentifier(String identifier) {

20
server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
View File

@ -37,14 +37,7 @@
import org.apache.sqoop.json.JsonBean; import org.apache.sqoop.json.JsonBean;
import org.apache.sqoop.json.SubmissionBean; import org.apache.sqoop.json.SubmissionBean;
import org.apache.sqoop.json.ValidationResultBean; import org.apache.sqoop.json.ValidationResultBean;
import org.apache.sqoop.model.ConfigUtils; import org.apache.sqoop.model.*;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MResource;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.repository.Repository; import org.apache.sqoop.repository.Repository;
import org.apache.sqoop.repository.RepositoryManager; import org.apache.sqoop.repository.RepositoryManager;
import org.apache.sqoop.request.HttpEventContext; import org.apache.sqoop.request.HttpEventContext;
@ -186,17 +179,14 @@ private JsonBean createUpdateJob(RequestContext ctx, boolean create) {
// Job object // Job object
MJob postedJob = jobs.get(0); MJob postedJob = jobs.get(0);
MLink fromLink = HandlerUtils.getLinkFromLinkId(postedJob.getFromLinkId());
MLink toLink = HandlerUtils.getLinkFromLinkId(postedJob.getToLinkId());
// Authorization check // Authorization check
if (create) { if (create) {
AuthorizationEngine.createJob(ctx.getUserName(), AuthorizationEngine.createJob(ctx.getUserName(), fromLink.getName(), toLink.getName());
HandlerUtils.getLinkNameFromIdentifier(String.valueOf(postedJob.getFromLinkId())),
HandlerUtils.getLinkNameFromIdentifier(String.valueOf(postedJob.getToLinkId())));
} else { } else {
AuthorizationEngine.updateJob(ctx.getUserName(), AuthorizationEngine.updateJob(ctx.getUserName(), fromLink.getName(), toLink.getName(), postedJob.getName());
HandlerUtils.getLinkNameFromIdentifier(String.valueOf(postedJob.getFromLinkId())),
HandlerUtils.getLinkNameFromIdentifier(String.valueOf(postedJob.getToLinkId())),
postedJob.getName());
} }
// Verify that user is not trying to spoof us // Verify that user is not trying to spoof us

38
server/src/main/java/org/apache/sqoop/handler/LinkRequestHandler.java
View File

@ -90,15 +90,15 @@ public JsonBean handleEvent(RequestContext ctx) {
*/ */
private JsonBean deleteLink(RequestContext ctx) { private JsonBean deleteLink(RequestContext ctx) {
Repository repository = RepositoryManager.getInstance().getRepository(); Repository repository = RepositoryManager.getInstance().getRepository();
String linkIdentifier = ctx.getLastURLElement(); String linkName = ctx.getLastURLElement();
// support linkName or linkId for the api // make sure the link exist, otherwise, the exception will be thrown
String linkName = HandlerUtils.getLinkNameFromIdentifier(linkIdentifier); MLink link = HandlerUtils.getLinkFromLinkName(linkName);
// Authorization check // Authorization check
AuthorizationEngine.deleteLink(ctx.getUserName(), linkName); AuthorizationEngine.deleteLink(ctx.getUserName(), link.getName());
AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(), AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(),
ctx.getRequest().getRemoteAddr(), "delete", "link", linkIdentifier); ctx.getRequest().getRemoteAddr(), "delete", "link", link.getName());
repository.deleteLink(linkName); repository.deleteLink(linkName);
MResource resource = new MResource(linkName, MResource.TYPE.LINK); MResource resource = new MResource(linkName, MResource.TYPE.LINK);
@ -142,7 +142,7 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
} else { } else {
AuthorizationEngine.updateLink(ctx.getUserName(), AuthorizationEngine.updateLink(ctx.getUserName(),
HandlerUtils.getConnectorNameFromIdentifier(String.valueOf(postedLink.getConnectorId())), HandlerUtils.getConnectorNameFromIdentifier(String.valueOf(postedLink.getConnectorId())),
HandlerUtils.getLinkNameFromIdentifier(String.valueOf(postedLink.getPersistenceId()))); postedLink.getName());
} }
MLinkConfig linkConfig = ConnectorManager.getInstance() MLinkConfig linkConfig = ConnectorManager.getInstance()
@ -152,11 +152,9 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
} }
// if update get the link id from the request URI // if update get the link id from the request URI
if (!create) { if (!create) {
String linkIdentifier = ctx.getLastURLElement(); String linkName = ctx.getLastURLElement();
// support linkName or linkId for the api MLink existingLink = repository.findLink(linkName);
String linkName = HandlerUtils.getLinkNameFromIdentifier(linkIdentifier);
if (postedLink.getPersistenceId() == MPersistableEntity.PERSISTANCE_ID_DEFAULT) { if (postedLink.getPersistenceId() == MPersistableEntity.PERSISTANCE_ID_DEFAULT) {
MLink existingLink = repository.findLink(linkName);
postedLink.setPersistenceId(existingLink.getPersistenceId()); postedLink.setPersistenceId(existingLink.getPersistenceId());
} }
} }
@ -194,15 +192,15 @@ private JsonBean createUpdateLink(RequestContext ctx, boolean create) {
} }
private JsonBean getLinks(RequestContext ctx) { private JsonBean getLinks(RequestContext ctx) {
String identifier = ctx.getLastURLElement(); String linkName = ctx.getLastURLElement();
LinkBean linkBean; LinkBean linkBean;
List<MLink> links; List<MLink> links;
Locale locale = ctx.getAcceptLanguageHeader(); Locale locale = ctx.getAcceptLanguageHeader();
Repository repository = RepositoryManager.getInstance().getRepository(); Repository repository = RepositoryManager.getInstance().getRepository();
AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(), "get", "link", identifier); AuditLoggerManager.getInstance().logAuditEvent(ctx.getUserName(), ctx.getRequest().getRemoteAddr(), "get", "link", linkName);
if(identifier.equals("all")) { // Return all links (by perhaps only for given connector) if(linkName.equals("all")) { // Return all links (by perhaps only for given connector)
String connectorName = ctx.getParameterValue(CONNECTOR_NAME_QUERY_PARAM); String connectorName = ctx.getParameterValue(CONNECTOR_NAME_QUERY_PARAM);
if(StringUtils.isEmpty(connectorName)) { if(StringUtils.isEmpty(connectorName)) {
@ -214,16 +212,16 @@ private JsonBean getLinks(RequestContext ctx) {
links = repository.findLinksForConnector(connectorName); links = repository.findLinksForConnector(connectorName);
} }
} else { // Return one specific link with name or id stored in identifier } else { // Return one specific link with name or id stored in identifier
String linkName = HandlerUtils.getLinkNameFromIdentifier(identifier); MLink link = HandlerUtils.getLinkFromLinkName(linkName);
links = new LinkedList<>(); links = new LinkedList<>();
links.add(repository.findLink(linkName)); links.add(link);
} }
// Authorization check // Authorization check
links = AuthorizationEngine.filterResource(ctx.getUserName(), MResource.TYPE.LINK, links); links = AuthorizationEngine.filterResource(ctx.getUserName(), MResource.TYPE.LINK, links);
// Return bean entity (we have to separate what we're returning here) // Return bean entity (we have to separate what we're returning here)
if(identifier.equals("all")) { if(linkName.equals("all")) {
linkBean = createLinksBean(links, locale); linkBean = createLinksBean(links, locale);
} else { } else {
linkBean = createLinkBean(links, locale); linkBean = createLinkBean(links, locale);
@ -257,13 +255,13 @@ private void addConnectorConfigBundle(Locale locale, LinkBean bean) {
private JsonBean enableLink(RequestContext ctx, boolean enabled) { private JsonBean enableLink(RequestContext ctx, boolean enabled) {
Repository repository = RepositoryManager.getInstance().getRepository(); Repository repository = RepositoryManager.getInstance().getRepository();
String[] elements = ctx.getUrlElements(); String[] elements = ctx.getUrlElements();
String linkIdentifier = elements[elements.length - 2]; String linkName = elements[elements.length - 2];
String linkName = HandlerUtils.getLinkNameFromIdentifier(linkIdentifier); MLink link = HandlerUtils.getLinkFromLinkName(linkName);
// Authorization check // Authorization check
AuthorizationEngine.enableDisableLink(ctx.getUserName(), linkName); AuthorizationEngine.enableDisableLink(ctx.getUserName(), link.getName());
repository.enableLink(linkName, enabled); repository.enableLink(link.getName(), enabled);
return JsonBean.EMPTY_BEAN; return JsonBean.EMPTY_BEAN;
} }
} }

2
test/src/test/java/org/apache/sqoop/integration/connector/hdfs/AppendModeTest.java
View File

@ -44,7 +44,7 @@ public void test() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

2
test/src/test/java/org/apache/sqoop/integration/connector/hdfs/FromHDFSToHDFSTest.java
View File

@ -51,7 +51,7 @@ public void test() throws Exception {
fillHdfsLink(hdfsLinkTo); fillHdfsLink(hdfsLinkTo);
saveLink(hdfsLinkTo); saveLink(hdfsLinkTo);
MJob job = getClient().createJob(hdfsLinkFrom.getPersistenceId(), hdfsLinkTo.getPersistenceId()); MJob job = getClient().createJob(hdfsLinkFrom.getName(), hdfsLinkTo.getName());
fillHdfsFromConfig(job); fillHdfsFromConfig(job);

2
test/src/test/java/org/apache/sqoop/integration/connector/hdfs/HdfsIncrementalReadTest.java
View File

@ -56,7 +56,7 @@ public void testBasic() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); MJob job = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
fillHdfsFromConfig(job); fillHdfsFromConfig(job);
job.getFromJobConfig().getEnumInput("incremental.incrementalType").setValue(IncrementalType.NEW_FILES); job.getFromJobConfig().getEnumInput("incremental.incrementalType").setValue(IncrementalType.NEW_FILES);
fillRdbmsToConfig(job); fillRdbmsToConfig(job);

6
test/src/test/java/org/apache/sqoop/integration/connector/hdfs/OutputDirectoryTest.java
View File

@ -54,7 +54,7 @@ public void testOutputDirectoryIsAFile() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -90,7 +90,7 @@ public void testOutputDirectoryIsNotEmpty() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -125,7 +125,7 @@ public void testOutputDirectoryIsEmpty() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

2
test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
View File

@ -106,7 +106,7 @@ public void createLinks() {
@Test @Test
public void testCities() throws Exception { public void testCities() throws Exception {
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), kiteLink.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

4
test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/AllTypesTest.java
View File

@ -94,7 +94,7 @@ public void testFrom() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Fill rdbms "FROM" config // Fill rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -139,7 +139,7 @@ public void testTo() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); MJob job = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
fillHdfsFromConfig(job); fillHdfsFromConfig(job);
// Set the rdbms "TO" config here // Set the rdbms "TO" config here

2
test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromHDFSToRDBMSTest.java
View File

@ -64,7 +64,7 @@ public void testBasic() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); MJob job = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
// set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs! // set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs!
fillHdfsFromConfig(job); fillHdfsFromConfig(job);

12
test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java
View File

@ -47,7 +47,7 @@ public void testCities() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -89,7 +89,7 @@ public void testStories() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// Connector values // Connector values
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -131,7 +131,7 @@ public void testColumns() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// Connector values // Connector values
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -173,7 +173,7 @@ public void testSql() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// Connector values // Connector values
MConfigList configs = job.getFromJobConfig(); MConfigList configs = job.getFromJobConfig();
@ -213,7 +213,7 @@ public void testDuplicateColumns() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// Connector values // Connector values
String partitionColumn = provider.escapeTableName(getTableName().getTableName()) + "." + provider.escapeColumnName("id"); String partitionColumn = provider.escapeTableName(getTableName().getTableName()) + "." + provider.escapeColumnName("id");
@ -263,7 +263,7 @@ public void testAllowNullsWithOneExtractor() throws Exception {
saveLink(hdfsConnection); saveLink(hdfsConnection);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), hdfsConnection.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

4
test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/IncrementalReadTest.java
View File

@ -90,7 +90,7 @@ public void testTable() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// Set the rdbms "FROM" config // Set the rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -142,7 +142,7 @@ public void testQuery() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
String query = "SELECT * FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}"; String query = "SELECT * FROM " + provider.escapeTableName(getTableName().getTableName()) + " WHERE ${CONDITIONS}";

2
test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/PartitionerTest.java
View File

@ -92,7 +92,7 @@ public void testSplitter() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// set the rdbms "FROM" config // set the rdbms "FROM" config
fillRdbmsFromConfig(job, partitionColumn); fillRdbmsFromConfig(job, partitionColumn);

4
test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableStagedRDBMSTest.java
View File

@ -56,8 +56,8 @@ public void testStagedTransfer() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), MJob job = getClient().createJob(hdfsLink.getName(),
rdbmsLink.getPersistenceId()); rdbmsLink.getName());
// fill HDFS "FROM" config // fill HDFS "FROM" config
fillHdfsFromConfig(job); fillHdfsFromConfig(job);

2
test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
View File

@ -49,7 +49,7 @@ public void testBasic() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(hdfsLink.getPersistenceId(), kafkaLink.getPersistenceId()); MJob job = getClient().createJob(hdfsLink.getName(), kafkaLink.getName());
// Job connector configs // Job connector configs
fillHdfsFromConfig(job); fillHdfsFromConfig(job);

2
test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
View File

@ -52,7 +52,7 @@ public void testBasic() throws Exception {
saveLink(rdbmsLink); saveLink(rdbmsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kafkaLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), kafkaLink.getName());
// set rdbms "FROM" job config // set rdbms "FROM" job config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

2
test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
View File

@ -67,7 +67,7 @@ public void testCities() throws Exception {
saveLink(kiteLink); saveLink(kiteLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), kiteLink.getName());
// Set rdbms "FROM" config // Set rdbms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

2
test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
View File

@ -175,7 +175,7 @@ private MJob prepareJob() {
MLink testConnection = getClient().createLink("test-connector"); MLink testConnection = getClient().createLink("test-connector");
saveLink(testConnection); saveLink(testConnection);
MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), testConnection.getPersistenceId()); MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName());
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

9
test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
View File

@ -69,8 +69,8 @@ public Map<Integer, Integer> getNumberOfSubmissions() {
} }
@Override @Override
public Integer[] getDisabledLinkIds() { public String[] getDisabledLinkNames() {
return new Integer[] {4}; return new String[] {linkIdToNameMap.get(4L)};
} }
@Override @Override
@ -79,8 +79,9 @@ public String[] getDisabledJobNames() {
} }
@Override @Override
public Integer[] getDeleteLinkIds() { public String[] getDeleteLinkNames() {
return new Integer[] {1, 2, 3, 4, 5}; return new String[] {linkIdToNameMap.get(1L), linkIdToNameMap.get(2L),
linkIdToNameMap.get(3L), linkIdToNameMap.get(4L), linkIdToNameMap.get(5L)};
} }
@Test @Test

9
test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
View File

@ -71,8 +71,8 @@ public Map<Integer, Integer> getNumberOfSubmissions() {
} }
@Override @Override
public Integer[] getDisabledLinkIds() { public String[] getDisabledLinkNames() {
return new Integer[] {4, 5}; return new String[] {linkIdToNameMap.get(4L), linkIdToNameMap.get(5L)};
} }
@Override @Override
@ -81,7 +81,8 @@ public String[] getDisabledJobNames() {
} }
@Override @Override
public Integer[] getDeleteLinkIds() { public String[] getDeleteLinkNames() {
return new Integer[] {1, 2, 3, 4, 5, 6}; return new String[] {linkIdToNameMap.get(1L), linkIdToNameMap.get(2L),
linkIdToNameMap.get(3L), linkIdToNameMap.get(4L), linkIdToNameMap.get(5L), linkIdToNameMap.get(6L)};
} }
} }

11
test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
View File

@ -80,8 +80,8 @@ public Map<Integer, Integer> getNumberOfSubmissions() {
} }
@Override @Override
public Integer[] getDisabledLinkIds() { public String[] getDisabledLinkNames() {
return new Integer[] {4, 5}; return new String[] {linkIdToNameMap.get(4L), linkIdToNameMap.get(5L)};
} }
@Override @Override
@ -90,8 +90,9 @@ public String[] getDisabledJobNames() {
} }
@Override @Override
public Integer[] getDeleteLinkIds() { public String[] getDeleteLinkNames() {
return new Integer[] {1, 2, 3, 4, 5, 6}; return new String[] {linkIdToNameMap.get(1L), linkIdToNameMap.get(2L),
linkIdToNameMap.get(3L), linkIdToNameMap.get(4L), linkIdToNameMap.get(5L), linkIdToNameMap.get(6L)};
} }
@Test @Test
@ -101,7 +102,7 @@ public void testJobNameNotNull() {
assertNotNull(job.getName()); assertNotNull(job.getName());
} }
MJob job = getClient().createJob(1, 1); MJob job = getClient().createJob(linkIdToNameMap.get(1L), linkIdToNameMap.get(1L));
assertNull(job.getName()); assertNull(job.getName());
assertEquals(getClient().saveJob(job), Status.ERROR); assertEquals(getClient().saveJob(job), Status.ERROR);
} }

11
test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
View File

@ -81,8 +81,8 @@ public Map<Integer, Integer> getNumberOfSubmissions() {
} }
@Override @Override
public Integer[] getDisabledLinkIds() { public String[] getDisabledLinkNames() {
return new Integer[] {4, 5}; return new String[] {linkIdToNameMap.get(4L), linkIdToNameMap.get(5L)};
} }
@Override @Override
@ -91,8 +91,9 @@ public String[] getDisabledJobNames() {
} }
@Override @Override
public Integer[] getDeleteLinkIds() { public String[] getDeleteLinkNames() {
return new Integer[] {1, 2, 3, 4, 5, 6}; return new String[] {linkIdToNameMap.get(1L), linkIdToNameMap.get(2L),
linkIdToNameMap.get(3L), linkIdToNameMap.get(4L), linkIdToNameMap.get(5L), linkIdToNameMap.get(6L)};
} }
@Test @Test
@ -102,7 +103,7 @@ public void testJobNameNotNull() {
assertNotNull(job.getName()); assertNotNull(job.getName());
} }
MJob job = getClient().createJob(1, 1); MJob job = getClient().createJob(linkIdToNameMap.get(1L), linkIdToNameMap.get(1L));
assertNull(job.getName()); assertNull(job.getName());
assertEquals(getClient().saveJob(job), Status.ERROR); assertEquals(getClient().saveJob(job), Status.ERROR);
} }

19
test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/DerbyRepositoryUpgradeTest.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster; import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
import org.apache.sqoop.test.testcases.JettyTestCase; import org.apache.sqoop.test.testcases.JettyTestCase;
import org.apache.sqoop.test.utils.CompressionUtils; import org.apache.sqoop.test.utils.CompressionUtils;
@ -56,6 +57,7 @@ public abstract class DerbyRepositoryUpgradeTest extends JettyTestCase {
private static final Logger LOG = Logger.getLogger(DerbyRepositoryUpgradeTest.class); private static final Logger LOG = Logger.getLogger(DerbyRepositoryUpgradeTest.class);
protected Map<Long, String> jobIdToNameMap; protected Map<Long, String> jobIdToNameMap;
protected Map<Long, String> linkIdToNameMap;
/** /**
* Custom Sqoop mini cluster that points derby repository to real on-disk structures. * Custom Sqoop mini cluster that points derby repository to real on-disk structures.
@ -109,7 +111,7 @@ protected Map<String, String> getRepositoryConfiguration() {
/** /**
* List of link ids that should be disabled * List of link ids that should be disabled
*/ */
public abstract Integer[] getDisabledLinkIds(); public abstract String[] getDisabledLinkNames();
/** /**
* List of job ids that should be disabled * List of job ids that should be disabled
@ -119,7 +121,7 @@ protected Map<String, String> getRepositoryConfiguration() {
/** /**
* List of link ids that we should delete using the id * List of link ids that we should delete using the id
*/ */
public abstract Integer[] getDeleteLinkIds(); public abstract String[] getDeleteLinkNames();
public String getRepositoryPath() { public String getRepositoryPath() {
return HdfsUtils.joinPathFragments(getTemporaryJettyPath(), "repo"); return HdfsUtils.joinPathFragments(getTemporaryJettyPath(), "repo");
@ -159,6 +161,11 @@ public void startSqoopMiniCluster(ITestContext context) throws Exception {
for(MJob job : getClient().getJobs()) { for(MJob job : getClient().getJobs()) {
jobIdToNameMap.put(job.getPersistenceId(), job.getName()); jobIdToNameMap.put(job.getPersistenceId(), job.getName());
} }
linkIdToNameMap = new HashMap<Long, String>();
for(MLink link : getClient().getLinks()) {
linkIdToNameMap.put(link.getPersistenceId(), link.getName());
}
} }
@AfterMethod @AfterMethod
@ -185,8 +192,8 @@ public void testPostUpgrade() throws Exception {
} }
// Verify that disabled status is preserved // Verify that disabled status is preserved
for(Integer id : getDisabledLinkIds()) { for(String linkName : getDisabledLinkNames()) {
assertFalse(getClient().getLink(id).getEnabled()); assertFalse(getClient().getLink(linkName).getEnabled());
} }
for(String name : getDisabledJobNames()) { for(String name : getDisabledJobNames()) {
assertFalse(getClient().getJob(name).getEnabled()); assertFalse(getClient().getJob(name).getEnabled());
@ -196,8 +203,8 @@ public void testPostUpgrade() throws Exception {
for(String name : jobIdToNameMap.values()) { for(String name : jobIdToNameMap.values()) {
getClient().deleteJob(name); getClient().deleteJob(name);
} }
for(Integer id : getDeleteLinkIds()) { for(String linkName : getDeleteLinkNames()) {
getClient().deleteLink(id); getClient().deleteLink(linkName);
} }
// We should end up with empty repository // We should end up with empty repository

4
test/src/test/java/org/apache/sqoop/integration/server/InformalObjectNameTest.java
View File

@ -69,7 +69,7 @@ public void testInformalJobName() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// rdms "FROM" config // rdms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -81,7 +81,7 @@ public void testInformalJobName() throws Exception {
saveJob(job); saveJob(job);
assertEquals(job, getClient().getJob(JOB_NAME_CONTAINS_WHITESPACE)); assertEquals(job, getClient().getJob(JOB_NAME_CONTAINS_WHITESPACE));
job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// rdms "FROM" config // rdms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");

4
test/src/test/java/org/apache/sqoop/integration/server/InvalidRESTCallsTest.java
View File

@ -216,8 +216,8 @@ void validate() throws Exception {
@Override @Override
void validate() throws Exception { void validate() throws Exception {
assertResponseCode(500); assertResponseCode(500);
assertServerException("org.apache.sqoop.server.common.ServerError", "SERVER_0005"); assertServerException("org.apache.sqoop.server.common.ServerError", "SERVER_0006");
assertContains("Invalid link: i-dont-exists"); assertContains("Invalid link name: i-dont-exists");
}}), }}),
new TestDescription("Get links for non existing connector", "v1/link/all?cname=i-dont-exists", "GET", null, new Validator() { new TestDescription("Get links for non existing connector", "v1/link/all?cname=i-dont-exists", "GET", null, new Validator() {
@Override @Override

8
test/src/test/java/org/apache/sqoop/integration/server/ShowJobInOrderTest.java
View File

@ -55,7 +55,7 @@ public void testShowJobInOrder() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// rdms "FROM" config // rdms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -66,7 +66,7 @@ public void testShowJobInOrder() throws Exception {
saveJob(job); saveJob(job);
// Job creation // Job creation
job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); job = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
// rdms "To" config // rdms "To" config
fillRdbmsToConfig(job); fillRdbmsToConfig(job);
@ -77,7 +77,7 @@ public void testShowJobInOrder() throws Exception {
saveJob(job); saveJob(job);
// Job creation // Job creation
job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
// rdms "FROM" config // rdms "FROM" config
fillRdbmsFromConfig(job, "id"); fillRdbmsFromConfig(job, "id");
@ -88,7 +88,7 @@ public void testShowJobInOrder() throws Exception {
saveJob(job); saveJob(job);
// Job creation // Job creation
job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId()); job = getClient().createJob(hdfsLink.getName(), rdbmsLink.getName());
// hdfs "From" config // hdfs "From" config

4
test/src/test/java/org/apache/sqoop/integration/server/SubmissionWithDisabledModelObjectsTest.java
View File

@ -89,7 +89,7 @@ public void testWithDisabledObjects() throws Exception {
saveLink(hdfsLink); saveLink(hdfsLink);
// Job creation // Job creation
MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); MJob job = getClient().createJob(rdbmsLink.getName(), hdfsLink.getName());
job.setName(jobName); job.setName(jobName);
// rdms "FROM" config // rdms "FROM" config
@ -101,7 +101,7 @@ public void testWithDisabledObjects() throws Exception {
saveJob(job); saveJob(job);
// Disable model entities as per parameterized run // Disable model entities as per parameterized run
getClient().enableLink(rdbmsLink.getPersistenceId(), enabledLink); getClient().enableLink(rdbmsLink.getName(), enabledLink);
getClient().enableJob(jobName, enabledJob); getClient().enableJob(jobName, enabledJob);
// Try to execute the job and verify that the it was not executed // Try to execute the job and verify that the it was not executed