github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-10 22:13:07 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2769: Sqoop2: Remove the notion of JobsBeans

Browse Source 
(Jarek Jarcec Cecho via Colin Ma)
This commit is contained in:
Colin Ma 2016-01-15 11:29:35 +08:00
parent 63002273d0
commit f24e76050b
9 changed files with 82 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java
View File

@ -20,7 +20,6 @@
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.sqoop.json.JSONUtils;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.JobsBean;
import org.apache.sqoop.json.SubmissionBean;
import org.apache.sqoop.json.ValidationResultBean;
import org.apache.sqoop.model.MJob;
@ -50,7 +49,7 @@ public JobResourceRequest(DelegationTokenAuthenticatedURL.Token token){
}
public JobBean readByConnector(String serverUrl, String cArg) {
JobsBean bean = new JobsBean();
JobBean bean = new JobBean();
if (cArg != null) {
String response = super.get(serverUrl + RESOURCE + "?cname=" + UrlSafeUtils.urlEncode(cArg));
JSONObject jsonObject = JSONUtils.parse(response);
@ -68,7 +67,7 @@ public JobBean read(String serverUrl, String jobArg) {
}
JSONObject jsonObject = JSONUtils.parse(response);
// defaults to all
JobBean bean = new JobsBean();
JobBean bean = new JobBean();
if (jobArg != null) {
bean = new JobBean();
}

28
common/src/main/java/org/apache/sqoop/json/JobBean.java
View File

@ -55,7 +55,7 @@ public class JobBean implements JsonBean {
static final String FROM_CONFIG_VALUES = "from-config-values";
static final String TO_CONFIG_VALUES = "to-config-values";
static final String DRIVER_CONFIG_VALUES = "driver-config-values";
private static final String JOB = "job";
private static final String JOBS = "jobs";
// Required
private List<MJob> jobs;
@ -67,7 +67,7 @@ public class JobBean implements JsonBean {
// For "extract"
public JobBean(MJob job) {
this();
this.jobs = new ArrayList<MJob>();
this.jobs = new ArrayList<>();
this.jobs.add(job);
}
@ -78,7 +78,7 @@ public JobBean(List<MJob> jobs) {
// For "restore"
public JobBean() {
connectorConfigBundles = new HashMap<String, ResourceBundle>();
connectorConfigBundles = new HashMap<>();
}
public void setDriverConfigBundle(ResourceBundle driverConfigBundle) {
@ -108,12 +108,19 @@ public ResourceBundle getDriverConfigBundle() {
@Override
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONObject job = new JSONObject();
job.put(JOB, extractJob(skipSensitive, jobs.get(0)));
return job;
JSONArray jobArray = extractJobs(skipSensitive);
JSONObject jobs = new JSONObject();
jobs.put(JOBS, jobArray);
return jobs;
}
@Override
@SuppressWarnings("unchecked")
public void restore(JSONObject jsonObject) {
JSONArray array = JSONUtils.getJSONArray(jsonObject, JOBS);
restoreJobs(array);
}
protected JSONArray extractJobs(boolean skipSensitive) {
JSONArray jobArray = new JSONArray();
for (MJob job : jobs) {
@ -152,15 +159,8 @@ private JSONObject extractJob(boolean skipSensitive, MJob job) {
return object;
}
@Override
public void restore(JSONObject jsonObject) {
jobs = new ArrayList<MJob>();
JSONObject obj = JSONUtils.getJSONObject(jsonObject, JOB);
jobs.add(restoreJob(obj));
}
protected void restoreJobs(JSONArray array) {
jobs = new ArrayList<MJob>();
jobs = new ArrayList<>();
for (Object obj : array) {
jobs.add(restoreJob(obj));
}

63
common/src/main/java/org/apache/sqoop/json/JobsBean.java
View File

@ -1,63 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import java.util.List;
import org.apache.sqoop.classification.InterfaceAudience;
import org.apache.sqoop.classification.InterfaceStability;
import org.apache.sqoop.model.MJob;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
/**
* Json representation of the jobs
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class JobsBean extends JobBean {
private static final String JOBS = "jobs";
public JobsBean(MJob job) {
super(job);
}
public JobsBean(List<MJob> jobs) {
super(jobs);
}
// For "restore"
public JobsBean() {
}
@Override
@SuppressWarnings("unchecked")
public JSONObject extract(boolean skipSensitive) {
JSONArray jobArray = super.extractJobs(skipSensitive);
JSONObject jobs = new JSONObject();
jobs.put(JOBS, jobArray);
return jobs;
}
@Override
public void restore(JSONObject jsonObject) {
JSONArray array = JSONUtils.getJSONArray(jsonObject, JOBS);
restoreJobs(array);
}
}

58
common/src/test/java/org/apache/sqoop/json/TestJobBean.java
View File

@ -19,7 +19,9 @@
import static org.testng.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.json.util.BeanTestUtil;
@ -78,5 +80,61 @@ public void testJobSerialization() throws ParseException {
assertEquals("Hi there again!", targetInput.getValue());
}
@Test
public void testJobsSerialization() throws ParseException {
Date created = new Date();
Date updated = new Date();
MJob job1 = BeanTestUtil.createJob("ahoj", "The big Job", 22L, created, updated);
MJob job2 = BeanTestUtil.createJob("ahoj", "The small Job", 44L, created, updated);
List<MJob> jobs = new ArrayList<>();
jobs.add(job1);
jobs.add(job2);
// Fill some data at the beginning
MStringInput input = (MStringInput) job1.getFromJobConfig().getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
input = (MStringInput) job1.getToJobConfig().getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
JobBean jobsBean = new JobBean(jobs);
JSONObject json = jobsBean.extract(false);
// "Move" it across network in text form
String jobJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedJobsJson = JSONUtils.parse(jobJsonString);
JobBean parsedJobsBean = new JobBean();
parsedJobsBean.restore(parsedJobsJson);
assertEquals(parsedJobsBean.getJobs().size(), 2);
MJob retrievedJob1 = parsedJobsBean.getJobs().get(0);
MJob retrievedJob2 = parsedJobsBean.getJobs().get(1);
// Check id and name
assertEquals(22L, retrievedJob1.getPersistenceId());
assertEquals("The big Job", retrievedJob1.getName());
assertEquals(44L, retrievedJob2.getPersistenceId());
assertEquals("The small Job", retrievedJob2.getName());
assertEquals(retrievedJob1.getFromLinkName(), "fromLinkName");
assertEquals(retrievedJob1.getToLinkName(), "toLinkName");
assertEquals(retrievedJob1.getFromConnectorName(), "from_ahoj");
assertEquals(retrievedJob1.getToConnectorName(), "to_ahoj");
assertEquals(created, retrievedJob1.getCreationDate());
assertEquals(updated, retrievedJob1.getLastUpdateDate());
assertEquals(false, retrievedJob1.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) retrievedJob1.getFromJobConfig()
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) retrievedJob1.getToJobConfig().getConfigs().get(0)
.getInputs().get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}

91
common/src/test/java/org/apache/sqoop/json/TestJobsBean.java
View File

@ -1,91 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.sqoop.json;
import static org.testng.Assert.assertEquals;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.sqoop.common.Direction;
import org.apache.sqoop.json.util.BeanTestUtil;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MStringInput;
import org.json.simple.JSONObject;
import org.json.simple.parser.ParseException;
import org.testng.annotations.Test;
public class TestJobsBean {
@Test
public void testJobsSerialization() throws ParseException {
Date created = new Date();
Date updated = new Date();
MJob job1 = BeanTestUtil.createJob("ahoj", "The big Job", 22L, created, updated);
MJob job2 = BeanTestUtil.createJob("ahoj", "The small Job", 44L, created, updated);
List<MJob> jobs = new ArrayList<MJob>();
jobs.add(job1);
jobs.add(job2);
// Fill some data at the beginning
MStringInput input = (MStringInput) job1.getFromJobConfig().getConfigs().get(0)
.getInputs().get(0);
input.setValue("Hi there!");
input = (MStringInput) job1.getToJobConfig().getConfigs().get(0).getInputs().get(0);
input.setValue("Hi there again!");
// Serialize it to JSON object
JobsBean jobsBean = new JobsBean(jobs);
JSONObject json = jobsBean.extract(false);
// "Move" it across network in text form
String jobJsonString = json.toJSONString();
// Retrieved transferred object
JSONObject parsedJobsJson = JSONUtils.parse(jobJsonString);
JobsBean parsedJobsBean = new JobsBean();
parsedJobsBean.restore(parsedJobsJson);
MJob retrievedJob1 = parsedJobsBean.getJobs().get(0);
MJob retrievedJob2 = parsedJobsBean.getJobs().get(1);
// Check id and name
assertEquals(22L, retrievedJob1.getPersistenceId());
assertEquals("The big Job", retrievedJob1.getName());
assertEquals(44L, retrievedJob2.getPersistenceId());
assertEquals("The small Job", retrievedJob2.getName());
assertEquals(retrievedJob1.getFromLinkName(), "fromLinkName");
assertEquals(retrievedJob1.getToLinkName(), "toLinkName");
assertEquals(retrievedJob1.getFromConnectorName(), "from_ahoj");
assertEquals(retrievedJob1.getToConnectorName(), "to_ahoj");
assertEquals(created, retrievedJob1.getCreationDate());
assertEquals(updated, retrievedJob1.getLastUpdateDate());
assertEquals(false, retrievedJob1.getEnabled());
// Test that value was correctly moved
MStringInput targetInput = (MStringInput) retrievedJob1.getFromJobConfig()
.getConfigs().get(0).getInputs().get(0);
assertEquals("Hi there!", targetInput.getValue());
targetInput = (MStringInput) retrievedJob1.getToJobConfig().getConfigs().get(0)
.getInputs().get(0);
assertEquals("Hi there again!", targetInput.getValue());
}
}

11
server/src/main/java/org/apache/sqoop/handler/JobRequestHandler.java
View File

@ -33,7 +33,6 @@
import org.apache.sqoop.driver.JobManager;
import org.apache.sqoop.json.JSONUtils;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.JobsBean;
import org.apache.sqoop.json.JsonBean;
import org.apache.sqoop.json.SubmissionBean;
import org.apache.sqoop.json.ValidationResultBean;
@ -279,7 +278,7 @@ private JsonBean getJobs(RequestContext ctx) {
// Authorization check
jobList = AuthorizationEngine.filterResource(ctx.getUserName(), MResource.TYPE.JOB, jobList);
jobBean = createJobsBean(jobList, locale);
jobBean = createJobBean(jobList, locale);
} else
// all jobs in the system
if (ctx.getPath().contains(JOBS_PATH)
@ -291,7 +290,7 @@ private JsonBean getJobs(RequestContext ctx) {
// Authorization check
jobList = AuthorizationEngine.filterResource(ctx.getUserName(), MResource.TYPE.JOB, jobList);
jobBean = createJobsBean(jobList, locale);
jobBean = createJobBean(jobList, locale);
}
// job by Id
else {
@ -315,12 +314,6 @@ private JobBean createJobBean(List<MJob> jobs, Locale locale) {
return jobBean;
}
private JobsBean createJobsBean(List<MJob> jobs, Locale locale) {
JobsBean jobsBean = new JobsBean(jobs);
addConnectorConfigBundle(jobsBean, locale);
return jobsBean;
}
private void addConnectorConfigBundle(JobBean bean, Locale locale) {
// Add associated resources into the bean
for (MJob job : bean.getJobs()) {

4
test/src/test/java/org/apache/sqoop/integration/tools/RepositoryDumpLoadToolTest.java
View File

@ -21,7 +21,7 @@
import org.apache.commons.io.IOUtils;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.json.JSONUtils;
import org.apache.sqoop.json.JobsBean;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.SubmissionsBean;
import org.apache.sqoop.model.*;
@ -75,7 +75,7 @@ public void testDump() throws Exception {
// verify the job
JSONObject jsonJobs = (JSONObject) json.get(JSONConstants.JOBS);
JobsBean jobsBean = new JobsBean();
JobBean jobsBean = new JobBean();
jobsBean.restore(jsonJobs);
verifyJobs(jobsBean.getJobs());

4
tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
View File

@ -33,7 +33,7 @@
import org.apache.sqoop.cli.SqoopGnuParser;
import org.apache.sqoop.common.VersionInfo;
import org.apache.sqoop.connector.ConnectorManager;
import org.apache.sqoop.json.JobsBean;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.SubmissionsBean;
import org.apache.sqoop.model.MLink;
@ -112,7 +112,7 @@ private JSONObject dump(boolean skipSensitive) {
result.put(JSONConstants.LINKS, linksJsonObject);
LOG.info("Dumping Jobs with skipSensitive=" + String.valueOf(skipSensitive));
JobsBean jobs = new JobsBean(repository.findJobs());
JobBean jobs = new JobBean(repository.findJobs());
JSONObject jobsJsonObject = jobs.extract(skipSensitive);
result.put(JSONConstants.JOBS, jobsJsonObject);

4
tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
View File

@ -41,7 +41,7 @@
import org.apache.sqoop.driver.Driver;
import org.apache.sqoop.driver.DriverUpgrader;
import org.apache.sqoop.json.JSONUtils;
import org.apache.sqoop.json.JobsBean;
import org.apache.sqoop.json.JobBean;
import org.apache.sqoop.json.LinkBean;
import org.apache.sqoop.json.SubmissionsBean;
import org.apache.sqoop.model.ConfigUtils;
@ -181,7 +181,7 @@ private boolean load(JSONObject repo) {
removeJobIfLinkNotExist((JSONArray) jsonJobs.get(JSONConstants.JOBS),
JSONConstants.TO_LINK_NAME);
JobsBean jobsBean = new JobsBean();
JobBean jobsBean = new JobBean();
jobsBean.restore(jsonJobs);
for (MJob job : jobsBean.getJobs()) {