diff --git a/client/pom.xml b/client/pom.xml index 0d144609..c6351ed8 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -32,6 +32,11 @@ limitations under the License. Sqoop Client + + org.mockito + mockito-all + test + org.apache.sqoop sqoop-common diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java index 232ef20f..f9137bb2 100644 --- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java +++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java @@ -18,6 +18,8 @@ package org.apache.sqoop.client; import org.apache.sqoop.client.request.SqoopRequests; +import org.apache.sqoop.json.ConnectorBean; +import org.apache.sqoop.json.FrameworkBean; import org.apache.sqoop.json.ValidationBean; import org.apache.sqoop.model.FormUtils; import org.apache.sqoop.model.MConnection; @@ -28,30 +30,90 @@ import org.apache.sqoop.validation.Status; import org.apache.sqoop.validation.Validation; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.ResourceBundle; /** * Sqoop client API. * - * High level Sqoop client API to communicate with Sqoop server. + * High level Sqoop client API to communicate with Sqoop server. Current + * implementation is not thread safe. + * + * SqoopClient is keeping cache of objects that are unlikely to be changed + * (Resources, Connector structures). Volatile structures (Connections, Jobs) + * are not cached. */ public class SqoopClient { + /** + * Underlying request object to fetch data from Sqoop server. + */ private SqoopRequests requests; + /** + * True if user retrieved all connectors at once. + */ + private boolean allConnectors; + + /** + * All cached bundles for all connectors. + */ + private Map bundles; + + /** + * Cached framework bundle. + */ + private ResourceBundle frameworkBundle; + + /** + * All cached connectors. + */ + private Map connectors; + + /** + * Cached framework. + */ + private MFramework framework; + public SqoopClient(String serverUrl) { requests = new SqoopRequests(); - requests.setServerUrl(serverUrl); + setServerUrl(serverUrl); } /** * Set new server URL. * + * Setting new URL will also clear all caches used by the client. + * * @param serverUrl Server URL */ public void setServerUrl(String serverUrl) { requests.setServerUrl(serverUrl); + clearCache(); + } + + /** + * Set arbitrary request object. + * + * @param requests SqoopRequests object + */ + public void setSqoopRequests(SqoopRequests requests) { + this.requests = requests; + clearCache(); + } + + /** + * Clear internal cache. + */ + public void clearCache() { + bundles = new HashMap(); + frameworkBundle = null; + connectors = new HashMap(); + framework = null; + allConnectors = false; } /** @@ -61,7 +123,23 @@ public void setServerUrl(String serverUrl) { * @return */ public MConnector getConnector(long cid) { - return requests.readConnector(cid).getConnectors().get(0); + if(connectors.containsKey(cid)) { + return connectors.get(cid); + } + + retrieveConnector(cid); + return connectors.get(cid); + } + + /** + * Retrieve connector structure from server and cache it. + * + * @param cid Connector id + */ + private void retrieveConnector(long cid) { + ConnectorBean request = requests.readConnector(cid); + connectors.put(cid, request.getConnectors().get(0)); + bundles.put(cid, request.getResourceBundles().get(cid)); } /** @@ -69,18 +147,34 @@ public MConnector getConnector(long cid) { * * @return */ - public List getConnectors() { - return requests.readConnector(null).getConnectors(); + public Collection getConnectors() { + if(allConnectors) { + return connectors.values(); + } + + ConnectorBean bean = requests.readConnector(null); + allConnectors = true; + for(MConnector connector : bean.getConnectors()) { + connectors.put(connector.getPersistenceId(), connector); + } + bundles = bean.getResourceBundles(); + + return connectors.values(); } /** - * Get resouce bundle for given connector. + * Get resource bundle for given connector. * * @param cid Connector id. * @return */ public ResourceBundle getResourceBundle(long cid) { - return requests.readConnector(cid).getResourceBundles().get(cid); + if(bundles.containsKey(cid)) { + return bundles.get(cid); + } + + retrieveConnector(cid); + return bundles.get(cid); } /** @@ -89,7 +183,22 @@ public ResourceBundle getResourceBundle(long cid) { * @return */ public MFramework getFramework() { - return requests.readFramework().getFramework(); + if(framework != null) { + return framework; + } + + retrieveFramework(); + return framework; + + } + + /** + * Retrieve framework structure and cache it. + */ + private void retrieveFramework() { + FrameworkBean request = requests.readFramework(); + framework = request.getFramework(); + frameworkBundle = request.getResourceBundle(); } /** @@ -98,7 +207,12 @@ public MFramework getFramework() { * @return */ public ResourceBundle getFrameworkResourceBundle() { - return requests.readFramework().getResourceBundle(); + if(frameworkBundle != null) { + return frameworkBundle; + } + + retrieveFramework(); + return frameworkBundle; } /** diff --git a/client/src/main/java/org/apache/sqoop/client/shell/ShowConnectorFunction.java b/client/src/main/java/org/apache/sqoop/client/shell/ShowConnectorFunction.java index 19a81239..b053339b 100644 --- a/client/src/main/java/org/apache/sqoop/client/shell/ShowConnectorFunction.java +++ b/client/src/main/java/org/apache/sqoop/client/shell/ShowConnectorFunction.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.client.shell; +import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -56,7 +57,7 @@ public Object executeFunction(CommandLine line) { } private void showSummary() { - List connectors = client.getConnectors(); + Collection connectors = client.getConnectors(); List header = new LinkedList(); header.add(resourceString(Constants.RES_TABLE_HEADER_ID)); @@ -80,7 +81,7 @@ private void showSummary() { } private void showConnectors() { - List connectors = client.getConnectors(); + Collection connectors = client.getConnectors(); printlnResource(Constants.RES_SHOW_PROMPT_CONNECTORS_TO_SHOW, connectors.size()); diff --git a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java new file mode 100644 index 00000000..1778cf1e --- /dev/null +++ b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.client; + +import org.apache.sqoop.client.request.SqoopRequests; +import org.apache.sqoop.json.ConnectorBean; +import org.apache.sqoop.json.FrameworkBean; +import org.apache.sqoop.model.MConnectionForms; +import org.apache.sqoop.model.MConnector; +import org.apache.sqoop.model.MFramework; +import org.apache.sqoop.model.MJobForms; +import org.apache.sqoop.utils.MapResourceBundle; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.ResourceBundle; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.*; + +public class TestSqoopClient { + + SqoopRequests requests; + SqoopClient client; + + @Before + public void setUp() { + requests = mock(SqoopRequests.class); + client = new SqoopClient("my-cool-server"); + client.setSqoopRequests(requests); + } + + /** + * Retrieve connector information, request to bundle for same connector should + * not require additional HTTP request. + */ + @Test + public void testGetConnector() { + when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1))); + MConnector connector = client.getConnector(1); + assertEquals(1, connector.getPersistenceId()); + + client.getResourceBundle(1L); + + verify(requests, times(1)).readConnector(1L); + } + + /** + * Retrieve connector bundle, request for metadata for same connector should + * not require additional HTTP request. + */ + @Test + public void testGetConnectorBundle() { + when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1))); + client.getResourceBundle(1L); + + MConnector connector = client.getConnector(1); + assertEquals(1, connector.getPersistenceId()); + + verify(requests, times(1)).readConnector(1L); + } + + /** + * Retrieve framework information, request to framework bundle should not + * require additional HTTP request. + */ + @Test + public void testGetFramework() { + when(requests.readFramework()).thenReturn(frameworkBean(framework())); + + client.getFramework(); + client.getFrameworkResourceBundle(); + + verify(requests, times(1)).readFramework(); + } + + /** + * Retrieve framework bundle, request to framework metadata should not + * require additional HTTP request. + */ + @Test + public void testGetFrameworkBundle() { + when(requests.readFramework()).thenReturn(frameworkBean(framework())); + + client.getFrameworkResourceBundle(); + client.getFramework(); + + verify(requests, times(1)).readFramework(); + } + + /** + * Getting all connectors at once should avoid any other HTTP request to + * specific connectors. + */ + @Test + public void testGetConnectors() { + MConnector connector; + + when(requests.readConnector(null)).thenReturn(connectorBean(connector(1), connector(2))); + Collection connectors = client.getConnectors(); + assertEquals(2, connectors.size()); + + client.getResourceBundle(1); + connector = client.getConnector(1); + assertEquals(1, connector.getPersistenceId()); + + connector = client.getConnector(2); + client.getResourceBundle(2); + assertEquals(2, connector.getPersistenceId()); + + connectors = client.getConnectors(); + assertEquals(2, connectors.size()); + + verify(requests, times(1)).readConnector(null); + verifyNoMoreInteractions(requests); + } + + + /** + * Getting connectors one by one should not be equivalent to getting all connectors + * at once as Client do not know how many connectors server have. + */ + @Test + public void testGetConnectorOneByOne() { + ConnectorBean bean = connectorBean(connector(1), connector(2)); + when(requests.readConnector(null)).thenReturn(bean); + when(requests.readConnector(1L)).thenReturn(bean); + when(requests.readConnector(2L)).thenReturn(bean); + + client.getResourceBundle(1); + client.getConnector(1); + + client.getConnector(2); + client.getResourceBundle(2); + + Collection connectors = client.getConnectors(); + assertEquals(2, connectors.size()); + + verify(requests, times(1)).readConnector(null); + verify(requests, times(1)).readConnector(1L); + verify(requests, times(1)).readConnector(2L); + verifyNoMoreInteractions(requests); + } + + private ConnectorBean connectorBean(MConnector...connectors) { + List connectorList = new ArrayList(); + Map bundles = new HashMap(); + + for(MConnector connector : connectors) { + connectorList.add(connector); + bundles.put(connector.getPersistenceId(), null); + } + return new ConnectorBean(connectorList, bundles); + } + private FrameworkBean frameworkBean(MFramework framework) { + return new FrameworkBean(framework, new MapResourceBundle(null)); + } + + private MConnector connector(long id) { + MConnector connector = new MConnector("A" + id, "A" + id, "1.0" + id, new MConnectionForms(null), new LinkedList()); + connector.setPersistenceId(id); + return connector; + } + + private MFramework framework() { + MFramework framework = new MFramework(new MConnectionForms(null), new LinkedList()); + framework.setPersistenceId(1); + return framework; + } +} diff --git a/pom.xml b/pom.xml index 25dfba6c..0abbb184 100644 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,7 @@ limitations under the License. 11.0.2 1.1 4.9 + 1.9.5 1.2.16 2.5 1.3.2 @@ -374,6 +375,12 @@ limitations under the License. sqljdbc4 ${jdbc.sqlserver.version} + + org.mockito + mockito-all + ${mockito.version} + test +