SQOOP-647: Provide facility to cache server responses on client side

(Jarek Jarcec Cecho via Kate Ting)

client/pom.xml

@@ -32,6 +32,11 @@ limitations under the License.
   <name>Sqoop Client</name>
 
   <dependencies>
+     <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.sqoop</groupId>
       <artifactId>sqoop-common</artifactId>

client/src/main/java/org/apache/sqoop/client/SqoopClient.java

@@ -18,6 +18,8 @@
 package org.apache.sqoop.client;
 
 import org.apache.sqoop.client.request.SqoopRequests;
+import org.apache.sqoop.json.ConnectorBean;
+import org.apache.sqoop.json.FrameworkBean;
 import org.apache.sqoop.json.ValidationBean;
 import org.apache.sqoop.model.FormUtils;
 import org.apache.sqoop.model.MConnection;
@@ -28,30 +30,90 @@
 import org.apache.sqoop.validation.Status;
 import org.apache.sqoop.validation.Validation;
 
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.ResourceBundle;
 
 /**
  * Sqoop client API.
  *
- * High level Sqoop client API to communicate with Sqoop server.
+ * High level Sqoop client API to communicate with Sqoop server. Current
+ * implementation is not thread safe.
+ *
+ * SqoopClient is keeping cache of objects that are unlikely to be changed
+ * (Resources, Connector structures). Volatile structures (Connections, Jobs)
+ * are not cached.
  */
 public class SqoopClient {
 
+  /**
+   * Underlying request object to fetch data from Sqoop server.
+   */
   private SqoopRequests requests;
 
+  /**
+   * True if user retrieved all connectors at once.
+   */
+  private boolean allConnectors;
+
+  /**
+   * All cached bundles for all connectors.
+   */
+  private Map<Long, ResourceBundle> bundles;
+
+  /**
+   * Cached framework bundle.
+   */
+  private ResourceBundle frameworkBundle;
+
+  /**
+   * All cached connectors.
+   */
+  private Map<Long, MConnector> connectors;
+
+  /**
+   * Cached framework.
+   */
+  private MFramework framework;
+
   public SqoopClient(String serverUrl) {
     requests = new SqoopRequests();
-    requests.setServerUrl(serverUrl);
+    setServerUrl(serverUrl);
   }
 
   /**
    * Set new server URL.
    *
+   * Setting new URL will also clear all caches used by the client.
+   *
    * @param serverUrl Server URL
    */
   public void setServerUrl(String serverUrl) {
     requests.setServerUrl(serverUrl);
+    clearCache();
+  }
+
+  /**
+   * Set arbitrary request object.
+   *
+   * @param requests SqoopRequests object
+   */
+  public void setSqoopRequests(SqoopRequests requests) {
+    this.requests = requests;
+    clearCache();
+  }
+
+  /**
+   * Clear internal cache.
+   */
+  public void clearCache() {
+    bundles = new HashMap<Long, ResourceBundle>();
+    frameworkBundle = null;
+    connectors = new HashMap<Long, MConnector>();
+    framework = null;
+    allConnectors = false;
   }
 
   /**
@@ -61,7 +123,23 @@ public void setServerUrl(String serverUrl) {
    * @return
    */
   public MConnector getConnector(long cid) {
-    return requests.readConnector(cid).getConnectors().get(0);
+    if(connectors.containsKey(cid)) {
+      return connectors.get(cid);
+    }
+
+    retrieveConnector(cid);
+    return connectors.get(cid);
+  }
+
+  /**
+   * Retrieve connector structure from server and cache it.
+   *
+   * @param cid Connector id
+   */
+  private void retrieveConnector(long cid) {
+    ConnectorBean request = requests.readConnector(cid);
+    connectors.put(cid, request.getConnectors().get(0));
+    bundles.put(cid, request.getResourceBundles().get(cid));
   }
 
   /**
@@ -69,18 +147,34 @@ public MConnector getConnector(long cid) {
    *
    * @return
    */
-  public List<MConnector> getConnectors() {
+  public Collection<MConnector> getConnectors() {
-    return requests.readConnector(null).getConnectors();
+    if(allConnectors) {
+      return connectors.values();
+    }
+
+    ConnectorBean bean = requests.readConnector(null);
+    allConnectors = true;
+    for(MConnector connector : bean.getConnectors()) {
+      connectors.put(connector.getPersistenceId(), connector);
+    }
+    bundles = bean.getResourceBundles();
+
+    return connectors.values();
   }
 
   /**
-   * Get resouce bundle for given connector.
+   * Get resource bundle for given connector.
    *
    * @param cid Connector id.
    * @return
    */
   public ResourceBundle getResourceBundle(long cid) {
-    return requests.readConnector(cid).getResourceBundles().get(cid);
+    if(bundles.containsKey(cid)) {
+      return bundles.get(cid);
+    }
+
+    retrieveConnector(cid);
+    return bundles.get(cid);
   }
 
   /**
@@ -89,7 +183,22 @@ public ResourceBundle getResourceBundle(long cid) {
    * @return
    */
   public MFramework getFramework() {
-    return requests.readFramework().getFramework();
+    if(framework != null) {
+      return framework;
+    }
+
+    retrieveFramework();
+    return framework;
+
+  }
+
+  /**
+   * Retrieve framework structure and cache it.
+   */
+  private void retrieveFramework() {
+    FrameworkBean request =  requests.readFramework();
+    framework = request.getFramework();
+    frameworkBundle = request.getResourceBundle();
   }
 
   /**
@@ -98,7 +207,12 @@ public MFramework getFramework() {
    * @return
    */
   public ResourceBundle getFrameworkResourceBundle() {
-    return requests.readFramework().getResourceBundle();
+    if(frameworkBundle != null) {
+      return frameworkBundle;
+    }
+
+    retrieveFramework();
+    return frameworkBundle;
   }
 
   /**

client/src/main/java/org/apache/sqoop/client/shell/ShowConnectorFunction.java

@@ -17,6 +17,7 @@
  */
 package org.apache.sqoop.client.shell;
 
+import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -56,7 +57,7 @@ public Object executeFunction(CommandLine line) {
   }
 
   private void showSummary() {
-    List<MConnector> connectors = client.getConnectors();
+    Collection<MConnector> connectors = client.getConnectors();
 
     List<String> header = new LinkedList<String>();
     header.add(resourceString(Constants.RES_TABLE_HEADER_ID));
@@ -80,7 +81,7 @@ private void showSummary() {
   }
 
   private void showConnectors() {
-    List<MConnector> connectors = client.getConnectors();
+    Collection<MConnector> connectors = client.getConnectors();
 
     printlnResource(Constants.RES_SHOW_PROMPT_CONNECTORS_TO_SHOW, connectors.size());
 

client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java

@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.client;
+
+import org.apache.sqoop.client.request.SqoopRequests;
+import org.apache.sqoop.json.ConnectorBean;
+import org.apache.sqoop.json.FrameworkBean;
+import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MFramework;
+import org.apache.sqoop.model.MJobForms;
+import org.apache.sqoop.utils.MapResourceBundle;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.ResourceBundle;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.*;
+
+public class TestSqoopClient {
+
+  SqoopRequests requests;
+  SqoopClient client;
+
+  @Before
+  public void setUp() {
+    requests = mock(SqoopRequests.class);
+    client = new SqoopClient("my-cool-server");
+    client.setSqoopRequests(requests);
+  }
+
+  /**
+   * Retrieve connector information, request to bundle for same connector should
+   * not require additional HTTP request.
+   */
+  @Test
+  public void testGetConnector() {
+    when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
+    MConnector connector = client.getConnector(1);
+    assertEquals(1, connector.getPersistenceId());
+
+    client.getResourceBundle(1L);
+
+    verify(requests, times(1)).readConnector(1L);
+  }
+
+  /**
+   * Retrieve connector bundle, request for metadata for same connector should
+   * not require additional HTTP request.
+   */
+  @Test
+  public void testGetConnectorBundle() {
+    when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1)));
+    client.getResourceBundle(1L);
+
+    MConnector connector = client.getConnector(1);
+    assertEquals(1, connector.getPersistenceId());
+
+    verify(requests, times(1)).readConnector(1L);
+  }
+
+  /**
+   * Retrieve framework information, request to framework bundle should not
+   * require additional HTTP request.
+   */
+  @Test
+  public void testGetFramework() {
+    when(requests.readFramework()).thenReturn(frameworkBean(framework()));
+
+    client.getFramework();
+    client.getFrameworkResourceBundle();
+
+    verify(requests, times(1)).readFramework();
+  }
+
+  /**
+   * Retrieve framework bundle, request to framework metadata should not
+   * require additional HTTP request.
+   */
+  @Test
+  public void testGetFrameworkBundle() {
+    when(requests.readFramework()).thenReturn(frameworkBean(framework()));
+
+    client.getFrameworkResourceBundle();
+    client.getFramework();
+
+    verify(requests, times(1)).readFramework();
+  }
+
+  /**
+   * Getting all connectors at once should avoid any other HTTP request to
+   * specific connectors.
+   */
+  @Test
+  public void testGetConnectors() {
+    MConnector connector;
+
+    when(requests.readConnector(null)).thenReturn(connectorBean(connector(1), connector(2)));
+    Collection<MConnector> connectors = client.getConnectors();
+    assertEquals(2, connectors.size());
+
+    client.getResourceBundle(1);
+    connector = client.getConnector(1);
+    assertEquals(1, connector.getPersistenceId());
+
+    connector = client.getConnector(2);
+    client.getResourceBundle(2);
+    assertEquals(2, connector.getPersistenceId());
+
+    connectors = client.getConnectors();
+    assertEquals(2, connectors.size());
+
+    verify(requests, times(1)).readConnector(null);
+    verifyNoMoreInteractions(requests);
+  }
+
+
+  /**
+   * Getting connectors one by one should not be equivalent to getting all connectors
+   * at once as Client do not know how many connectors server have.
+   */
+  @Test
+  public void testGetConnectorOneByOne() {
+    ConnectorBean bean = connectorBean(connector(1), connector(2));
+    when(requests.readConnector(null)).thenReturn(bean);
+    when(requests.readConnector(1L)).thenReturn(bean);
+    when(requests.readConnector(2L)).thenReturn(bean);
+
+    client.getResourceBundle(1);
+    client.getConnector(1);
+
+    client.getConnector(2);
+    client.getResourceBundle(2);
+
+    Collection<MConnector> connectors = client.getConnectors();
+    assertEquals(2, connectors.size());
+
+    verify(requests, times(1)).readConnector(null);
+    verify(requests, times(1)).readConnector(1L);
+    verify(requests, times(1)).readConnector(2L);
+    verifyNoMoreInteractions(requests);
+  }
+
+  private ConnectorBean connectorBean(MConnector...connectors) {
+    List<MConnector> connectorList = new ArrayList<MConnector>();
+    Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>();
+
+    for(MConnector connector : connectors) {
+      connectorList.add(connector);
+      bundles.put(connector.getPersistenceId(), null);
+    }
+    return new ConnectorBean(connectorList, bundles);
+  }
+  private FrameworkBean frameworkBean(MFramework framework) {
+    return new FrameworkBean(framework, new MapResourceBundle(null));
+  }
+
+  private MConnector connector(long id) {
+    MConnector connector = new MConnector("A" + id, "A" + id, "1.0" + id, new MConnectionForms(null), new LinkedList<MJobForms>());
+    connector.setPersistenceId(id);
+    return connector;
+  }
+
+  private MFramework framework() {
+    MFramework framework = new MFramework(new MConnectionForms(null), new LinkedList<MJobForms>());
+    framework.setPersistenceId(1);
+    return framework;
+  }
+}

pom.xml

@@ -100,6 +100,7 @@ limitations under the License.
     <guava.version>11.0.2</guava.version>
     <json-simple.version>1.1</json-simple.version>
     <junit.version>4.9</junit.version>
+    <mockito.version>1.9.5</mockito.version>
     <log4j.version>1.2.16</log4j.version>
     <servlet.version>2.5</servlet.version>
     <cargo.version>1.3.2</cargo.version>
@@ -374,6 +375,12 @@ limitations under the License.
         <artifactId>sqljdbc4</artifactId>
         <version>${jdbc.sqlserver.version}</version>
       </dependency>
+     <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>${mockito.version}</version>
+      <scope>test</scope>
+    </dependency>
    </dependencies>
   </dependencyManagement>