SQOOP-2269: Sqoop2: Parquet integration tests

(Abraham Elmahrek via Jarek Jarcec Cecho)

connector/connector-kite/pom.xml

@@ -80,6 +80,12 @@ limitations under the License.
     <dependency>
       <groupId>org.kitesdk</groupId>
       <artifactId>kite-data-hive</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.twitter</groupId>
+          <artifactId>parquet-hive-bundle</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>

connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java

@@ -75,12 +75,11 @@ public Set<String> getJars(InitializerContext context,
       jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
       jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
       jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
-      jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
       jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
-      jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"));
     }
     return jars;
   }

connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java

@@ -83,12 +83,11 @@ public Set<String> getJars(InitializerContext context,
       jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
       jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
       jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
-      jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
       jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
-      jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"));
     }
     return jars;
   }

test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java

@@ -18,6 +18,8 @@
 package org.apache.sqoop.integration.connector.hive;
 
 import org.apache.sqoop.common.test.asserts.ProviderAsserts;
+import org.apache.sqoop.common.test.db.DatabaseProvider;
+import org.apache.sqoop.common.test.db.DatabaseProviderFactory;
 import org.apache.sqoop.common.test.db.TableName;
 import org.apache.sqoop.connector.common.FileFormat;
 import org.apache.sqoop.model.MConfigList;
@@ -25,15 +27,56 @@
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.test.testcases.HiveConnectorTestCase;
+import org.testng.ITest;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Factory;
 import org.testng.annotations.Test;
 
-/**
+import java.lang.reflect.Method;
- */
+
-public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
+@Test(groups = "slow")
+public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest {
+  private String testName;
+
+  private FileFormat fileFormat;
+
   private MLink rdbmsLink;
   private MLink kiteLink;
+  private String hiveTableName;
+
+  @Factory(dataProvider="rdbms-to-kite-hive-test")
+  public FromRDBMSToKiteHiveTest(FileFormat fileFormat) {
+    this.fileFormat = fileFormat;
+  }
+
+  @DataProvider(name="rdbms-to-kite-hive-test", parallel=true)
+  public static Object[][] data() throws Exception {
+    DatabaseProvider provider = DatabaseProviderFactory.getProvider(System.getProperties());
+    return new Object[][]{
+        {FileFormat.AVRO},
+        {FileFormat.PARQUET}
+    };
+  }
+
+  @Override
+  public String getTestName() {
+    return testName + "[" + fileFormat.name() + "]";
+  }
+
+  public String getHiveTableName() {
+    return testName + "_" + fileFormat.name();
+  }
+
+  public String getDatasetURI() {
+    return "dataset:hive:" + getHiveTableName();
+  }
+
+  @BeforeMethod(alwaysRun = true)
+  public void setTestName(Method aMethod) {
+    this.testName = aMethod.getName();
+  }
 
   @BeforeMethod(alwaysRun = true)
   public void createTable() {
@@ -70,8 +113,8 @@ public void testCities() throws Exception {
 
     // Fill the Kite "TO" config
     MConfigList toConfig = job.getToJobConfig();
-    toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable");
+    toConfig.getStringInput("toJobConfig.uri").setValue(getDatasetURI());
-    toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO);
+    toConfig.getEnumInput("toJobConfig.fileFormat").setValue(this.fileFormat);
 
     // driver config
     MDriverConfig driverConfig = job.getDriverConfig();
@@ -81,11 +124,11 @@ public void testCities() throws Exception {
     executeJob(job);
 
     // Assert correct output
-    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1");
+    ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 1}, "1");
-    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2");
+    ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 2}, "2");
-    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3");
+    ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 3}, "3");
-    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4");
+    ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 4}, "4");
 
-    hiveProvider.dropTable(new TableName("testtable"));
+    hiveProvider.dropTable(new TableName(getHiveTableName()));
   }
 }