diff --git a/connector/connector-kite/pom.xml b/connector/connector-kite/pom.xml index 8593899b..41441f55 100644 --- a/connector/connector-kite/pom.xml +++ b/connector/connector-kite/pom.xml @@ -80,6 +80,12 @@ limitations under the License. org.kitesdk kite-data-hive + + + com.twitter + parquet-hive-bundle + + org.slf4j diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java index fd3017be..ed793142 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java @@ -75,12 +75,11 @@ public Set getJars(InitializerContext context, jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors")); jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse")); jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat")); - jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory")); jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService")); jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler")); jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery")); jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement")); - jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe")); + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe")); } return jars; } diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java index e27c0b89..50daba08 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java @@ -83,12 +83,11 @@ public Set getJars(InitializerContext context, jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors")); jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse")); jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat")); - jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory")); jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService")); jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler")); jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery")); jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement")); - jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe")); + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe")); } return jars; } diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java index a4a5b040..37a0a7d0 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java @@ -18,6 +18,8 @@ package org.apache.sqoop.integration.connector.hive; import org.apache.sqoop.common.test.asserts.ProviderAsserts; +import org.apache.sqoop.common.test.db.DatabaseProvider; +import org.apache.sqoop.common.test.db.DatabaseProviderFactory; import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.connector.common.FileFormat; import org.apache.sqoop.model.MConfigList; @@ -25,15 +27,56 @@ import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; import org.apache.sqoop.test.testcases.HiveConnectorTestCase; +import org.testng.ITest; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; import org.testng.annotations.Test; -/** - */ -public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase { +import java.lang.reflect.Method; + +@Test(groups = "slow") +public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest { + private String testName; + + private FileFormat fileFormat; + private MLink rdbmsLink; private MLink kiteLink; + private String hiveTableName; + + @Factory(dataProvider="rdbms-to-kite-hive-test") + public FromRDBMSToKiteHiveTest(FileFormat fileFormat) { + this.fileFormat = fileFormat; + } + + @DataProvider(name="rdbms-to-kite-hive-test", parallel=true) + public static Object[][] data() throws Exception { + DatabaseProvider provider = DatabaseProviderFactory.getProvider(System.getProperties()); + return new Object[][]{ + {FileFormat.AVRO}, + {FileFormat.PARQUET} + }; + } + + @Override + public String getTestName() { + return testName + "[" + fileFormat.name() + "]"; + } + + public String getHiveTableName() { + return testName + "_" + fileFormat.name(); + } + + public String getDatasetURI() { + return "dataset:hive:" + getHiveTableName(); + } + + @BeforeMethod(alwaysRun = true) + public void setTestName(Method aMethod) { + this.testName = aMethod.getName(); + } @BeforeMethod(alwaysRun = true) public void createTable() { @@ -70,8 +113,8 @@ public void testCities() throws Exception { // Fill the Kite "TO" config MConfigList toConfig = job.getToJobConfig(); - toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable"); - toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO); + toConfig.getStringInput("toJobConfig.uri").setValue(getDatasetURI()); + toConfig.getEnumInput("toJobConfig.fileFormat").setValue(this.fileFormat); // driver config MDriverConfig driverConfig = job.getDriverConfig(); @@ -81,11 +124,11 @@ public void testCities() throws Exception { executeJob(job); // Assert correct output - ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1"); - ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2"); - ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3"); - ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4"); + ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 1}, "1"); + ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 2}, "2"); + ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 3}, "3"); + ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 4}, "4"); - hiveProvider.dropTable(new TableName("testtable")); + hiveProvider.dropTable(new TableName(getHiveTableName())); } }