github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-16 00:41:23 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-2269: Sqoop2: Parquet integration tests

Browse Source 
(Abraham Elmahrek via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2015-04-14 12:28:59 -07:00
parent 5ea695a6d9
commit bfc10914c1
4 changed files with 61 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
connector/connector-kite/pom.xml
View File

@ -80,6 +80,12 @@ limitations under the License.
<dependency>
<groupId>org.kitesdk</groupId>
<artifactId>kite-data-hive</artifactId>
<exclusions>
<exclusion>
<groupId>com.twitter</groupId>
<artifactId>parquet-hive-bundle</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>

3
connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
View File

@ -75,12 +75,11 @@ public Set<String> getJars(InitializerContext context,
jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"));
}
return jars;
}

3
connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
View File

@ -83,12 +83,11 @@ public Set<String> getJars(InitializerContext context,
jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"));
}
return jars;
}

63
test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
View File

@ -18,6 +18,8 @@
package org.apache.sqoop.integration.connector.hive;
import org.apache.sqoop.common.test.asserts.ProviderAsserts;
import org.apache.sqoop.common.test.db.DatabaseProvider;
import org.apache.sqoop.common.test.db.DatabaseProviderFactory;
import org.apache.sqoop.common.test.db.TableName;
import org.apache.sqoop.connector.common.FileFormat;
import org.apache.sqoop.model.MConfigList;
@ -25,15 +27,56 @@
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.testcases.HiveConnectorTestCase;
import org.testng.ITest;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Factory;
import org.testng.annotations.Test;
/**
*/
public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
import java.lang.reflect.Method;
@Test(groups = "slow")
public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest {
private String testName;
private FileFormat fileFormat;
private MLink rdbmsLink;
private MLink kiteLink;
private String hiveTableName;
@Factory(dataProvider="rdbms-to-kite-hive-test")
public FromRDBMSToKiteHiveTest(FileFormat fileFormat) {
this.fileFormat = fileFormat;
}
@DataProvider(name="rdbms-to-kite-hive-test", parallel=true)
public static Object[][] data() throws Exception {
DatabaseProvider provider = DatabaseProviderFactory.getProvider(System.getProperties());
return new Object[][]{
{FileFormat.AVRO},
{FileFormat.PARQUET}
};
}
@Override
public String getTestName() {
return testName + "[" + fileFormat.name() + "]";
}
public String getHiveTableName() {
return testName + "_" + fileFormat.name();
}
public String getDatasetURI() {
return "dataset:hive:" + getHiveTableName();
}
@BeforeMethod(alwaysRun = true)
public void setTestName(Method aMethod) {
this.testName = aMethod.getName();
}
@BeforeMethod(alwaysRun = true)
public void createTable() {
@ -70,8 +113,8 @@ public void testCities() throws Exception {
// Fill the Kite "TO" config
MConfigList toConfig = job.getToJobConfig();
toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable");
toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO);
toConfig.getStringInput("toJobConfig.uri").setValue(getDatasetURI());
toConfig.getEnumInput("toJobConfig.fileFormat").setValue(this.fileFormat);
// driver config
MDriverConfig driverConfig = job.getDriverConfig();
@ -81,11 +124,11 @@ public void testCities() throws Exception {
executeJob(job);
// Assert correct output
ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1");
ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2");
ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3");
ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4");
ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 1}, "1");
ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 2}, "2");
ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 3}, "3");
ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 4}, "4");
hiveProvider.dropTable(new TableName("testtable"));
hiveProvider.dropTable(new TableName(getHiveTableName()));
}
}