diff --git a/test/pom.xml b/test/pom.xml index b7ea1ed5..8001fcec 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -33,7 +33,6 @@ limitations under the License. junit junit - test diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java new file mode 100644 index 00000000..056e6124 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.asserts; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.sqoop.test.utils.HdfsUtils; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.fail; + +/** + * Assert methods suitable for checking HDFS files and directories. + * + * TODO: This module will require clean up to work on MiniCluster/Real cluster. + */ +public class HdfsAsserts { + + private static final Logger LOG = Logger.getLogger(HdfsAsserts.class); + + /** + * Verify that mapreduce output (across all files) is as expected. + * + * @param directory Mapreduce output directory + * @param lines Expected lines + * @throws IOException + */ + public static void assertMapreduceOutput(String directory, String... lines) throws IOException { + Set setLines = new HashSet(Arrays.asList(lines)); + List notFound = new LinkedList(); + + String []files = HdfsUtils.getOutputMapreduceFiles(directory); + + for(String file : files) { + String filePath = directory + "/" + file; + BufferedReader br = new BufferedReader(new FileReader((filePath))); + + String line; + while ((line = br.readLine()) != null) { + if (!setLines.remove(line)) { + notFound.add(line); + } + } + br.close(); + } + + if(!setLines.isEmpty() || !notFound.isEmpty()) { + LOG.error("Expected lines that weren't present in the files:"); + LOG.error("\t" + StringUtils.join(setLines, "\n\t")); + LOG.error("Extra lines in files that weren't expected:"); + LOG.error("\t" + StringUtils.join(notFound, "\n\t")); + fail("Output do not match expectations."); + } + } + + + private HdfsAsserts() { + // Instantiation is prohibited + } +} diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java new file mode 100644 index 00000000..364f86d2 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.asserts; + +import org.apache.sqoop.test.db.DatabaseProvider; +import org.apache.log4j.Logger; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Database provider related asserts. + */ +public class ProviderAsserts { + + private static final Logger LOG = Logger.getLogger(ProviderAsserts.class); + + /** + * Assert row in the table. + * + * @param provider Provider that should be used to query the database + * @param tableName Table name + * @param conditions Conditions for identifying the row + * @param values Values that should be present in the table + */ + public static void assertRow(DatabaseProvider provider, String tableName, Object []conditions, Object ...values) { + ResultSet rs = null; + try { + rs = provider.getRows(tableName, conditions); + + if(! rs.next()) { + fail("No rows found."); + } + + int i = 1; + for(Object expectedValue : values) { + Object actualValue = rs.getObject(i); + assertEquals("Columns do not match on position: " + i, expectedValue, actualValue); + i++; + } + + if(rs.next()) { + fail("Found more than one row."); + } + } catch (SQLException e) { + LOG.error("Unexpected SQLException", e); + fail("Unexpected SQLException: " + e); + } finally { + provider.closeResultSetWithStatement(rs); + } + } + + private ProviderAsserts() { + // Instantiation is prohibited + } +} diff --git a/test/src/main/java/org/apache/sqoop/test/data/Cities.java b/test/src/main/java/org/apache/sqoop/test/data/Cities.java new file mode 100644 index 00000000..fb9be13f --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/data/Cities.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.data; + +import org.apache.sqoop.test.db.DatabaseProvider; + +/** + * Simple listing of few world's cities to do basic sanity tests. + */ +public class Cities extends DataSet { + + public Cities(DatabaseProvider provider, String tableBaseName) { + super(provider, tableBaseName); + } + + @Override + public DataSet createTables() { + provider.createTable( + tableBaseName, + "id", + "id", "int", + "country", "varchar(50)", + "city", "varchar(50)" + ); + + return this; + } + + @Override + public DataSet loadBasicData() { + provider.insertRow(tableBaseName, 1, "USA", "San Francisco"); + provider.insertRow(tableBaseName, 2, "USA", "Sunnyvale"); + provider.insertRow(tableBaseName, 3, "Czech Republic", "Brno"); + provider.insertRow(tableBaseName, 4, "USA", "Palo Alto"); + + return this; + } +} diff --git a/test/src/main/java/org/apache/sqoop/test/data/DataSet.java b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java new file mode 100644 index 00000000..69994084 --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.data; + +import org.apache.sqoop.test.db.DatabaseProvider; + +/** + * Abstract class for basic testing data sets. + * + * Each data set provides couple of generic methods that can be used to set up + * the tables and load example data. + */ +public abstract class DataSet { + + /** + * Database provider that will be used to populate the data. + */ + protected DatabaseProvider provider; + + /** + * Base name for created tables. + */ + protected String tableBaseName; + + public DataSet(DatabaseProvider provider, String tableBaseName) { + setProvider(provider); + setTableBaseName(tableBaseName); + } + + public DataSet setProvider(DatabaseProvider provider) { + this.provider = provider; + return this; + } + + public DataSet setTableBaseName(String tableBaseName) { + this.tableBaseName = tableBaseName; + return this; + } + + /** + * Crate all tables that this testing data set might need. + */ + public abstract DataSet createTables(); + + /** + * Load basic data. + * + * Basic data set should be small (around 10 rows) without any specialities. + */ + public abstract DataSet loadBasicData(); +} diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java similarity index 80% rename from test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java rename to test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index d4e432de..6aeadd4e 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -15,33 +15,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.sqoop.integration.connector; +package org.apache.sqoop.test.testcases; import org.apache.log4j.Logger; import org.apache.sqoop.framework.configuration.OutputFormat; import org.apache.sqoop.framework.configuration.StorageType; -import org.apache.sqoop.integration.TomcatTestCase; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MFormList; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MPersistableEntity; +import org.apache.sqoop.test.asserts.ProviderAsserts; +import org.apache.sqoop.test.data.Cities; import org.apache.sqoop.test.db.DatabaseProvider; import org.apache.sqoop.test.db.DatabaseProviderFactory; import org.apache.sqoop.validation.Status; import org.junit.AfterClass; import org.junit.BeforeClass; -import java.sql.ResultSet; -import java.sql.SQLException; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.fail; /** - * Base test case for connector testing. + * Base test case suitable for connector testing. * - * It will create and initialize database provider prior every test execution. + * In addition to pure Tomcat based test case it will also create and initialize + * the database provider prior every test execution. */ abstract public class ConnectorTestCase extends TomcatTestCase { @@ -124,22 +122,14 @@ protected void fillInputForm(MJob job) { * Create table cities. */ protected void createTableCities() { - createTable("id", - "id", "int", - "country", "varchar(50)", - "city", "varchar(50)" - ); + new Cities(provider, getTableName()).createTables(); } /** * Create table cities and load few rows. */ protected void createAndLoadTableCities() { - createTableCities(); - insertRow(1, "USA", "San Francisco"); - insertRow(2, "USA", "Sunnyvale"); - insertRow(3, "Czech Republic", "Brno"); - insertRow(4, "USA", "Palo Alto"); + new Cities(provider, getTableName()).createTables().loadBasicData(); } /** @@ -149,29 +139,7 @@ protected void createAndLoadTableCities() { * @param values Values that are expected in the table (with corresponding types) */ protected void assertRow(Object []conditions, Object ...values) { - ResultSet rs = provider.getRows(getTableName(), conditions); - - try { - if(! rs.next()) { - fail("No rows found."); - } - - int i = 1; - for(Object expectedValue : values) { - Object actualValue = rs.getObject(i); - assertEquals("Columns do not match on position: " + i, expectedValue, actualValue); - i++; - } - - if(rs.next()) { - fail("Found more than one row."); - } - } catch (SQLException e) { - LOG.error("Unexpected SQLException", e); - fail("Unexpected SQLException: " + e); - } finally { - provider.closeResultSetWithStatement(rs); - } + ProviderAsserts.assertRow(provider, getTableName(), conditions, values); } /** @@ -179,7 +147,7 @@ protected void assertRow(Object []conditions, Object ...values) { * * @param values Values that are expected */ - protected void assertRowInCitiesTable(Object ... values) { + protected void assertRowInCities(Object... values) { assertRow(new Object[]{"id", values[0]}, values); } diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java similarity index 69% rename from test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java rename to test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java index fa2c2b46..ca77e644 100644 --- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java @@ -15,30 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.sqoop.integration; +package org.apache.sqoop.test.testcases; -import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; -import org.apache.commons.lang.StringUtils; import org.apache.sqoop.client.SqoopClient; +import org.apache.sqoop.test.asserts.HdfsAsserts; import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; +import org.apache.sqoop.test.utils.HdfsUtils; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.rules.TestName; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FilenameFilter; import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import static org.junit.Assert.fail; /** * Basic test case that will bootstrap Sqoop server running in external Tomcat @@ -131,21 +120,6 @@ public String getMapreduceDirectory() { return getTemporaryPath() + "/mapreduce-job-io"; } - /** - * Return list of file names that are outputs of mapreduce job. - * - * @return - */ - public String[] getOutputFilesMapreduce() { - File dir = new File(getMapreduceDirectory()); - return dir.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("part-"); - } - }); - } - /** * Assert that mapreduce has generated following lines. * @@ -156,31 +130,7 @@ public boolean accept(File dir, String name) { * @throws IOException */ protected void assertMapreduceOutput(String... lines) throws IOException { - Set setLines = new HashSet(Arrays.asList(lines)); - List notFound = new LinkedList(); - - String []files = getOutputFilesMapreduce(); - - for(String file : files) { - String filePath = getMapreduceDirectory() + "/" + file; - BufferedReader br = new BufferedReader(new FileReader((filePath))); - - String line; - while ((line = br.readLine()) != null) { - if (!setLines.remove(line)) { - notFound.add(line); - } - } - br.close(); - } - - if(!setLines.isEmpty() || !notFound.isEmpty()) { - LOG.error("Expected lines that weren't present in the files:"); - LOG.error("\t" + StringUtils.join(setLines, "\n\t")); - LOG.error("Extra lines in files that weren't expected:"); - LOG.error("\t" + StringUtils.join(notFound, "\n\t")); - fail("Output do not match expectations."); - } + HdfsAsserts.assertMapreduceOutput(getMapreduceDirectory(), lines); } /** @@ -191,7 +141,6 @@ protected void assertMapreduceOutput(String... lines) throws IOException { * @throws IOException */ protected void createInputMapreduceFile(String filename, String...lines) throws IOException { - File outputFile = new File(getMapreduceDirectory(), filename); - FileUtils.writeLines(outputFile, Arrays.asList(lines)); + HdfsUtils.createFile(getMapreduceDirectory(), filename, lines); } } diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java new file mode 100644 index 00000000..95dd177a --- /dev/null +++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.test.utils; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Arrays; + +/** + * Handy utilities to work with HDFS + * + * TODO: This module will require clean up to work on MiniCluster/Real cluster. + */ +public class HdfsUtils { + + private static final Logger LOG = Logger.getLogger(HdfsUtils.class); + + /** + * Get list of mapreduce output files from given directory. + * + * @param directory Directory to be searched for files generated by MR + * @return + */ + public static String [] getOutputMapreduceFiles(String directory) { + File dir = new File(directory); + return dir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith("part-"); + } + }); + } + + /** + * Create HDFS file with given content. + * + * @param directory Directory where the file should be created + * @param filename File name + * @param lines Individual lines that should be written into the file + * @throws IOException + */ + public static void createFile(String directory, String filename, String ...lines) throws IOException { + File outputFile = new File(directory, filename); + FileUtils.writeLines(outputFile, Arrays.asList(lines)); + } + + private HdfsUtils() { + // Instantiation is not allowed + } +} diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java index b920655e..fbf46e99 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java @@ -18,7 +18,7 @@ package org.apache.sqoop.integration.connector.jdbc.generic; import org.apache.log4j.Logger; -import org.apache.sqoop.integration.connector.ConnectorTestCase; +import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MFormList; import org.apache.sqoop.model.MJob; @@ -70,10 +70,10 @@ public void testBasicImport() throws Exception { } while(submission.getStatus().isRunning()); assertEquals(4L, rowCount()); - assertRowInCitiesTable(1, "USA", "San Francisco"); - assertRowInCitiesTable(2, "USA", "Sunnyvale"); - assertRowInCitiesTable(3, "Czech Republic", "Brno"); - assertRowInCitiesTable(4, "USA", "Palo Alto"); + assertRowInCities(1, "USA", "San Francisco"); + assertRowInCities(2, "USA", "Sunnyvale"); + assertRowInCities(3, "Czech Republic", "Brno"); + assertRowInCities(4, "USA", "Palo Alto"); // Clean up testing table dropTable(); diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java index adcfbaf8..639d9ad1 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java @@ -20,7 +20,7 @@ import org.apache.log4j.Logger; import org.apache.sqoop.framework.configuration.OutputFormat; import org.apache.sqoop.framework.configuration.StorageType; -import org.apache.sqoop.integration.connector.ConnectorTestCase; +import org.apache.sqoop.test.testcases.ConnectorTestCase; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MFormList; import org.apache.sqoop.model.MJob; diff --git a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java index 0f48a8b6..cea24b96 100644 --- a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java @@ -19,7 +19,7 @@ import org.apache.sqoop.client.request.VersionRequest; import org.apache.sqoop.common.VersionInfo; -import org.apache.sqoop.integration.TomcatTestCase; +import org.apache.sqoop.test.testcases.TomcatTestCase; import org.apache.sqoop.json.VersionBean; import org.junit.Test;