From de6fa8279d8125afb4fd2d7f1e8c7e731d70451d Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Thu, 13 Aug 2015 17:10:42 -0700 Subject: [PATCH] SQOOP-2470: Incremental Hive import with append not working after validation check for --hive-import and --import (Aaditya Raj via Jarek Jarcec Cecho) --- .../org/apache/sqoop/tool/BaseSqoopTool.java | 4 +- .../cloudera/sqoop/TestIncrementalImport.java | 63 ++++++++++++++++++- .../hive/scripts/incrementalHiveAppend10.q | 17 +++++ .../hive/scripts/incrementalHiveAppend20.q | 17 +++++ .../hive/scripts/incrementalHiveAppendEmpty.q | 17 +++++ 5 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 testdata/hive/scripts/incrementalHiveAppend10.q create mode 100644 testdata/hive/scripts/incrementalHiveAppend20.q create mode 100644 testdata/hive/scripts/incrementalHiveAppendEmpty.q diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 4e2e66d5..a7f6aec0 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -40,6 +40,7 @@ import com.cloudera.sqoop.ConnFactory; import com.cloudera.sqoop.Sqoop; import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.SqoopOptions.IncrementalMode; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; import com.cloudera.sqoop.cli.RelatedOptions; import com.cloudera.sqoop.cli.ToolOptions; @@ -1398,7 +1399,8 @@ protected void validateHiveOptions(SqoopOptions options) } if (options.doHiveImport() - && options.isAppendMode()) { + && options.isAppendMode() + && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) { throw new InvalidOptionsException("Append mode for hive imports is not " + " yet supported. Please remove the parameter --append-mode"); } diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java index a6680e8d..3e3f9296 100644 --- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java +++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java @@ -19,6 +19,7 @@ package com.cloudera.sqoop; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.sql.Connection; @@ -29,9 +30,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; - import junit.framework.TestCase; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -39,7 +38,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; - +import org.apache.sqoop.hive.HiveImport; import com.cloudera.sqoop.manager.ConnManager; import com.cloudera.sqoop.manager.HsqldbManager; import com.cloudera.sqoop.manager.ManagerFactory; @@ -1216,6 +1215,64 @@ public void testIncrementalAppendTimestamp() throws Exception { runJob(TABLE_NAME); assertDirOfNumbers(TABLE_NAME, 20); } + + public void testIncrementalHiveAppendEmptyThenFull() throws Exception { + // This is to test Incremental Hive append feature. SQOOP-2470 + final String TABLE_NAME = "incrementalHiveAppendEmptyThenFull"; + Configuration conf = newConf(); + conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY, + InstrumentHsqldbManagerFactory.class.getName()); + clearDir(TABLE_NAME); + createIdTable(TABLE_NAME, 0); + List args = new ArrayList(); + args.add("--connect"); + args.add(SOURCE_DB_URL); + args.add("--table"); + args.add(TABLE_NAME); + args.add("--warehouse-dir"); + args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR); + args.add("--hive-import"); + args.add("--hive-table"); + args.add(TABLE_NAME + "hive"); + args.add("--incremental"); + args.add("append"); + args.add("--check-column"); + args.add("id"); + args.add("-m"); + args.add("1"); + createJob(TABLE_NAME, args, conf); + HiveImport.setTestMode(true); + String hiveHome = org.apache.sqoop.SqoopOptions.getHiveHomeDefault(); + assertNotNull("hive.home was not set", hiveHome); + String testDataPath = new Path(new Path(hiveHome), "scripts/" + + "incrementalHiveAppendEmpty.q").toString(); + System.clearProperty("expected.script"); + System.setProperty("expected.script", + new File(testDataPath).getAbsolutePath()); + runJob(TABLE_NAME); + assertDirOfNumbers(TABLE_NAME, 0); + // Now add some rows. + insertIdRows(TABLE_NAME, 0, 10); + String testDataPath10 = new Path(new Path(hiveHome), "scripts/" + + "incrementalHiveAppend10.q").toString(); + System.clearProperty("expected.script"); + System.setProperty("expected.script", + new File(testDataPath10).getAbsolutePath()); + System.getProperty("expected.script"); + // Running the job a second time should import 10 rows. + runJob(TABLE_NAME); + assertDirOfNumbers(TABLE_NAME, 10); + // Add some more rows. + insertIdRows(TABLE_NAME, 10, 20); + String testDataPath20 = new Path(new Path(hiveHome), "scripts/" + + "incrementalHiveAppend20.q").toString(); + System.clearProperty("expected.script"); + System.setProperty("expected.script", + new File(testDataPath20).getAbsolutePath()); + // Import only those rows. + runJob(TABLE_NAME); + assertDirOfNumbers(TABLE_NAME, 20); + } // SQOOP-1890 public void testTableNameWithSpecialCharacters() throws Exception { diff --git a/testdata/hive/scripts/incrementalHiveAppend10.q b/testdata/hive/scripts/incrementalHiveAppend10.q new file mode 100644 index 00000000..383c7b4a --- /dev/null +++ b/testdata/hive/scripts/incrementalHiveAppend10.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; \ No newline at end of file diff --git a/testdata/hive/scripts/incrementalHiveAppend20.q b/testdata/hive/scripts/incrementalHiveAppend20.q new file mode 100644 index 00000000..383c7b4a --- /dev/null +++ b/testdata/hive/scripts/incrementalHiveAppend20.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; \ No newline at end of file diff --git a/testdata/hive/scripts/incrementalHiveAppendEmpty.q b/testdata/hive/scripts/incrementalHiveAppendEmpty.q new file mode 100644 index 00000000..383c7b4a --- /dev/null +++ b/testdata/hive/scripts/incrementalHiveAppendEmpty.q @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; +LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; \ No newline at end of file