diff --git a/src/java/com/cloudera/sqoop/manager/SqlManager.java b/src/java/com/cloudera/sqoop/manager/SqlManager.java index 51ca65bb..72aecf7e 100644 --- a/src/java/com/cloudera/sqoop/manager/SqlManager.java +++ b/src/java/com/cloudera/sqoop/manager/SqlManager.java @@ -440,11 +440,19 @@ public void importQuery(ImportJobContext context) } String splitCol = getSplitColumn(opts, null); - if (null == splitCol && opts.getNumMappers() > 1) { - // Can't infer a primary key. - throw new ImportException("A split-by column must be specified for " - + "parallel free-form query imports. Please specify one with " - + "--split-by or perform a sequential import with '-m 1'."); + if (splitCol == null) { + String boundaryQuery = opts.getBoundaryQuery(); + if (opts.getNumMappers() > 1) { + // Can't infer a primary key. + throw new ImportException("A split-by column must be specified for " + + "parallel free-form query imports. Please specify one with " + + "--split-by or perform a sequential import with '-m 1'."); + } else if (boundaryQuery != null && !boundaryQuery.isEmpty()) { + // Query import with boundary query and no split column specified + throw new ImportException("Using a boundary query for a query based " + + "import requires specifying the split by column as well. Please " + + "specify a column name using --split-by and try again."); + } } importer.runImport(null, jarFile, splitCol, opts.getConf()); diff --git a/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java b/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java index 25b82807..d5c66dd2 100644 --- a/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java +++ b/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java @@ -171,11 +171,16 @@ protected void configureInputFormat(Job job, String tableName, String inputBoundingQuery = options.getBoundaryQuery(); if(inputBoundingQuery == null) { + inputBoundingQuery = mgr.getInputBoundsQuery(splitByCol, sanitizedQuery); - } - if (inputBoundingQuery == null) { - inputBoundingQuery = "SELECT MIN(" + splitByCol + "), MAX(" - + splitByCol + ") FROM (" + sanitizedQuery + ") AS t1"; + if (inputBoundingQuery == null) { + if (splitByCol != null) { + inputBoundingQuery = "SELECT MIN(" + splitByCol + "), MAX(" + + splitByCol + ") FROM (" + sanitizedQuery + ") AS t1"; + } else { + inputBoundingQuery = ""; + } + } } DataDrivenDBInputFormat.setInput(job, DBWritable.class, inputQuery, inputBoundingQuery); diff --git a/src/test/com/cloudera/sqoop/TestBoundaryQuery.java b/src/test/com/cloudera/sqoop/TestBoundaryQuery.java index d7e821fe..b0d3b8a0 100644 --- a/src/test/com/cloudera/sqoop/TestBoundaryQuery.java +++ b/src/test/com/cloudera/sqoop/TestBoundaryQuery.java @@ -46,8 +46,9 @@ public class TestBoundaryQuery extends ImportJobTestCase { * Create the argv to pass to Sqoop. * @return the argv as an array of strings. */ - protected String [] getArgv(boolean includeHadoopFlags, String boundaryQuery, - String targetDir) { + protected String [] getArgv(boolean includeHadoopFlags, boolean tableImport, + String boundaryQuery, + String targetDir, String... extraArgs) { ArrayList args = new ArrayList(); @@ -55,20 +56,29 @@ public class TestBoundaryQuery extends ImportJobTestCase { CommonArgs.addHadoopFlags(args); } - args.add("--table"); - args.add(HsqldbTestServer.getTableName()); - args.add("--split-by"); - args.add("INTFIELD1"); + if (tableImport) { + args.add("--table"); + args.add(HsqldbTestServer.getTableName()); + } else { + args.add("--query"); + args.add("SELECT INTFIELD1, INTFIELD2 FROM " + + HsqldbTestServer.getTableName() + " WHERE $CONDITIONS"); + } args.add("--connect"); args.add(HsqldbTestServer.getUrl()); - args.add("--boundary-query"); - args.add(boundaryQuery); + if (boundaryQuery != null) { + args.add("--boundary-query"); + args.add(boundaryQuery); + } args.add("--as-sequencefile"); args.add("--target-dir"); args.add(targetDir); args.add("--class-name"); args.add(getTableName()); args.add("--verbose"); + for (String extraArg : extraArgs) { + args.add(extraArg); + } return args.toArray(new String[0]); } @@ -89,18 +99,18 @@ private int getFirstInt(String str) { return Integer.parseInt(parts[0]); } - public void runQueryTest(String query, int numExpectedResults, - int expectedSum, String targetDir) - throws IOException { + public void runQueryTest(String query, boolean tableImport, + int numExpectedResults, int expectedSum, String targetDir, + String... extraArgs) throws IOException { ClassLoader prevClassLoader = null; SequenceFile.Reader reader = null; - String [] argv = getArgv(true, query, targetDir); + String [] argv = getArgv(true, tableImport, query, targetDir, extraArgs); runImport(argv); try { SqoopOptions opts = new ImportTool().parseArguments( - getArgv(false, query, targetDir), + getArgv(false, tableImport, query, targetDir, extraArgs), null, null, true); CompilationManager compileMgr = new CompilationManager(opts); @@ -161,6 +171,13 @@ public void testBoundaryQuery() throws IOException { String query = "select min(intfield1), max(intfield1) from " + getTableName() +" where intfield1 in (3, 5)"; - runQueryTest(query, 2, 8, getTablePath().toString()); + runQueryTest(query, true, 2, 8, getTablePath().toString(), + "--split-by", "INTFIELD1"); + } + + public void testNoBoundaryQuerySingleMapper() throws IOException { + + runQueryTest(null, false, 4, 16, getTablePath().toString(), + "--m", "1"); } }