diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java index e53a76fa..913cdbe7 100644 --- a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java +++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java @@ -28,6 +28,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.Charset; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; @@ -57,15 +58,14 @@ public static void assertMapreduceOutput(FileSystem fs, String directory, String Path[] files = HdfsUtils.getOutputMapreduceFiles(fs, directory); for(Path file : files) { - BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(file))); - - String line; - while ((line = br.readLine()) != null) { - if (!setLines.remove(line)) { - notFound.add(line); + try (BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(file), Charset.forName("UTF-8")))) { + String line; + while ((line = br.readLine()) != null) { + if (!setLines.remove(line)) { + notFound.add(line); + } } } - br.close(); } if(!setLines.isEmpty() || !notFound.isEmpty()) { diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java index eabdf0b3..08160674 100644 --- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java @@ -50,12 +50,7 @@ public Configuration prepareConfiguration(Configuration config) LOG.debug("Config path is: " + configPath); File dir = new File(configPath); - String [] files = dir.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.endsWith("-site.xml"); - } - }); + String [] files = dir.list(new HRCRRilenameFilter()); if(files == null) { throw new FileNotFoundException("Hadoop config files not found: " + configPath); @@ -81,4 +76,10 @@ public void stop() throws Exception { } + private static class HRCRRilenameFilter implements FilenameFilter { + @Override + public boolean accept(File dir, String name) { + return name.endsWith("-site.xml"); + } + } } diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java index 8b355bdd..1cdb0d20 100644 --- a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java @@ -23,7 +23,9 @@ import org.apache.sqoop.common.test.utils.NetworkUtils; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.PrintStream; +import java.io.UnsupportedEncodingException; /** * Hive server runner for testing purpose. @@ -129,7 +131,11 @@ public int getPort() { private void printConfig() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - config.logVars(new PrintStream(baos)); - LOG.debug("Hive server runner configuration:\n" + baos.toString()); + try { + config.logVars(new PrintStream(baos.toString("UTF-8"), "UTF-8")); + LOG.debug("Hive server runner configuration:\n" + baos.toString("UTF-8")); + } catch (UnsupportedEncodingException|FileNotFoundException e) { + LOG.warn("Error to print the Hive server runner configuration.", e); + } } } diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java index 32a6b359..8a822b53 100644 --- a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java +++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java @@ -23,7 +23,9 @@ import org.apache.sqoop.common.test.utils.NetworkUtils; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.PrintStream; +import java.io.UnsupportedEncodingException; import java.util.UUID; /** @@ -131,7 +133,11 @@ public int getPort() { private void printConfig() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - config.logVars(new PrintStream(baos)); - LOG.debug("Hive server runner configuration:\n" + baos.toString()); + try { + config.logVars(new PrintStream(baos.toString("UTF-8"), "UTF-8")); + LOG.debug("Hive server runner configuration:\n" + baos.toString("UTF-8")); + } catch (UnsupportedEncodingException |FileNotFoundException e) { + LOG.warn("Error to print the Hive server runner configuration.", e); + } } } diff --git a/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java index 5cae43b2..82043eaf 100644 --- a/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java @@ -74,7 +74,7 @@ public class SqoopTestCase implements ITest { /** * Default submission callbacks that are printing various status about the submission. */ - protected static SubmissionCallback DEFAULT_SUBMISSION_CALLBACKS = new SubmissionCallback() { + protected static final SubmissionCallback DEFAULT_SUBMISSION_CALLBACKS = new SubmissionCallback() { @Override public void submitted(MSubmission submission) { LOG.info("Submission submitted: " + submission); diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java index 622c3af3..e3e7bfe0 100644 --- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java +++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java @@ -55,7 +55,7 @@ abstract public class ConnectorTestCase extends TomcatTestCase { /** * Default submission callbacks that are printing various status about the submission. */ - protected static SubmissionCallback DEFAULT_SUBMISSION_CALLBACKS = new SubmissionCallback() { + protected static final SubmissionCallback DEFAULT_SUBMISSION_CALLBACKS = new SubmissionCallback() { @Override public void submitted(MSubmission submission) { LOG.info("Submission submitted: " + submission); diff --git a/test/src/main/java/org/apache/sqoop/test/testng/SqoopTestListener.java b/test/src/main/java/org/apache/sqoop/test/testng/SqoopTestListener.java index c86a765f..b9c14069 100644 --- a/test/src/main/java/org/apache/sqoop/test/testng/SqoopTestListener.java +++ b/test/src/main/java/org/apache/sqoop/test/testng/SqoopTestListener.java @@ -18,12 +18,14 @@ package org.apache.sqoop.test.testng; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; import org.testng.ITestResult; import org.testng.TestListenerAdapter; import java.io.FileDescriptor; import java.io.FileOutputStream; import java.io.PrintStream; +import java.io.UnsupportedEncodingException; /** * Sqoop specific listener that will print name of each particular test that being run. @@ -31,14 +33,22 @@ * Particularly suitable for integration tests that generally takes long time to finish. */ public class SqoopTestListener extends TestListenerAdapter { - + private static final Logger LOG = Logger.getLogger(SqoopTestListener.class); /** * Sqoop tests are generally running with redirectTestOutputToFile=true which means * that System.out is redirected to file. That is unpleasant as output user is suppose * to see is not available on the console. Hence instead of System.out we're using * directly the STDOUT file descriptor. */ - static PrintStream ps = new PrintStream(new FileOutputStream(FileDescriptor.out)); + static PrintStream ps; + + static { + try { + ps = new PrintStream(new FileOutputStream(FileDescriptor.out), false, "UTF-8"); + } catch (UnsupportedEncodingException e) { + LOG.warn("Error when initialize the SqoopTestListener.", e); + } + } @Override diff --git a/test/src/main/java/org/apache/sqoop/test/utils/CompressionUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/CompressionUtils.java index d7255c3e..766e2fc9 100644 --- a/test/src/main/java/org/apache/sqoop/test/utils/CompressionUtils.java +++ b/test/src/main/java/org/apache/sqoop/test/utils/CompressionUtils.java @@ -59,17 +59,20 @@ public static void untarStreamToDirectory(InputStream inputStream, String target LOG.info("Untaring file: " + entry.getName()); if (entry.isDirectory()) { - (new File( HdfsUtils.joinPathFragments(targetDirectory, entry.getName()))).mkdirs(); + File targetFile = new File( HdfsUtils.joinPathFragments(targetDirectory, entry.getName())); + if (!targetFile.mkdirs()) { + LOG.warn("Failed to create folder:" + targetFile.getAbsolutePath()); + } } else { int count; byte data[] = new byte[BUFFER_SIZE]; FileOutputStream fos = new FileOutputStream(HdfsUtils.joinPathFragments(targetDirectory, entry.getName())); - BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE); - while ((count = in.read(data, 0, BUFFER_SIZE)) != -1) { - dest.write(data, 0, count); + try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) { + while ((count = in.read(data, 0, BUFFER_SIZE)) != -1) { + dest.write(data, 0, count); + } } - dest.close(); } } in.close(); diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java index 7f83b2df..610156e3 100644 --- a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java +++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java @@ -32,6 +32,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStreamWriter; +import java.nio.charset.Charset; import java.util.LinkedList; /** @@ -43,7 +44,7 @@ public class HdfsUtils { private static final char PATH_SEPARATOR = '/'; - public static PathFilter filterHiddenFiles = new PathFilter() { + public static final PathFilter filterHiddenFiles = new PathFilter() { @Override public boolean accept(Path path) { String fileName = path.getName(); @@ -77,12 +78,13 @@ public boolean accept(Path path) { * @throws IOException */ public static void createFile(FileSystem fs, String path, String ...lines) throws IOException { - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(path), true))); - for (String line : lines) { - writer.write(line); - writer.newLine(); + try (BufferedWriter writer = new BufferedWriter( + new OutputStreamWriter(fs.create(new Path(path), true), Charset.forName("UTF-8")))) { + for (String line : lines) { + writer.write(line); + writer.newLine(); + } } - writer.close(); } /**