From a0ebf8f299feb3defb071186b9f44df489b6b76e Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Sat, 3 Oct 2015 06:50:23 -0700 Subject: [PATCH] SQOOP-2599: Sqoop2: Classutils behaves badly when the classpath changes. (Abraham Fine via Jarek Jarcec Cecho) --- .../org/apache/sqoop/utils/ClassUtils.java | 42 +++- .../apache/sqoop/utils/TestClassUtils.java | 203 +++++++++++++----- common/src/test/resources/TestJar/A.java | 40 ++++ common/src/test/resources/TestJar/Child.java | 19 ++ common/src/test/resources/TestJar/Parent.java | 19 ++ .../org/apache/sqoop/core/SqoopServer.java | 3 +- .../apache/sqoop/driver/TestJobRequest.java | 4 +- 7 files changed, 273 insertions(+), 57 deletions(-) create mode 100644 common/src/test/resources/TestJar/A.java create mode 100644 common/src/test/resources/TestJar/Child.java create mode 100644 common/src/test/resources/TestJar/Parent.java diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java index 3c073ca9..ca438486 100644 --- a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java +++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java @@ -17,19 +17,24 @@ */ package org.apache.sqoop.utils; +import org.apache.log4j.Logger; +import org.apache.sqoop.classification.InterfaceAudience; +import org.apache.sqoop.classification.InterfaceStability; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.error.code.CoreError; + +import java.io.IOException; import java.lang.ref.WeakReference; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLDecoder; import java.util.ArrayList; import java.util.Collections; import java.util.Map; import java.util.WeakHashMap; -import org.apache.sqoop.classification.InterfaceAudience; -import org.apache.sqoop.classification.InterfaceStability; -import org.apache.log4j.Logger; - @InterfaceAudience.Public @InterfaceStability.Unstable public final class ClassUtils { @@ -209,11 +214,29 @@ public static String jarForClassWithClassLoader(String className, ClassLoader lo /** * Return jar path for given class. * + * This method is based on the getJar method in Hadoop's JarFinder + * * @param klass Class object * @return Path on local filesystem to jar where given jar is present */ public static String jarForClass(Class klass) { - return klass.getProtectionDomain().getCodeSource().getLocation().toString(); + String jarPath = null; + String class_file = klass.getName().replaceAll("\\.", "/") + ".class"; + try { + URL url = defaultClassLoader.getResource(class_file); + String path = url.getPath(); + path = URLDecoder.decode(path, "UTF-8"); + if ("jar".equals(url.getProtocol())) { + jarPath = path.replaceAll("!.*$", ""); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + if (jarPath == null) { + throw new SqoopException(CoreError.CORE_0009, "failed to find jar for " + + "class: " + klass.getName()); + } + return jarPath; } /** @@ -243,6 +266,15 @@ public static String[] getEnumStrings(Class klass) { return values.toArray(new String[values.size()]); } + + public static void clearCache() { + CACHE_CLASSES.clear(); + } + + public static void setDefaultClassLoader(ClassLoader classLoader) { + defaultClassLoader = classLoader; + } + private ClassUtils() { // Disable explicit object creation } diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java index eca35055..8851440c 100644 --- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java +++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java @@ -17,10 +17,27 @@ */ package org.apache.sqoop.utils; -import java.util.Arrays; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.lang.reflect.Field; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import javax.tools.*; + import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; @@ -31,16 +48,130 @@ */ public class TestClassUtils { + private ClassLoader classLoader; + private Path outputDir; + private Class testAClass; + private Class testParentClass; + private Class testChildClass; + + @BeforeMethod + public void captureClassLoader() throws Exception { + classLoader = Thread.currentThread().getContextClassLoader(); + File jarFile = compileAJar(); + URL[] urlArray = { jarFile.toURI().toURL() }; + URLClassLoader newClassLoader = new URLClassLoader(urlArray, classLoader); + ClassUtils.setDefaultClassLoader(newClassLoader); + testAClass = newClassLoader.loadClass("A"); + testParentClass = newClassLoader.loadClass("Parent"); + testChildClass = newClassLoader.loadClass("Child"); + } + + private File compileAJar() throws Exception{ + String jarName = "test-jar.jar"; + + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new IllegalStateException( + "Cannot find the system Java compiler. " + + "Check that your class path includes tools.jar"); + } + + outputDir = Files.createTempDirectory(null); + + ClassLoader classLoader = getClass().getClassLoader(); + List sourceFiles = new ArrayList<>(); + File sourceFile = new File(classLoader.getResource("TestJar/A.java").getFile()); + sourceFiles.add(sourceFile); + sourceFile = new File(classLoader.getResource("TestJar/Child.java").getFile()); + sourceFiles.add(sourceFile); + sourceFile = new File(classLoader.getResource("TestJar/Parent.java").getFile()); + sourceFiles.add(sourceFile); + + StandardJavaFileManager fileManager = compiler.getStandardFileManager + (null, null, null); + + fileManager.setLocation(StandardLocation.CLASS_OUTPUT, + Arrays.asList(new File(outputDir.toString()))); + + Iterable compilationUnits1 = + fileManager.getJavaFileObjectsFromFiles(sourceFiles); + + boolean compiled = compiler.getTask(null, fileManager, null, null, null, compilationUnits1).call(); + if (!compiled) { + throw new RuntimeException("failed to compile"); + } + + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, "."); + + JarOutputStream target = new JarOutputStream(new FileOutputStream(outputDir.toString() + File.separator + jarName), manifest); + + List classesForJar = new ArrayList<>(); + //split the file on dot to get the filename from FILENAME.java + for (File source : sourceFiles) { + String fileName = source.getName().split("\\.")[0]; + classesForJar.add(fileName); + } + + File[] directoryListing = outputDir.toFile().listFiles(); + for (File compiledClass : directoryListing) { + String classFileName = compiledClass.getName().split("\\$")[0].split("\\.")[0]; + if (classesForJar.contains(classFileName)){ + addFileToJar(compiledClass, target); + } + } + + target.close(); + + //delete non jar files + for (File file : outputDir.toFile().listFiles()) { + String extension = file.getName().split("\\.")[1]; + if (!extension.equals("jar")) { + file.delete(); + } + } + return new File(outputDir.toString() + File.separator + jarName); + } + + private void addFileToJar(File source, JarOutputStream target) throws Exception { + JarEntry entry = new JarEntry(source.getName()); + entry.setTime(source.lastModified()); + target.putNextEntry(entry); + BufferedInputStream in = new BufferedInputStream(new FileInputStream(source)); + + long bufferSize = source.length(); + if (bufferSize < Integer.MIN_VALUE || bufferSize > Integer.MAX_VALUE) { + throw new RuntimeException("file to large to be added to jar"); + } + + byte[] buffer = new byte[(int) bufferSize]; + while (true) { + int count = in.read(buffer); + if (count == -1) + break; + target.write(buffer, 0, count); + } + target.closeEntry(); + if (in != null) in.close(); + } + + @AfterMethod + public void restoreClassLoader() { + Thread.currentThread().setContextClassLoader(classLoader); + ClassUtils.setDefaultClassLoader(classLoader); + } + @Test public void testLoadClass() { - assertNull(ClassUtils.loadClass("A")); - assertEquals(A.class, ClassUtils.loadClass(A.class.getName())); + assertNull(ClassUtils.loadClass("IDONTEXIST")); + assertEquals(testAClass, ClassUtils.loadClass(testAClass.getName())); } @Test public void testLoadClassWithClassLoader() throws Exception { - String classpath = ClassUtils.jarForClass(A.class); - assertNotEquals(A.class, ClassUtils.loadClassWithClassLoader(A.class.getName(), + String classpath = ClassUtils.jarForClass(testAClass); + assertNotEquals(testAClass, ClassUtils.loadClassWithClassLoader(testAClass.getName(), new ConnectorClassLoader(classpath, getClass().getClassLoader(), Arrays.asList("java.")))); } @@ -50,20 +181,25 @@ public void testInstantiateNull() { } @Test - public void testInstantiate() { + public void testInstantiate() throws Exception { // Just object calls - A a = (A) ClassUtils.instantiate(A.class, "a"); + Object a = ClassUtils.instantiate(testAClass, "a"); + Field numField = testAClass.getField("num"); + Field aField = testAClass.getField("a"); + Field bField = testAClass.getField("b"); + Field cField = testAClass.getField("c"); + Field pField = testAClass.getField("p"); assertNotNull(a); - assertEquals(1, a.num); - assertEquals("a", a.a); + assertEquals(1, numField.get(a)); + assertEquals("a", aField.get(a)); // Automatic wrapping primitive -> objects - A b = (A) ClassUtils.instantiate(A.class, "b", 3, 5); + Object b = ClassUtils.instantiate(testAClass, "b", 3, 5); assertNotNull(b); - assertEquals(3, b.num); - assertEquals("b", b.a); - assertEquals(3, b.b); - assertEquals(5, b.c); + assertEquals(3, numField.get(b)); + assertEquals("b", aField.get(b)); + assertEquals(3, bField.get(b)); + assertEquals(5, cField.get(b)); // Primitive types in the constructor definition Primitive p = (Primitive) ClassUtils.instantiate(Primitive.class, 1, 1.0f, true); @@ -73,41 +209,10 @@ public void testInstantiate() { assertEquals(true, p.b); // Subclasses can be used in the constructor call - A c = (A) ClassUtils.instantiate(A.class, new Child()); + Object c = ClassUtils.instantiate(testAClass, ClassUtils.instantiate(testChildClass)); assertNotNull(c); - assertNotNull(c.p); - assertEquals(Child.class, c.p.getClass()); - } - - public static class Parent { - } - - public static class Child extends Parent { - } - - - public static class A { - String a; - int b; - int c; - int num; - Parent p; - - public A(String a) { - num = 1; - this.a = a; - } - public A(String a, Integer b, Integer c) { - this(a); - - num = 3; - this.b = b; - this.c = c; - } - - public A(Parent p) { - this.p = p; - } + assertNotNull(pField.get(c)); + assertEquals(testChildClass, pField.get(c).getClass()); } public static class Primitive { @@ -124,7 +229,7 @@ public Primitive(int i, float f, boolean b) { @Test public void testGetEnumStrings() { - assertEquals(new String[]{}, ClassUtils.getEnumStrings(A.class)); + assertEquals(new String[]{}, ClassUtils.getEnumStrings(testAClass)); assertEquals( new String[]{"A", "B", "C"}, diff --git a/common/src/test/resources/TestJar/A.java b/common/src/test/resources/TestJar/A.java new file mode 100644 index 00000000..7fd0f329 --- /dev/null +++ b/common/src/test/resources/TestJar/A.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class A { + public String a; + public int b; + public int c; + public int num; + public Parent p; + + public A(String a) { + num = 1; + this.a = a; + } + public A(String a, Integer b, Integer c) { + this(a); + + num = 3; + this.b = b; + this.c = c; + } + + public A(Parent p) { + this.p = p; + } +} \ No newline at end of file diff --git a/common/src/test/resources/TestJar/Child.java b/common/src/test/resources/TestJar/Child.java new file mode 100644 index 00000000..3e76ef1d --- /dev/null +++ b/common/src/test/resources/TestJar/Child.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class Child extends Parent { +} \ No newline at end of file diff --git a/common/src/test/resources/TestJar/Parent.java b/common/src/test/resources/TestJar/Parent.java new file mode 100644 index 00000000..3c3f26a7 --- /dev/null +++ b/common/src/test/resources/TestJar/Parent.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class Parent { +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java index 555728c1..80a7b88a 100644 --- a/core/src/main/java/org/apache/sqoop/core/SqoopServer.java +++ b/core/src/main/java/org/apache/sqoop/core/SqoopServer.java @@ -19,13 +19,13 @@ import org.apache.log4j.Logger; import org.apache.sqoop.audit.AuditLoggerManager; -import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.ConnectorManager; import org.apache.sqoop.driver.Driver; import org.apache.sqoop.driver.JobManager; import org.apache.sqoop.repository.RepositoryManager; import org.apache.sqoop.security.AuthenticationManager; import org.apache.sqoop.security.AuthorizationManager; +import org.apache.sqoop.utils.ClassUtils; /** * Entry point for initializing and destroying Sqoop server @@ -44,6 +44,7 @@ public static void destroy() { AuthorizationManager.getInstance().destroy(); AuthenticationManager.getInstance().destroy(); SqoopConfiguration.getInstance().destroy(); + ClassUtils.clearCache(); LOG.info("Sqoop server has been correctly terminated"); } diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java index 0a861129..7586b847 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java @@ -51,9 +51,9 @@ public void testAddJar() { @Test public void testAddJarForClass() { - jobRequest.addJarForClass(TestJobRequest.class); + jobRequest.addJarForClass(String.class); assertEquals(1, jobRequest.getJars().size()); - assertTrue(jobRequest.getJars().contains(ClassUtils.jarForClass(TestJobRequest.class))); + assertTrue(jobRequest.getJars().contains(ClassUtils.jarForClass(String.class))); } @Test