/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ plugins { id "org.nosphere.apache.rat" version "0.3.0" } apply plugin: 'java' apply plugin: 'eclipse' apply plugin: 'idea' apply plugin: 'checkstyle' apply plugin: 'jacoco' apply plugin: 'findbugs' apply plugin: 'maven-publish' apply from: 'gradle/sqoop-version-gen.gradle' apply from: 'gradle/sqoop-package.gradle' group = 'org.apache.sqoopgradle' sourceCompatibility = javaSourceCompatibilityVersion repositories { mavenCentral() maven { url 'http://conjars.org/repo/' } maven { url 'http://maven.cubrid.org/' } // DB2 driver repository maven { url 'https://artifacts.alfresco.com/nexus/content/repositories/public/' } // Oracle driver repository maven { url 'http://www.datanucleus.org/downloads/maven2/' } } sourceSets { main { java { srcDirs = ['src/java'] } resources { srcDirs = ['src/java'] } } test { java { srcDirs = ['src/test'] exclude 'aop/' } resources { srcDirs = ['src/test', 'testdata/hcatalog/conf', 'conf'] exclude 'aop/' } } aop { java { srcDirs = ['src/test/aop'] } resources { srcDirs = ['src/test/aop'] } } } configurations { common.extendsFrom(redist) compile.extendsFrom(common) } compileJava.dependsOn(SqoopVersionFileGen) compileJava { options.compilerArgs += '-proc:none' options.compilerArgs += '-Xlint:all' } compileTestJava { options.compilerArgs += '-proc:none' options.compilerArgs += '-Xlint:all' } configurations.all { resolutionStrategy { force group: 'org.apache.avro', name: 'avro', version: avroVersion force group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion force group: 'com.google.guava', name: 'guava', version: guavaVersion force group: 'com.google.protobuf', name: 'protobuf-java', version: hiveProtobufVersion } exclude group: 'org.apache.hadoop', module: 'avro' } def forkEveryDefault = Integer.valueOf(System.getProperty("forkEvery.default", "0")) def ignoreTestFailures = Boolean.getBoolean("ignoreTestFailures") dependencies { compile group: 'org.apache.hadoop', name: 'hadoop-aws', version: hadoopVersion compile (group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion) { exclude group: "asm", module: "asm" } compile (group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion) { exclude group: "asm", module: "asm" } compile (group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: hadoopVersion) { exclude group: "asm", module: "asm" } compile (group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: hadoopVersion) { exclude group: "asm", module: "asm" } compile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion aopCompile group: 'org.aspectj', name: 'aspectjtools', version: aspectjVersion aopCompile group: 'org.aspectj', name: 'aspectjrt', version: aspectjVersion aopCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion common group: 'org.apache.accumulo', name: 'accumulo-core', version: accumuloVersion common group: 'org.apache.accumulo', name: 'accumulo-minicluster', version: accumuloVersion common group: 'org.eclipse.jetty', name: 'jetty-runner', version: jettyVersion common group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion common group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion common group: 'org.apache.hbase', name: 'hbase-client', version: hbaseVersion common group: 'org.apache.hbase', name: 'hbase-common', version: hbaseVersion common(group: 'org.apache.hive.hcatalog', name: 'hive-hcatalog-core', version: hcatalogVersion) { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.hive', module: 'hive-exec' } common(group: 'org.apache.hive', name: 'hive-exec', version: hcatalogVersion, classifier: 'core') { exclude group : 'org.apache.calcite', module :'calcite-core' } // Kryo and calcite are dependencies of hive-exec:core common group: 'com.esotericsoftware', name: 'kryo', version: kryoVersion common group: 'org.apache.calcite', name: 'calcite-core', version: calciteVersion common(group: 'org.apache.hive', name: 'hive-jdbc', version: hcatalogVersion) { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'asm', module: 'asm' } common group: 'commons-cli', name: 'commons-cli', version: commonscliVersion common group: 'commons-logging', name: 'commons-logging', version: commonsloggingVersion common group: 'commons-net', name: 'commons-net', version: commonsnetVersion common group: 'log4j', name: 'log4j', version: log4jVersion common group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion common group: 'org.apache.parquet', name: 'parquet-hadoop-bundle', version: parquetVersion testCompile group: 'com.h2database', name: 'h2', version: h2Version testCompile group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion, classifier: 'tests' testCompile group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion, classifier: 'tests' testCompile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion, classifier: 'tests' testCompile (group: 'org.apache.hadoop', name: 'hadoop-minikdc', version: hadoopVersion) { exclude group: 'org.apache.directory.api', module: 'api-ldap-schema-data' } testCompile group: 'junit', name: 'junit', version: junitVersion testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar' // JDBC drivers testCompile group: 'mysql', name: 'mysql-connector-java', version: mySqlConnectorVersion testCompile group: 'cubrid', name: 'cubrid-jdbc', version: cubridJdbcDriverVersion testCompile group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: msSqlJdbcDriverVersion testCompile group: 'com.ibm.db2', name: 'jcc', version: db2JdbcDriverVersion testCompile group: 'oracle', name: 'ojdbc6', version: oracleJdbcDriverVersion } task unitTest (type: Test) { description 'Run unit tests which do not need proprietary JDBC driver' useJUnit { includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest' excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest' } } task integrationTest(type: Test) { description 'Run integration tests which do not need a docker container or an external database/service' useJUnit { includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest' } } task thirdPartyTest (type: Test) { description 'Run Third-party Tests - you need to specify -Dsqoop.thirdparty.lib.dir where the Third party driver ' + 'jars reside (relative to the project directory)' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt" systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt" systemProperty "ms.datatype.test.data.file.delim" ,"," systemProperty "ms.datatype.test.hdfsprefix" ,"file:///" } task postgreSqlTest (type: Test) { description 'Run Postgresql third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task mySqlTest (type: Test) { description 'Run MySql third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MysqlTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task cubridTest (type: Test) { description 'Run Cubrid third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.CubridTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task db2Test (type: Test) { description 'Run DB2 third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.Db2Test' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task mainFrameTest (type: Test) { description 'Run Mainframe third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MainFrameTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task oracleTest (type: Test) { description 'Run Oracle third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task oracleXeTest (type: Test) { description 'Run Oracle third party tests which can be run with Oracle Express Edition too.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest' excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task oracleEeTest (type: Test) { description 'Run Oracle third party tests which require Oracle Enterprise Edition.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } } task sqlServerTest (type: Test) { description 'Run MS SqlServer third party tests.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt" systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt" systemProperty "ms.datatype.test.data.file.delim" ,"," systemProperty "ms.datatype.test.hdfsprefix" ,"file:///" } test { description 'Run tests that do not need external JDBC driver and/or a docker container. ' + 'This is the same as running unitTest, integrationTest and kerberizedTest.' useJUnit { includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest' includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' } } task s3Test(type: Test) { description 'Run S3 tests' useJUnit { includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.S3Test' } } task allTest (type: Test){ description 'Run all Sqoop tests' useJUnit { includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest' includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest' includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest' excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest' excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest' } } def testBuildDir = "$buildDir/test/" def testBuildDirData ="$testBuildDir/data/" tasks.withType(Test) { doFirst { project.mkdir(testBuildDir) project.mkdir(testBuildDirData) } testLogging { events 'passed', 'skipped' } workingDir = testBuildDirData systemProperties(System.getProperties()) systemProperty "test.data.dir", "$projectDir/testdata" systemProperty 'test.build.data', "$testBuildDir/data" systemProperty "hadoop.root.logger", "DEBUG,console" systemProperty "user.dir", workingDir systemProperty "sqoop.src.dir", "$testBuildDir/data" systemProperty "hadoop.tmp.dir", "$testBuildDir/hadoop" systemProperty "fs.default.name", "" systemProperty "hadoop.log.dir", "$testBuildDir/logs" systemProperty "hive.home", "$projectDir/testdata/hive" systemProperty "sqoop.throwOnError", "" minHeapSize = "512m" maxHeapSize = "8G" jvmArgs "-da:org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge" jacoco{ excludes = ["**/SqoopVersion*"] } forkEvery forkEveryDefault ignoreFailures ignoreTestFailures } task kerberizedTest (type: Test){ description 'Run Kerberized Test' // A Gradle test task with forkEvery 1 and includeCategories performs poorly because it starts a new JVM even for the filtered tests. // To work around this performance problem we need to add every kerberized test in a separate include field here. include '**/TestKerberosAuthenticator*' include '**/HBaseKerberizedConnectivityTest*' include '**/TestHiveMiniCluster*' include '**/TestHiveServer2TextImport*' useJUnit { includeCategories 'org.apache.sqoop.testcategories.KerberizedTest' } forkEvery 1 ignoreFailures ignoreTestFailures } test.finalizedBy(kerberizedTest) allTest.finalizedBy(kerberizedTest) tasks.withType(Checkstyle) { reports { xml.enabled false html.enabled true html.stylesheet resources.text.fromFile('config/checkstyle/checkstyle-noframes.xsl') } ignoreFailures = true } tasks.withType(FindBugs) { reports { xml.enabled false html.enabled true } ignoreFailures = false } findbugs { sourceSets = [] } checkstyle { sourceSets = [] toolVersion checkstyleVersion } jacocoTestReport { description "Generates a coverage report of the unit test results under $buildDir/jacocoHtml/test/index.html" executionData test, kerberizedTest reports { xml.enabled true csv.enabled false html.destination file("$buildDir/jacocoHtml/test") } } task jacocoThirdPartyReport(type: JacocoReport){ description "Generates a coverage report of the thirdparty test results under $buildDir/jacocoHtml/3rd/index.html" sourceSets sourceSets.main executionData thirdPartyTest reports { xml.enabled true csv.enabled false html.destination file("$buildDir/jacocoHtml/3rd") } } task jacocoCompositeReport(type: JacocoReport){ description "Generates a composite coverage report of test and thirdparty test results under $buildDir/jacocoHtml/index.html" sourceSets sourceSets.main executionData test, kerberizedTest, thirdPartyTest reports { xml.enabled true csv.enabled false html.destination file("$buildDir/jacocoHtml/") } } javadoc { source = sourceSets.main.allJava include 'com/cloudera/sqoop/lib/**' include 'org/apache/sqoop/lib/**' destinationDir = reporting.file("${project.docsDir}/api") } task faultInjectionJar(type: Jar) { description 'Create the test jar' appendix = 'test' from sourceSets.aop.output } def archiveName = jar.baseName + "-" + jar.version def binArtifactName = archiveName + ".bin__hadoop-" + hadoopVersion def distDir = "$buildDir/$binArtifactName" rat { inputDir = distDir reportDir = project.file(buildDir) excludes = ['**/docs/**', '**/testdata/**', '**/.idea/**', '**/test-dir/**', '**/.settings/**', '**/.gradle/**','**/gradle/**' ,'**/gradle/build/**', '*.iml', '*.iws', '*.ipr', 'derby.log', '**/gradlew**' ] failOnError = true xmlOutput = false htmlOutput = true plainOutput = false } rat.dependsOn('packageDist') def scripts = ['export', 'list-databases', 'metastore', 'create-hive-table', 'help', 'import-mainframe', 'list-tables', 'version', 'eval', 'import', 'job', 'merge', 'import-all-tables', 'codegen'] task runSqoopHelp(type: JavaExec, dependsOn: 'classes') { main = 'org.apache.sqoop.Sqoop' jvmArgs = ['-Dhadoop.security.log.file=./build/security-audit.log'] classpath = sourceSets.main.runtimeClasspath + configurations.runtime args = ['help'] standardOutput = new ByteArrayOutputStream() ext.helpOut = { standardOutput.toString() } } task createAllStartScripts(dependsOn: 'runSqoopHelp' ) doLast{ // Placeholder task with validation // We call Sqoop help to verify that all tasks that we create scripts for are present in Sqoop help and vice-versa // We currently depend on the format of the help output to parse the relevant information out def matcher = (tasks.runSqoopHelp.helpOut() =~/(?m)^\s\s([a-z-]+)\s+\w+/) def matches = new String[matcher.getCount()] for (def i=0; i < matcher.getCount(); i++) { matches[i] = matcher[i][1] } if (matches.sort() != scripts.sort()) throw new GradleException('Error: script list in gradle build script and result of Sqoop help are not the same, but should be') } scripts.each() { scriptName -> def t = tasks.create(name: scriptName + 'StartScript', type: CreateStartScripts) { unixStartScriptGenerator.template = resources.text.fromFile('gradle/customUnixStartScript.txt') windowsStartScriptGenerator.template = resources.text.fromFile('gradle/customWindowsStartScript.txt') outputDir = file("$buildDir/bin") applicationName = scriptName mainClassName = 'com.cloudera.sqoop.Sqoop' classpath = project.configurations.runtime doLast { file("$buildDir/bin/"+ scriptName).renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName)) file("$buildDir/bin/"+ scriptName +".bat").renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName+".bat")) } } t.dependsOn('compileJava') createAllStartScripts.dependsOn(t) } task checkVersion doLast { if (version.contains('SNAPSHOT')) throw new GradleException('Error: cannot release a snapshot. Set -Pversion') } task release(dependsOn: ['checkVersion', 'tar', 'rat']) { doLast { println 'Release complete' println "Binary tar: $buildDir/$binArtifactName-${version}.tar.gz" println "Documentation: $buildDir/docs" println "Release notes: $buildDir/docs/sqoop-$version" + ".releasenotes.html" println "Release audit report: $buildDir/index.html" } } task relnotes(type: Exec) { workingDir 'src/scripts' if (!version.contains('SNAPSHOT')) { commandLine "python", "relnotes.py", "$buildDir/docs", "$projectDir", "$oldHash\\..HEAD", "$version", "$oldVersion" } else { commandLine "true" //noop println "Will not run releasenotes for SNAPSHOT version." } } task realDocs(type: Exec, dependsOn: ['relnotes', 'javadoc']) { workingDir projectDir commandLine "make", "-C", "$projectDir/src/docs", "BUILDROOT=$buildDir", "VERSION=$version" } task docs(dependsOn: ['realDocs', 'relnotes', 'javadoc']) { //placeholder } def $mvnRepo = System.getProperty("mvnRepo", "snapshots") publishing { repositories { maven { name "cloudera." + $mvnRepo + ".repo" url "https://repository.cloudera.com/content/repositories/"+$mvnRepo credentials { username 'username' password 'password' } } } } publishing { publications { sqoop(MavenPublication) { groupId group version version from components.java artifact sourceJar { classifier "sources" } } sqoopTest(MavenPublication) { groupId group version version artifactId 'sqoop-test' artifact testJar artifact sourceTestJar pom.withXml { def dependencies = asNode().appendNode('dependencies') configurations.testRuntime.getResolvedConfiguration().getFirstLevelModuleDependencies().each { def dependency = dependencies.appendNode('dependency') dependency.appendNode('groupId', it.moduleGroup) dependency.appendNode('artifactId', it.moduleName) dependency.appendNode('version', it.moduleVersion) } } } } } wrapper { gradleVersion = '4.9' }