mirror of
https://github.com/apache/sqoop.git
synced 2025-05-02 04:10:09 +08:00
416 lines
15 KiB
Groovy
416 lines
15 KiB
Groovy
/**
|
|
* Licensed to the Apache Software Foundation (ASF) under one
|
|
* or more contributor license agreements. See the NOTICE file
|
|
* distributed with this work for additional information
|
|
* regarding copyright ownership. The ASF licenses this file
|
|
* to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance
|
|
* with the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
plugins {
|
|
id "org.nosphere.apache.rat" version "0.3.0"
|
|
}
|
|
apply plugin: 'java'
|
|
apply plugin: 'eclipse'
|
|
apply plugin: 'idea'
|
|
apply plugin: 'checkstyle'
|
|
apply plugin: 'jacoco'
|
|
apply plugin: 'findbugs'
|
|
apply plugin: 'maven-publish'
|
|
|
|
apply from: 'gradle/sqoop-version-gen.gradle'
|
|
apply from: 'gradle/sqoop-package.gradle'
|
|
|
|
group = 'org.apache.sqoopgradle'
|
|
|
|
sourceCompatibility = javaSourceCompatibilityVersion
|
|
|
|
repositories {
|
|
mavenCentral()
|
|
maven {
|
|
url 'http://conjars.org/repo/'
|
|
}
|
|
}
|
|
|
|
sourceSets {
|
|
main {
|
|
java {
|
|
srcDirs = ['src/java']
|
|
}
|
|
resources {
|
|
srcDirs = ['src/java']
|
|
}
|
|
}
|
|
test {
|
|
java {
|
|
srcDirs = ['src/test']
|
|
exclude 'aop/'
|
|
}
|
|
resources {
|
|
srcDirs = ['src/test', 'testdata/hcatalog/conf', 'conf']
|
|
exclude 'aop/'
|
|
}
|
|
}
|
|
aop {
|
|
java {
|
|
srcDirs = ['src/test/aop']
|
|
}
|
|
resources {
|
|
srcDirs = ['src/test/aop']
|
|
}
|
|
}
|
|
}
|
|
|
|
configurations {
|
|
common.extendsFrom(redist)
|
|
compile.extendsFrom(common)
|
|
}
|
|
|
|
compileJava.dependsOn(SqoopVersionFileGen)
|
|
|
|
configurations.all {
|
|
resolutionStrategy {
|
|
force group: 'org.apache.avro', name: 'avro', version: avroVersion
|
|
force group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion
|
|
force group: 'com.google.guava', name: 'guava', version: guavaVersion
|
|
force group: 'com.google.protobuf', name: 'protobuf-java', version: hiveProtobufVersion
|
|
}
|
|
exclude group: 'org.apache.hadoop', module: 'avro'
|
|
}
|
|
|
|
def sqoopThirdPartyLib = System.getProperty("sqoop.thirdparty.lib.dir")
|
|
|
|
dependencies {
|
|
if (sqoopThirdPartyLib != null) runtime fileTree(dir: sqoopThirdPartyLib, include: '*.jar')
|
|
compile group: 'org.apache.hadoop', name: 'hadoop-aws', version: hadoopVersion
|
|
compile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion
|
|
compile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion
|
|
compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: hadoopVersion
|
|
compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: hadoopVersion
|
|
compile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion
|
|
|
|
aopCompile group: 'org.aspectj', name: 'aspectjtools', version: aspectjVersion
|
|
aopCompile group: 'org.aspectj', name: 'aspectjrt', version: aspectjVersion
|
|
aopCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion
|
|
|
|
common group: 'org.apache.accumulo', name: 'accumulo-core', version: accumuloVersion
|
|
common group: 'org.apache.accumulo', name: 'accumulo-minicluster', version: accumuloVersion
|
|
|
|
common group: 'org.eclipse.jetty', name: 'jetty-runner', version: jettyVersion
|
|
|
|
common group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion
|
|
common group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion
|
|
common group: 'org.apache.hbase', name: 'hbase-client', version: hbaseVersion
|
|
common group: 'org.apache.hbase', name: 'hbase-common', version: hbaseVersion
|
|
common(group: 'org.apache.hive.hcatalog', name: 'hive-hcatalog-core', version: hcatalogVersion) {
|
|
exclude group: 'org.apache.avro', module: 'avro'
|
|
exclude group: 'org.apache.hive', module: 'hive-exec'
|
|
}
|
|
common(group: 'org.apache.hive', name: 'hive-exec', version: hcatalogVersion, classifier: 'core') {
|
|
exclude group : 'org.apache.calcite', module :'calcite-core'
|
|
}
|
|
// Kryo and calcite are dependencies of hive-exec:core
|
|
common group: 'com.esotericsoftware', name: 'kryo', version: kryoVersion
|
|
common group: 'org.apache.calcite', name: 'calcite-core', version: calciteVersion
|
|
common(group: 'org.apache.hive', name: 'hive-jdbc', version: hcatalogVersion) {
|
|
exclude group: 'org.apache.avro', module: 'avro'
|
|
exclude group: 'asm', module: 'asm'
|
|
}
|
|
common group: 'commons-cli', name: 'commons-cli', version: commonscliVersion
|
|
common group: 'commons-logging', name: 'commons-logging', version: commonsloggingVersion
|
|
common group: 'commons-net', name: 'commons-net', version: commonsnetVersion
|
|
common group: 'log4j', name: 'log4j', version: log4jVersion
|
|
common group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion
|
|
common group: 'org.apache.parquet', name: 'parquet-hadoop-bundle', version: parquetVersion
|
|
|
|
testCompile group: 'com.h2database', name: 'h2', version: h2Version
|
|
testCompile group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion, classifier: 'tests'
|
|
testCompile group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion, classifier: 'tests'
|
|
testCompile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion, classifier: 'tests'
|
|
testCompile (group: 'org.apache.hadoop', name: 'hadoop-minikdc', version: hadoopVersion) {
|
|
exclude group: 'org.apache.directory.api', module: 'api-ldap-schema-data'
|
|
}
|
|
testCompile group: 'junit', name: 'junit', version: junitVersion
|
|
testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion
|
|
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
|
|
testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'
|
|
}
|
|
|
|
test {
|
|
include '**/Test*.*'
|
|
}
|
|
|
|
task thirdPartyTest(type: Test) {
|
|
description 'Run ThirdParty tests - you need to specify -Dsqoop.thirdparty.lib.dir where the Third party driver jars reside (relative to the project directory)'
|
|
exclude '**/*ManualTest.*'
|
|
exclude '**/Test*.*'
|
|
include '**/*Test*.*'
|
|
systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt"
|
|
systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt"
|
|
systemProperty "ms.datatype.test.data.file.delim" ,","
|
|
systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
|
|
}
|
|
|
|
task manualTest(type: Test) {
|
|
description 'Run manual tests'
|
|
include '**/*ManualTest.*'
|
|
}
|
|
|
|
def testBuildDir = "$buildDir/test/"
|
|
def testBuildDirData ="$testBuildDir/data/"
|
|
|
|
task buildFolder {
|
|
project.mkdir(testBuildDirData)
|
|
}
|
|
|
|
tasks.withType(Test) {
|
|
testLogging {
|
|
events 'started', 'passed', 'skipped'
|
|
}
|
|
workingDir = testBuildDirData
|
|
project.mkdir(workingDir)
|
|
systemProperties(System.getProperties())
|
|
systemProperty "test.data.dir", "$projectDir/testdata"
|
|
systemProperty 'test.build.data', "$testBuildDir/data"
|
|
systemProperty "hadoop.root.logger", "DEBUG,console"
|
|
systemProperty "user.dir", workingDir
|
|
systemProperty "sqoop.src.dir", "$testBuildDir/data"
|
|
systemProperty "hadoop.tmp.dir", "$testBuildDir/hadoop"
|
|
systemProperty "fs.default.name", ""
|
|
systemProperty "hadoop.log.dir", "$testBuildDir/logs"
|
|
systemProperty "hive.home", "$projectDir/testdata/hive"
|
|
systemProperty "sqoop.throwOnError", ""
|
|
|
|
minHeapSize = "512m"
|
|
maxHeapSize = "5120m"
|
|
jvmArgs '-Xmx5012m', "-da:org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge"
|
|
forkEvery 1
|
|
|
|
dependsOn buildFolder
|
|
|
|
jacoco{
|
|
excludes = ["**/SqoopVersion*"]
|
|
}
|
|
}
|
|
|
|
tasks.withType(Checkstyle) {
|
|
reports {
|
|
xml.enabled false
|
|
html.enabled true
|
|
html.stylesheet resources.text.fromFile('config/checkstyle/checkstyle-noframes.xsl')
|
|
}
|
|
ignoreFailures = true
|
|
}
|
|
|
|
tasks.withType(FindBugs) {
|
|
reports {
|
|
xml.enabled false
|
|
html.enabled true
|
|
}
|
|
ignoreFailures = false
|
|
}
|
|
|
|
findbugs {
|
|
sourceSets = []
|
|
}
|
|
|
|
checkstyle {
|
|
sourceSets = []
|
|
toolVersion checkstyleVersion
|
|
}
|
|
|
|
jacocoTestReport {
|
|
description "Generates a coverage report of the unit test results under $buildDir/jacocoHtml/test/index.html"
|
|
reports {
|
|
xml.enabled true
|
|
csv.enabled false
|
|
html.destination file("$buildDir/jacocoHtml/test")
|
|
}
|
|
}
|
|
|
|
task jacocoThirdPartyReport(type: JacocoReport){
|
|
description "Generates a coverage report of the thirdparty test results under $buildDir/jacocoHtml/3rd/index.html"
|
|
sourceSets sourceSets.main
|
|
executionData thirdPartyTest
|
|
reports {
|
|
xml.enabled true
|
|
csv.enabled false
|
|
html.destination file("$buildDir/jacocoHtml/3rd")
|
|
}
|
|
}
|
|
|
|
task jacocoCompositeReport(type: JacocoReport){
|
|
description "Generates a composite coverage report of test and thirdparty test results under $buildDir/jacocoHtml/index.html"
|
|
sourceSets sourceSets.main
|
|
executionData test, thirdPartyTest
|
|
reports {
|
|
xml.enabled true
|
|
csv.enabled false
|
|
html.destination file("$buildDir/jacocoHtml/")
|
|
}
|
|
}
|
|
|
|
javadoc {
|
|
source = sourceSets.main.allJava
|
|
include 'com/cloudera/sqoop/lib/**'
|
|
include 'org/apache/sqoop/lib/**'
|
|
destinationDir = reporting.file("${project.docsDir}/api")
|
|
}
|
|
|
|
task faultInjectionJar(type: Jar) {
|
|
description 'Create the test jar'
|
|
appendix = 'test'
|
|
from sourceSets.aop.output
|
|
}
|
|
|
|
def archiveName = jar.baseName + "-" + jar.version
|
|
def binArtifactName = archiveName + ".bin__hadoop-" + hadoopVersion
|
|
def distDir = "$buildDir/$binArtifactName"
|
|
|
|
rat {
|
|
inputDir = distDir
|
|
reportDir = project.file(buildDir)
|
|
excludes = ['**/docs/**', '**/testdata/**', '**/.idea/**', '**/test-dir/**', '**/.settings/**', '**/.gradle/**','**/gradle/**' ,'**/gradle/build/**', '*.iml', '*.iws', '*.ipr', 'derby.log', '**/gradlew**' ]
|
|
failOnError = true
|
|
xmlOutput = false
|
|
htmlOutput = true
|
|
plainOutput = false
|
|
}
|
|
rat.dependsOn('packageDist')
|
|
|
|
def scripts = ['export', 'list-databases', 'metastore', 'create-hive-table', 'help',
|
|
'import-mainframe', 'list-tables', 'version',
|
|
'eval', 'import', 'job', 'merge',
|
|
'import-all-tables', 'codegen']
|
|
|
|
|
|
task runSqoopHelp(type: JavaExec, dependsOn: 'classes') {
|
|
main = 'org.apache.sqoop.Sqoop'
|
|
jvmArgs = ['-Dhadoop.security.log.file=./build/security-audit.log']
|
|
classpath = sourceSets.main.runtimeClasspath + configurations.runtime
|
|
args = ['help']
|
|
standardOutput = new ByteArrayOutputStream()
|
|
ext.helpOut = {
|
|
standardOutput.toString()
|
|
}
|
|
|
|
}
|
|
|
|
task createAllStartScripts(dependsOn: 'runSqoopHelp' ) doLast{
|
|
// Placeholder task with validation
|
|
// We call Sqoop help to verify that all tasks that we create scripts for are present in Sqoop help and vice-versa
|
|
// We currently depend on the format of the help output to parse the relevant information out
|
|
def matcher = (tasks.runSqoopHelp.helpOut() =~/(?m)^\s\s([a-z-]+)\s+\w+/)
|
|
def matches = new String[matcher.getCount()]
|
|
for (def i=0; i < matcher.getCount(); i++) {
|
|
matches[i] = matcher[i][1]
|
|
}
|
|
if (matches.sort() != scripts.sort())
|
|
throw new GradleException('Error: script list in gradle build script and result of Sqoop help are not the same, but should be')
|
|
}
|
|
|
|
scripts.each() { scriptName ->
|
|
def t = tasks.create(name: scriptName + 'StartScript', type: CreateStartScripts) {
|
|
unixStartScriptGenerator.template = resources.text.fromFile('gradle/customUnixStartScript.txt')
|
|
windowsStartScriptGenerator.template = resources.text.fromFile('gradle/customWindowsStartScript.txt')
|
|
outputDir = file("$buildDir/bin")
|
|
applicationName = scriptName
|
|
mainClassName = 'com.cloudera.sqoop.Sqoop'
|
|
classpath = project.configurations.runtime
|
|
doLast {
|
|
file("$buildDir/bin/"+ scriptName).renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName))
|
|
file("$buildDir/bin/"+ scriptName +".bat").renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName+".bat"))
|
|
}
|
|
}
|
|
t.dependsOn('compileJava')
|
|
createAllStartScripts.dependsOn(t)
|
|
}
|
|
|
|
task checkVersion doLast {
|
|
if (version.contains('SNAPSHOT'))
|
|
throw new GradleException('Error: cannot release a snapshot. Set -Pversion')
|
|
}
|
|
|
|
task release(dependsOn: ['checkVersion', 'tar', 'rat']) {
|
|
doLast {
|
|
println 'Release complete'
|
|
println "Binary tar: $buildDir/$binArtifactName-${version}.tar.gz"
|
|
println "Documentation: $buildDir/docs"
|
|
println "Release notes: $buildDir/docs/sqoop-$version" + ".releasenotes.html"
|
|
println "Release audit report: $buildDir/index.html"
|
|
}
|
|
}
|
|
|
|
task relnotes(type: Exec) {
|
|
workingDir 'src/scripts'
|
|
if (!version.contains('SNAPSHOT')) {
|
|
commandLine "python", "relnotes.py", "$buildDir/docs", "$projectDir", "$oldHash\\..HEAD", "$version", "$oldVersion"
|
|
} else {
|
|
commandLine "true" //noop
|
|
println "Will not run releasenotes for SNAPSHOT version."
|
|
}
|
|
}
|
|
|
|
task realDocs(type: Exec, dependsOn: ['relnotes', 'javadoc']) {
|
|
workingDir projectDir
|
|
commandLine "make", "-C", "$projectDir/src/docs", "BUILDROOT=$buildDir", "VERSION=$version"
|
|
}
|
|
|
|
task docs(dependsOn: ['realDocs', 'relnotes', 'javadoc']) {
|
|
//placeholder
|
|
}
|
|
def $mvnRepo = System.getProperty("mvnRepo", "snapshots")
|
|
|
|
publishing {
|
|
repositories {
|
|
maven {
|
|
name "cloudera." + $mvnRepo + ".repo"
|
|
url "https://repository.cloudera.com/content/repositories/"+$mvnRepo
|
|
credentials {
|
|
username 'username'
|
|
password 'password'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
publishing {
|
|
publications {
|
|
sqoop(MavenPublication) {
|
|
groupId group
|
|
version version
|
|
from components.java
|
|
artifact sourceJar {
|
|
classifier "sources"
|
|
}
|
|
}
|
|
sqoopTest(MavenPublication) {
|
|
groupId group
|
|
version version
|
|
artifactId 'sqoop-test'
|
|
artifact testJar
|
|
artifact sourceTestJar
|
|
pom.withXml {
|
|
def dependencies = asNode().appendNode('dependencies')
|
|
configurations.testRuntime.getResolvedConfiguration().getFirstLevelModuleDependencies().each {
|
|
def dependency = dependencies.appendNode('dependency')
|
|
dependency.appendNode('groupId', it.moduleGroup)
|
|
dependency.appendNode('artifactId', it.moduleName)
|
|
dependency.appendNode('version', it.moduleVersion)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
wrapper {
|
|
gradleVersion = '4.9'
|
|
}
|