github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 18:30:15 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-3275: HBase test cases should start mini DFS cluster as well

Browse Source 
(Szabolcs Vasas via Boglarka Egyed)
This commit is contained in:
Boglarka Egyed 2018-01-08 18:08:37 +01:00
parent 010042dd47
commit bf8b8a6acd
2 changed files with 32 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ivy.xml
View File

@ -225,7 +225,6 @@ under the License.
</dependency>
<exclude org="org.apache.hadoop" module="avro"/>
<exclude org="commons-daemon" module="commons-daemon" />
<exclude type="pom" />
</dependencies>

39
src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
View File

@ -24,6 +24,16 @@
import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.KRB_PRINCIPAL;
import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.MASTER_KRB_PRINCIPAL;
import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_CONF_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PRINCIPAL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@ -49,6 +59,7 @@
import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
import org.apache.hadoop.hbase.security.token.TokenProvider;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.infrastructure.kerberos.KerberosConfigurationProvider;
@ -66,9 +77,8 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
public static final Log LOG = LogFactory.getLog(
HBaseTestCase.class.getName());
private static final int NUM_MASTERS = 1;
private static final int NUM_SLAVES = 1;
private static final String MASTER_INFO_PORT_DISABLE_WEB_UI = "-1";
private static final String DEFAULT_DFS_HTTPS_ADDRESS = "localhost:0";
private final KerberosConfigurationProvider kerberosConfigurationProvider;
private HBaseTestingUtility hbaseTestUtil;
@ -163,8 +173,7 @@ public void setUp() {
hbaseTestUtil.getConfiguration().set(MASTER_INFO_PORT, MASTER_INFO_PORT_DISABLE_WEB_UI);
setupKerberos();
hbaseTestUtil.startMiniZKCluster();
hbaseTestUtil.startMiniHBaseCluster(NUM_MASTERS, NUM_SLAVES);
hbaseTestUtil.startMiniCluster();
super.setUp();
} catch (Throwable e) {
throw new RuntimeException(e);
@ -179,10 +188,26 @@ private void setupKerberos() {
String servicePrincipal = kerberosConfigurationProvider.getTestPrincipal() + "@" + kerberosConfigurationProvider.getRealm();
HBaseKerberosUtils.setPrincipalForTesting(servicePrincipal);
HBaseKerberosUtils.setKeytabFileForTesting(kerberosConfigurationProvider.getKeytabFilePath());
HBaseKerberosUtils.setSecuredConfiguration(hbaseTestUtil.getConfiguration());
UserGroupInformation.setConfiguration(hbaseTestUtil.getConfiguration());
hbaseTestUtil.getConfiguration().setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName());
Configuration configuration = hbaseTestUtil.getConfiguration();
HBaseKerberosUtils.setSecuredConfiguration(configuration);
UserGroupInformation.setConfiguration(configuration);
configuration.setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName());
setupKerberosForHdfs(servicePrincipal, configuration);
}
private void setupKerberosForHdfs(String servicePrincipal, Configuration configuration) {
configuration.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, servicePrincipal);
configuration.set(DFS_NAMENODE_KEYTAB_FILE_KEY, kerberosConfigurationProvider.getKeytabFilePath());
configuration.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, servicePrincipal);
configuration.set(DFS_DATANODE_KEYTAB_FILE_KEY, kerberosConfigurationProvider.getKeytabFilePath());
configuration.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
configuration.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, servicePrincipal);
configuration.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTP_ONLY.name());
configuration.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, DEFAULT_DFS_HTTPS_ADDRESS);
configuration.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, DEFAULT_DFS_HTTPS_ADDRESS);
configuration.setBoolean(IGNORE_SECURE_PORTS_FOR_TESTING_KEY, true);
}
public void shutdown() throws Exception {