github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-02 05:00:01 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1259: Sqoop on Windows can't run HCatalog/HBase multinode jobs

Browse Source 
(Venkat Ranganathan via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2013-12-16 03:18:59 -08:00
parent 28f415ff2f
commit c688915e8e
3 changed files with 37 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored
View File

@ -31,5 +31,6 @@
*.sh text eol=lf *.sh text eol=lf
*.bat text eol=crlf *.bat text eol=crlf
*.cmd text eol=crlf
*.csproj text merge=union eol=crlf *.csproj text merge=union eol=crlf
*.sln text merge=union eol=crlf *.sln text merge=union eol=crlf

31
bin/configure-sqoop.cmd
View File

@ -72,6 +72,20 @@ if not defined HBASE_HOME (
echo Warning: HBASE_HOME and HBASE_VERSION not set. echo Warning: HBASE_HOME and HBASE_VERSION not set.
) )
) )
:: Check for HCatalog dependency
if not defined HCAT_HOME (
if defined HCATALOG_HOME (
set HCAT_HOME=%HCATALOG_HOME%
) else (
echo Warning: HCAT_HOME not set
)
)
if not exist "%HCATALOG_HOME%" (
echo Warning: HCATALOG_HOME does not exist! HCatalog imports will fail.
echo Please set HCATALOG_HOME to the root of your HCatalog installation.
)
:: ::
:: Check for Accumulo dependency :: Check for Accumulo dependency
if not defined ACCUMULO_HOME ( if not defined ACCUMULO_HOME (
@ -125,11 +139,25 @@ if exist "%SQOOP_HOME%\lib" (
call :add_dir_to_classpath %SQOOP_HOME%\lib call :add_dir_to_classpath %SQOOP_HOME%\lib
) )
:: Add HCatalog Home to the dependency list so that newer thrift libraries are
:: used instead of HBase version
if exist "%HCATALOG_HOME%" (
if defined PYTHON_CMD (
for /F "usebackq eol==" %%G IN ( `%PYTHON_CMD% %HCAT_HOME%\bin\hcat.py -classpath`) DO SET SQOOP_CLASSPATH=%SQOOP_CLASSPATH%;%%G
) else (
for /f "usebackq eol==" %%G IN ( `python %HCAT_HOME%\bin\hcat.py -classpath`) DO SET SQOOP_CLASSPATH=%SQOOP_CLASSPATH%;%%G
)
)
:: Add HBase to dependency list :: Add HBase to dependency list
if exist "%HBASE_HOME%" ( if exist "%HBASE_HOME%" (
call :add_dir_to_classpath %HBASE_HOME% call :add_dir_to_classpath %HBASE_HOME%
call :add_dir_to_classpath %HBASE_HOME%\lib call :add_dir_to_classpath %HBASE_HOME%\lib
set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%HBASE_HOME%\conf
) )
:: ::
:: Add Accumulo to dependency list :: Add Accumulo to dependency list
if exist "%ACCUMULO_HOME%" ( if exist "%ACCUMULO_HOME%" (
@ -147,7 +175,7 @@ if not defined ZOOCFGDIR (
) )
if "%ZOOCFGDIR%" NEQ "" ( if "%ZOOCFGDIR%" NEQ "" (
call :add_dir_to_classpath %ZOOCFGDIR% set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%ZOOCFGDIR%
) )
call :add_dir_to_classpath %SQOOP_CONF_DIR% call :add_dir_to_classpath %SQOOP_CONF_DIR%
@ -173,3 +201,4 @@ if not "%1"=="" (
set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%1\* set SQOOP_CLASSPATH=!SQOOP_CLASSPATH!;%1\*
) )
goto :eof goto :eof

6
src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
View File

@ -105,7 +105,7 @@ public final class SqoopHCatUtilities {
"sqoop.hcat.debug.import.mapper"; "sqoop.hcat.debug.import.mapper";
public static final String DEBUG_HCAT_EXPORT_MAPPER_PROP = public static final String DEBUG_HCAT_EXPORT_MAPPER_PROP =
"sqoop.hcat.debug.export.mapper"; "sqoop.hcat.debug.export.mapper";
private static final String HCATCMD = Shell.WINDOWS ? "hcat.cmd" : "hcat"; private static final String HCATCMD = Shell.WINDOWS ? "hcat.py" : "hcat";
private SqoopOptions options; private SqoopOptions options;
private ConnManager connManager; private ConnManager connManager;
private String hCatTableName; private String hCatTableName;
@ -970,6 +970,10 @@ public void executeExternalHCatProgram(List<String> env, String[] cmdLine)
// run HCat command with the given args // run HCat command with the given args
String hCatProgram = getHCatPath(); String hCatProgram = getHCatPath();
ArrayList<String> args = new ArrayList<String>(); ArrayList<String> args = new ArrayList<String>();
if (Shell.WINDOWS) {
// windows depends on python to be available
args.add("python");
}
args.add(hCatProgram); args.add(hCatProgram);
if (cmdLine != null && cmdLine.length > 0) { if (cmdLine != null && cmdLine.length > 0) {
for (String s : cmdLine) { for (String s : cmdLine) {