github/sqoop
5
0
Fork 0
mirror of https://github.com/apache/sqoop.git synced 2025-05-08 19:02:10 +08:00
Code Issues Projects Releases Wiki Activity

SQOOP-1841: Sqoop2: Upgrade from 1.99.1 to 1.99.4 onwards is broken

Browse Source 
(Abraham Elmahrek via Jarek Jarcec Cecho)
This commit is contained in:
Jarek Jarcec Cecho 2015-02-24 09:07:04 -08:00
parent cbf95514e1
commit 901bc966f5
4 changed files with 182 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoUtils.java
View File

@ -18,22 +18,22 @@
package org.apache.sqoop.repository.common;
public class CommonRepoUtils {
public static final String TABLE_NAME_QUOTE_CHAR = "\"";
public static final String QUOTE_CHARACTER = "\"";
public static final String escapeTableName(String tableName) {
return TABLE_NAME_QUOTE_CHAR + tableName + TABLE_NAME_QUOTE_CHAR;
return QUOTE_CHARACTER + tableName + QUOTE_CHARACTER;
}
public static final String escapeColumnName(String columnName) {
return TABLE_NAME_QUOTE_CHAR + columnName + TABLE_NAME_QUOTE_CHAR;
return QUOTE_CHARACTER + columnName + QUOTE_CHARACTER;
}
public static final String escapeSchemaName(String schemaName) {
return TABLE_NAME_QUOTE_CHAR + schemaName + TABLE_NAME_QUOTE_CHAR;
return QUOTE_CHARACTER + schemaName + QUOTE_CHARACTER;
}
public static final String escapeConstraintName(String constraintName) {
return TABLE_NAME_QUOTE_CHAR + constraintName + TABLE_NAME_QUOTE_CHAR;
return QUOTE_CHARACTER + constraintName + QUOTE_CHARACTER;
}
public static final String getTableName(String schemaName, String tableName) {

86
repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
View File

@ -17,9 +17,9 @@
*/
package org.apache.sqoop.repository.derby;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.SCHEMA_SQOOP;
import static org.apache.sqoop.repository.derby.DerbySchemaCreateQuery.*;
import static org.apache.sqoop.repository.derby.DerbySchemaInsertUpdateDeleteSelectQuery.*;
import static org.apache.sqoop.repository.common.CommonRepositoryInsertUpdateDeleteSelectQuery.*;
import static org.apache.sqoop.repository.derby.DerbySchemaUpgradeQuery.*;
import java.net.URL;
@ -31,6 +31,7 @@
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@ -236,6 +237,7 @@ public void createOrUpgradeRepository(Connection conn) {
runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_ADD_COLUMN_UPDATE_USER, conn);
}
if(repositoryVersion <= 2) {
migrateFromUnnamedConstraintsToNamedConstraints(conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_SUBMISSION_MODIFY_COLUMN_SQS_EXTERNAL_ID_VARCHAR_50, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_CONNECTOR_MODIFY_COLUMN_SQC_VERSION_VARCHAR_64, conn);
}
@ -307,6 +309,88 @@ public void createOrUpgradeRepository(Connection conn) {
upgradeRepositoryVersion(conn);
}
/**
* In reality, this method simply drops all constrains on particular tables
* and creates new constraints. Pre-1.99.3 these will be unnamed,
* Post-1.99.3 these will be named.
*
* @param conn
*/
private void migrateFromUnnamedConstraintsToNamedConstraints(Connection conn) {
// Get unnamed constraints
PreparedStatement fetchUnnamedConstraintsStmt = null;
Statement dropUnnamedConstraintsStmt = null;
Map<String, List<String>> autoConstraintNameMap = new TreeMap<String, List<String>>();
try {
fetchUnnamedConstraintsStmt = conn.prepareStatement(STMT_FETCH_TABLE_FOREIGN_KEYS);
for (String tableName : new String[] {
DerbySchemaConstants.TABLE_SQ_FORM_NAME,
CommonRepositorySchemaConstants.TABLE_SQ_INPUT_NAME,
DerbySchemaConstants.TABLE_SQ_CONNECTION_NAME,
CommonRepositorySchemaConstants.TABLE_SQ_JOB_NAME,
DerbySchemaConstants.TABLE_SQ_CONNECTION_INPUT_NAME,
CommonRepositorySchemaConstants.TABLE_SQ_JOB_INPUT_NAME,
CommonRepositorySchemaConstants.TABLE_SQ_SUBMISSION_NAME,
CommonRepositorySchemaConstants.TABLE_SQ_COUNTER_SUBMISSION_NAME
}) {
fetchUnnamedConstraintsStmt.setString(1, tableName);
if (!autoConstraintNameMap.containsKey(tableName)) {
autoConstraintNameMap.put(tableName, new ArrayList<String>());
}
List<String> autoConstraintNames = autoConstraintNameMap.get(tableName);
if (fetchUnnamedConstraintsStmt.execute()) {
LOG.info("QUERY(" + STMT_FETCH_TABLE_FOREIGN_KEYS + ") with args: [" + tableName + "]");
ResultSet rs = fetchUnnamedConstraintsStmt.getResultSet();
while (rs.next()) {
autoConstraintNames.add(rs.getString(1));
}
rs.close();
}
}
} catch (SQLException e) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
} finally {
closeStatements(fetchUnnamedConstraintsStmt);
}
// Drop constraints
for (String tableName : autoConstraintNameMap.keySet()) {
for (String constraintName : autoConstraintNameMap.get(tableName)) {
String query = DerbySchemaUpgradeQuery.getDropConstraintQuery(
SCHEMA_SQOOP, tableName, constraintName);
try {
dropUnnamedConstraintsStmt = conn.createStatement();
dropUnnamedConstraintsStmt.execute(query);
} catch (SQLException e) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
} finally {
LOG.info("QUERY(" + query + ")");
closeStatements(dropUnnamedConstraintsStmt);
}
}
}
// Create named constraints
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQF_SQC, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQI_SQF, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQN_SQC, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQB_SQN, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQNI_SQN, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQNI_SQI, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQBI_SQB, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQBI_SQI, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQS_SQB, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQRS_SQG, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQRS_SQR, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQRS_SQS, conn);
}
// SQOOP-1498 refactoring related upgrades for table and column names
void renameEntitiesForConnectionAndForm(Connection conn) {
// LINK

7
repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaInsertUpdateDeleteSelectQuery.java
View File

@ -98,6 +98,13 @@ public final class DerbySchemaInsertUpdateDeleteSelectQuery extends CommonReposi
+ CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_DIRECTION)
+ " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME);
public static final String STMT_FETCH_TABLE_FOREIGN_KEYS = "SELECT "
+ CommonRepoUtils.escapeColumnName("CONSTRAINTNAME")
+ " FROM " + CommonRepoUtils.getTableName("SYS", "SYSCONSTRAINTS")
+ " C LEFT JOIN " + CommonRepoUtils.getTableName("SYS", "SYSTABLES")
+ " T ON C." + CommonRepoUtils.escapeColumnName("TABLEID") + " = T." + CommonRepoUtils.escapeColumnName("TABLEID")
+ " WHERE C.TYPE = 'F' AND T." + CommonRepoUtils.escapeColumnName("TABLENAME") + " = ?";
private DerbySchemaInsertUpdateDeleteSelectQuery() {
// Disable explicit object creation
}

87
repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
View File

@ -38,7 +38,6 @@
// enhancing this code
public final class DerbySchemaUpgradeQuery {
// DDL: Increased size of SQ_CONNECTOR.SQC_VERSION to 64
public static final String QUERY_UPGRADE_TABLE_SQ_CONNECTOR_MODIFY_COLUMN_SQC_VERSION_VARCHAR_64 =
"ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTOR_NAME) + " ALTER COLUMN "
@ -515,7 +514,91 @@ public final class DerbySchemaUpgradeQuery {
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " ADD COLUMN "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + " VARCHAR(32)";
private DerbySchemaUpgradeQuery() {
// Add 1.99.3 constraints
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQF_SQC = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_FORM_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQF_SQC_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQF_CONNECTOR) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTOR_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQC_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQI_SQF = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQI_SQF_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_FORM) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_FORM_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQF_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQN_SQC = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTION_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQN_SQC_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQN_CONNECTOR) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTOR_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQC_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQB_SQN = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQB_SQN_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQB_CONNECTION) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTION_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQN_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQNI_SQN = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTION_INPUT_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQNI_SQN_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQNI_CONNECTION) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTION_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQN_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQNI_SQI = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONNECTION_INPUT_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQNI_SQI_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQNI_INPUT) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQBI_SQB = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQBI_SQB_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQB_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQBI_SQI = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQBI_SQI_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQS_SQB = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_SUBMISSION_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQS_SQB_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQS_JOB) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQB_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQRS_SQG = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_COUNTER_SUBMISSION_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQRS_SQG_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQRS_GROUP) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_COUNTER_GROUP_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQG_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQRS_SQR = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_COUNTER_SUBMISSION_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQRS_SQR_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQRS_COUNTER) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_COUNTER_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQR_ID) + ")";
public static final String QUERY_UPGRADE_TABLE_SQ_FORM_ADD_CONSTRAINT_SQRS_SQS = "ALTER TABLE "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_COUNTER_SUBMISSION_NAME)
+ " ADD CONSTRAINT " + CommonRepoUtils.getConstraintName(SCHEMA_SQOOP, CONSTRAINT_SQRS_SQS_NAME)
+ " FOREIGN KEY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQRS_SUBMISSION) + ") REFERENCES "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_SUBMISSION_NAME) + " (" + CommonRepoUtils.escapeColumnName(COLUMN_SQS_ID) + ") ON DELETE CASCADE";
public static final String getDropConstraintQuery(String schemaName, String tableName, String constraintName) {
StringBuilder queryBuilder = new StringBuilder();
queryBuilder.append("ALTER TABLE ");
queryBuilder.append(CommonRepoUtils.getTableName(schemaName, tableName));
queryBuilder.append(" DROP CONSTRAINT ");
queryBuilder.append(CommonRepoUtils.getConstraintName(schemaName, constraintName));
return queryBuilder.toString();
}
private DerbySchemaUpgradeQuery() {
// Disable explicit object creation
}
}