diff --git a/pom.xml b/pom.xml index 8859398b..9ff51d12 100644 --- a/pom.xml +++ b/pom.xml @@ -284,6 +284,11 @@ limitations under the License. sqoop-spi ${project.version} + + org.apache.sqoop.repository + sqoop-repository-common + ${project.version} + org.apache.sqoop.repository sqoop-repository-derby diff --git a/repository/pom.xml b/repository/pom.xml index e3345c4b..8c95c0e2 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -33,6 +33,7 @@ limitations under the License. pom + repository-common repository-derby diff --git a/repository/repository-common/pom.xml b/repository/repository-common/pom.xml new file mode 100644 index 00000000..37378c66 --- /dev/null +++ b/repository/repository-common/pom.xml @@ -0,0 +1,60 @@ + + + + + 4.0.0 + + + org.apache.sqoop + repository + 2.0.0-SNAPSHOT + + + org.apache.sqoop.repository + sqoop-repository-common + Sqoop Common Repository + + + + org.apache.sqoop + sqoop-core + + + + commons-lang + commons-lang + + + + junit + junit + test + + + + org.apache.sqoop + sqoop-core + test-jar + test + + + + + diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoConstants.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoConstants.java new file mode 100644 index 00000000..d1890b89 --- /dev/null +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoConstants.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository.common; + +public class CommonRepoConstants { + + public static final String SYSKEY_VERSION = "repository.version"; + +} diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryError.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryError.java new file mode 100644 index 00000000..7a7b28c6 --- /dev/null +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryError.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository.common; + +import org.apache.sqoop.common.ErrorCode; + +public enum CommonRepositoryError implements ErrorCode { + // JDBC Repository Handler Errors: Prefix COMMON + + /** The system was unable to run the specified query. */ + COMMON_0000("Unable to run specified query"), + + /** The system was unable to query the repository for given entity */ + COMMON_0001("Unable to retrieve entity data"), + + /** The repository contains more than one connector with same name */ + COMMON_0002("Invalid entity state - multiple connectors with name"), + + /** The system does not support the given input type.*/ + COMMON_0003("Unknown input type encountered"), + + /** The system does not support the given config type.*/ + COMMON_0004("Unknown config type encountered"), + + /** No input was found for the given config. */ + COMMON_0005("The config contains no input"), + + /** The system could not load the config due to unexpected position of input.*/ + COMMON_0006("The config input retrieved does not match expected position"), + + /** + * The system could not load the connector due to unexpected position + * of config. + */ + COMMON_0007("The config retrieved does not match expected position"), + + /** + * The system was not able to register entity due to a pre-assigned + * persistence identifier. + */ + COMMON_0008("Entity cannot have preassigned persistence id"), + + /** + * The system was unable to register various entities. + */ + COMMON_0009("Unexpected update count when registering entity"), + + /** + * The system was unable to register entity due to a failure to retrieve + * the generated identifier. + */ + COMMON_0010("Unable to retrieve generated identifier"), + + /** + * The system was unable to register connector due to a server + * error. + */ + COMMON_0011("Registration of connector failed"), + + /** + * The system was not able to register connector due to an unexpected + * update count. + */ + COMMON_0012("Unexpected update count on config registration"), + + /** + * The system was unable to register connector due to a failure to + * retrieve the generated identifier for a config. + */ + COMMON_0013("Unable to retrieve generated identifier for config"), + + /** + * The system was unable to register connector due to an unexpected + * update count for config input registration. + */ + COMMON_0014("Unexpected update count for config input"), + + /** + * The system was unable to register connector due to a failure to + * retrieve the generated identifier for a config input. + */ + COMMON_0015("Unable to retrieve generated identifier for config input"), + + /** We cant create new link in repository **/ + COMMON_0016("Unable to create new link data"), + + /** We can't save values for input to repository **/ + COMMON_0017("Unable to save input values to the repository"), + + /** We can't update link in repository **/ + COMMON_0018("Unable to update link in repository"), + + /** We can't delete link in repository **/ + COMMON_0019("Unable to delete link in repository"), + + /** We can't restore link from repository **/ + COMMON_0020("Unable to load link from repository"), + + /** We can't restore specific link from repository **/ + COMMON_0021("Unable to load specific link from repository"), + + /** We're unable to check if given link already exists */ + COMMON_0022("Unable to check if given link exists"), + + /** We cant create new job in repository **/ + COMMON_0023("Unable to create new job data"), + + /** We can't update job in repository **/ + COMMON_0024("Unable to update job in repository"), + + /** We can't delete job in repository **/ + COMMON_0025("Unable to delete job in repository"), + + /** We're unable to check if given job already exists */ + COMMON_0026("Unable to check if given job exists"), + + /** We can't restore specific job from repository **/ + COMMON_0027("Unable to load specific job from repository"), + + /** We can't restore job from repository **/ + COMMON_0028("Unable to load job from repository"), + + /** Can't verify if link is referenced from somewhere **/ + COMMON_0029("Unable to check if link is in use"), + + /** We're unable to check if given submission already exists */ + COMMON_0030("Unable to check if given submission exists"), + + /** We cant create new submission in repository **/ + COMMON_0031("Unable to create new submission data"), + + /** We can't update submission in repository **/ + COMMON_0032("Unable to update submission in the repository"), + + /** Can't purge old submissions **/ + COMMON_0033("Unable to purge old submissions"), + + /** Can't retrieve unfinished submissions **/ + COMMON_0034("Can't retrieve unfinished submissions"), + + /** Can't update connector **/ + COMMON_0035("Update of connector failed"), + + /** Can't retrieve all submissions **/ + COMMON_0036("Can't retrieve all submissions"), + + /** Can't retrieve submissions for a job **/ + COMMON_0037("Can't retrieve submissions for a job"), + + /** Can't enable/disable link **/ + COMMON_0038("Can't enable/disable link"), + + /** Can't enable/disable job **/ + COMMON_0039("Can't enable/disable job"), + + /** Can't update driver config **/ + COMMON_0040("Update of driver config failed"), + + /** Can't retrieve all connectors **/ + COMMON_0041("Can't retrieve all connectors"), + + /** Can't register config direction **/ + COMMON_0042("Could not register config direction"), + + /** Can't set connector direction **/ + COMMON_0043("Could not set connector direction"), + + /** The system was unable to register driver due to a server error **/ + COMMON_0044("Registration of driver failed") + + ; + + private final String message; + + private CommonRepositoryError(String message) { + this.message = message; + } + + public String getCode() { + return name(); + } + + public String getMessage() { + return message; + } +} diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java new file mode 100644 index 00000000..9f4ecbc1 --- /dev/null +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java @@ -0,0 +1,2286 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository.common; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.Direction; +import org.apache.sqoop.common.DirectionError; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.common.SupportedDirections; +import org.apache.sqoop.driver.Driver; +import org.apache.sqoop.model.MBooleanInput; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MConfigType; +import org.apache.sqoop.model.MConfigurableType; +import org.apache.sqoop.model.MConnector; +import org.apache.sqoop.model.MDriver; +import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MEnumInput; +import org.apache.sqoop.model.MFromConfig; +import org.apache.sqoop.model.MInput; +import org.apache.sqoop.model.MInputType; +import org.apache.sqoop.model.MIntegerInput; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.model.MLinkConfig; +import org.apache.sqoop.model.MMapInput; +import org.apache.sqoop.model.MStringInput; +import org.apache.sqoop.model.MSubmission; +import org.apache.sqoop.model.MToConfig; +import org.apache.sqoop.repository.JdbcRepositoryHandler; +import org.apache.sqoop.submission.SubmissionStatus; +import org.apache.sqoop.submission.counter.Counter; +import org.apache.sqoop.submission.counter.CounterGroup; +import org.apache.sqoop.submission.counter.Counters; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +/** + * Set of methods required from each JDBC based repository. + */ +public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler { + + private static final Logger LOG = + Logger.getLogger(CommonRepositoryHandler.class); + + /** + * Name to be used during logging + * + * @return String + */ + public abstract String name(); + + /** + * {@inheritDoc} + */ + @Override + public MConnector findConnector(String shortName, Connection conn) { + if (LOG.isDebugEnabled()) { + LOG.debug("Looking up connector: " + shortName); + } + MConnector mc = null; + PreparedStatement connectorFetchStmt = null; + try { + connectorFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_FROM_CONFIGURABLE); + connectorFetchStmt.setString(1, shortName); + + List connectors = loadConnectors(connectorFetchStmt, conn); + + if (connectors.size() == 0) { + LOG.debug("No connector found by name: " + shortName); + return null; + } else if (connectors.size() == 1) { + LOG.debug("Looking up connector: " + shortName + ", found: " + mc); + return connectors.get(0); + } else { + throw new SqoopException(CommonRepositoryError.COMMON_0002, shortName); + } + + } catch (SQLException ex) { + logException(ex, shortName); + throw new SqoopException(CommonRepositoryError.COMMON_0001, shortName, ex); + } finally { + closeStatements(connectorFetchStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public List findConnectors(Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIGURABLE_ALL_FOR_TYPE); + stmt.setString(1, MConfigurableType.CONNECTOR.name()); + + return loadConnectors(stmt, conn); + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0041, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void registerConnector(MConnector mc, Connection conn) { + if (mc.hasPersistenceId()) { + throw new SqoopException(CommonRepositoryError.COMMON_0008, + mc.getUniqueName()); + } + mc.setPersistenceId(insertAndGetConnectorId(mc, conn)); + insertConfigsForConnector(mc, conn); + } + + /** + * {@inheritDoc} + */ + @Override + public List findJobsForConnector(long connectorId, Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_ALL_JOBS_FOR_CONNECTOR_CONFIGURABLE); + stmt.setLong(1, connectorId); + stmt.setLong(2, connectorId); + return loadJobs(stmt, conn); + + } catch (SQLException ex) { + logException(ex, connectorId); + throw new SqoopException(CommonRepositoryError.COMMON_0028, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void upgradeConnectorAndConfigs(MConnector mConnector, Connection conn) { + updateConnectorAndDeleteConfigs(mConnector, conn); + insertConfigsForConnector(mConnector, conn); + } + + private void updateConnectorAndDeleteConfigs(MConnector mConnector, Connection conn) { + PreparedStatement updateConnectorStatement = null; + PreparedStatement deleteConfig = null; + PreparedStatement deleteInput = null; + try { + updateConnectorStatement = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_UPDATE_CONFIGURABLE); + deleteInput = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_INPUTS_FOR_CONFIGURABLE); + deleteConfig = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_CONFIGS_FOR_CONFIGURABLE); + updateConnectorStatement.setString(1, mConnector.getUniqueName()); + updateConnectorStatement.setString(2, mConnector.getClassName()); + updateConnectorStatement.setString(3, mConnector.getVersion()); + updateConnectorStatement.setString(4, mConnector.getType().name()); + updateConnectorStatement.setLong(5, mConnector.getPersistenceId()); + + if (updateConnectorStatement.executeUpdate() != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0035); + } + deleteInput.setLong(1, mConnector.getPersistenceId()); + deleteConfig.setLong(1, mConnector.getPersistenceId()); + deleteInput.executeUpdate(); + deleteConfig.executeUpdate(); + + } catch (SQLException e) { + logException(e, mConnector); + throw new SqoopException(CommonRepositoryError.COMMON_0035, e); + } finally { + closeStatements(updateConnectorStatement, deleteConfig, deleteInput); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void upgradeDriverAndConfigs(MDriver mDriver, Connection conn) { + updateDriverAndDeleteConfigs(mDriver, conn); + insertConfigsForDriver(mDriver, conn); + } + + private void updateDriverAndDeleteConfigs(MDriver mDriver, Connection conn) { + PreparedStatement updateDriverStatement = null; + PreparedStatement deleteConfig = null; + PreparedStatement deleteInput = null; + try { + updateDriverStatement = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_UPDATE_CONFIGURABLE); + deleteInput = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_INPUTS_FOR_CONFIGURABLE); + deleteConfig = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_CONFIGS_FOR_CONFIGURABLE); + updateDriverStatement.setString(1, mDriver.getUniqueName()); + updateDriverStatement.setString(2, Driver.getClassName()); + updateDriverStatement.setString(3, mDriver.getVersion()); + updateDriverStatement.setString(4, mDriver.getType().name()); + updateDriverStatement.setLong(5, mDriver.getPersistenceId()); + + if (updateDriverStatement.executeUpdate() != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0035); + } + deleteInput.setLong(1, mDriver.getPersistenceId()); + deleteConfig.setLong(1, mDriver.getPersistenceId()); + deleteInput.executeUpdate(); + deleteConfig.executeUpdate(); + + } catch (SQLException e) { + logException(e, mDriver); + throw new SqoopException(CommonRepositoryError.COMMON_0040, e); + } finally { + closeStatements(updateDriverStatement, deleteConfig, deleteInput); + } + } + + /** + * Helper method to insert the configs from the into the + * repository. + * + * @param mDriver The driver instance to use to upgrade. + * @param conn JDBC link to use for updating the configs + */ + private void insertConfigsForDriver(MDriver mDriver, Connection conn) { + PreparedStatement baseConfigStmt = null; + PreparedStatement baseInputStmt = null; + try { + baseConfigStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_CONFIG, + Statement.RETURN_GENERATED_KEYS); + + baseInputStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_INPUT, + Statement.RETURN_GENERATED_KEYS); + + // Register the job config type, since driver config is per job + registerConfigs(null, null, mDriver.getDriverConfig().getConfigs(), + MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); + + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0011, mDriver.toString(), ex); + } finally { + closeStatements(baseConfigStmt, baseInputStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public MDriver findDriver(String shortName, Connection conn) { + LOG.debug("Looking up Driver and config "); + PreparedStatement driverFetchStmt = null; + PreparedStatement driverConfigFetchStmt = null; + PreparedStatement driverConfigInputFetchStmt = null; + + MDriver mDriver; + try { + driverFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_FROM_CONFIGURABLE); + driverFetchStmt.setString(1, shortName); + + ResultSet rsDriverSet = driverFetchStmt.executeQuery(); + if (!rsDriverSet.next()) { + return null; + } + Long driverId = rsDriverSet.getLong(1); + String driverVersion = rsDriverSet.getString(4); + + driverConfigFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIG_FOR_CONFIGURABLE); + driverConfigFetchStmt.setLong(1, driverId); + + driverConfigInputFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_INPUT); + List driverConfigs = new ArrayList(); + loadDriverConfigs(driverConfigs, driverConfigFetchStmt, driverConfigInputFetchStmt, 1); + + if (driverConfigs.isEmpty()) { + return null; + } + mDriver = new MDriver(new MDriverConfig(driverConfigs), driverVersion); + mDriver.setPersistenceId(driverId); + + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0001, "Driver", ex); + } finally { + if (driverConfigFetchStmt != null) { + try { + driverConfigFetchStmt.close(); + } catch (SQLException ex) { + LOG.error("Unable to close driver config fetch statement", ex); + } + } + if (driverConfigInputFetchStmt != null) { + try { + driverConfigInputFetchStmt.close(); + } catch (SQLException ex) { + LOG.error("Unable to close driver input fetch statement", ex); + } + } + if (driverFetchStmt != null) { + try { + driverFetchStmt.close(); + } catch (SQLException ex) { + LOG.error("Unable to close driver fetch statement", ex); + } + } + } + + LOG.debug("Looked up Driver and config"); + return mDriver; + } + + /** + * {@inheritDoc} + */ + @Override + public void registerDriver(MDriver mDriver, Connection conn) { + if (mDriver.hasPersistenceId()) { + throw new SqoopException(CommonRepositoryError.COMMON_0008, mDriver.getUniqueName()); + } + mDriver.setPersistenceId(insertAndGetDriverId(mDriver, conn)); + insertConfigsforDriver(mDriver, conn); + } + + /** + * {@inheritDoc} + */ + @Override + public void createLink(MLink link, Connection conn) { + PreparedStatement stmt = null; + int result; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_LINK, + Statement.RETURN_GENERATED_KEYS); + stmt.setString(1, link.getName()); + stmt.setLong(2, link.getConnectorId()); + stmt.setBoolean(3, link.getEnabled()); + stmt.setString(4, link.getCreationUser()); + stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime())); + stmt.setString(6, link.getLastUpdateUser()); + stmt.setTimestamp(7, new Timestamp(link.getLastUpdateDate().getTime())); + + result = stmt.executeUpdate(); + if (result != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0009, + Integer.toString(result)); + } + + ResultSet rsetConnectionId = stmt.getGeneratedKeys(); + + if (!rsetConnectionId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + + long connectionId = rsetConnectionId.getLong(1); + + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_LINK_INPUT, + connectionId, + link.getConnectorLinkConfig().getConfigs(), + conn); + link.setPersistenceId(connectionId); + + } catch (SQLException ex) { + logException(ex, link); + throw new SqoopException(CommonRepositoryError.COMMON_0016, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void updateLink(MLink link, Connection conn) { + PreparedStatement deleteStmt = null; + PreparedStatement updateStmt = null; + try { + // Firstly remove old values + deleteStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_LINK_INPUT); + deleteStmt.setLong(1, link.getPersistenceId()); + deleteStmt.executeUpdate(); + + // Update LINK_CONFIG table + updateStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_UPDATE_LINK); + updateStmt.setString(1, link.getName()); + updateStmt.setString(2, link.getLastUpdateUser()); + updateStmt.setTimestamp(3, new Timestamp(new Date().getTime())); + + updateStmt.setLong(4, link.getPersistenceId()); + updateStmt.executeUpdate(); + + // And reinsert new values + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_LINK_INPUT, + link.getPersistenceId(), + link.getConnectorLinkConfig().getConfigs(), + conn); + + } catch (SQLException ex) { + logException(ex, link); + throw new SqoopException(CommonRepositoryError.COMMON_0018, ex); + } finally { + closeStatements(deleteStmt, updateStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean existsLink(long linkId, Connection conn) { + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_LINK_CHECK_BY_ID); + stmt.setLong(1, linkId); + rs = stmt.executeQuery(); + + // Should be always valid in query with count + rs.next(); + + return rs.getLong(1) == 1; + } catch (SQLException ex) { + logException(ex, linkId); + throw new SqoopException(CommonRepositoryError.COMMON_0022, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean inUseLink(long linkId, Connection conn) { + PreparedStatement stmt = null; + ResultSet rs = null; + + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_JOBS_FOR_LINK_CHECK); + stmt.setLong(1, linkId); + rs = stmt.executeQuery(); + + // Should be always valid in case of count(*) query + rs.next(); + + return rs.getLong(1) != 0; + + } catch (SQLException e) { + logException(e, linkId); + throw new SqoopException(CommonRepositoryError.COMMON_0029, e); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void enableLink(long linkId, boolean enabled, Connection conn) { + PreparedStatement enableConn = null; + + try { + enableConn = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_ENABLE_LINK); + enableConn.setBoolean(1, enabled); + enableConn.setLong(2, linkId); + enableConn.executeUpdate(); + } catch (SQLException ex) { + logException(ex, linkId); + throw new SqoopException(CommonRepositoryError.COMMON_0038, ex); + } finally { + closeStatements(enableConn); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteLink(long linkId, Connection conn) { + PreparedStatement dltConn = null; + + try { + deleteLinkInputs(linkId, conn); + dltConn = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_LINK); + dltConn.setLong(1, linkId); + dltConn.executeUpdate(); + } catch (SQLException ex) { + logException(ex, linkId); + throw new SqoopException(CommonRepositoryError.COMMON_0019, ex); + } finally { + closeStatements(dltConn); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteLinkInputs(long id, Connection conn) { + PreparedStatement dltConnInput = null; + try { + dltConnInput = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_LINK_INPUT); + dltConnInput.setLong(1, id); + dltConnInput.executeUpdate(); + } catch (SQLException ex) { + logException(ex, id); + throw new SqoopException(CommonRepositoryError.COMMON_0019, ex); + } finally { + closeStatements(dltConnInput); + } + } + + /** + * {@inheritDoc} + */ + @Override + public MLink findLink(long linkId, Connection conn) { + PreparedStatement linkFetchStmt = null; + try { + linkFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_LINK_SINGLE); + linkFetchStmt.setLong(1, linkId); + + List links = loadLinks(linkFetchStmt, conn); + + if (links.size() != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0021, "Couldn't find" + + " link with id " + linkId); + } + + // Return the first and only one link object with the given id + return links.get(0); + + } catch (SQLException ex) { + logException(ex, linkId); + throw new SqoopException(CommonRepositoryError.COMMON_0020, ex); + } finally { + closeStatements(linkFetchStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public MLink findLink(String linkName, Connection conn) { + PreparedStatement linkFetchStmt = null; + try { + linkFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_LINK_SINGLE_BY_NAME); + linkFetchStmt.setString(1, linkName); + + List links = loadLinks(linkFetchStmt, conn); + + if (links.size() != 1) { + return null; + } + + // Return the first and only one link object with the given name + return links.get(0); + + } catch (SQLException ex) { + logException(ex, linkName); + throw new SqoopException(CommonRepositoryError.COMMON_0020, ex); + } finally { + closeStatements(linkFetchStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public List findLinks(Connection conn) { + PreparedStatement linksFetchStmt = null; + try { + linksFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_LINK_ALL); + + return loadLinks(linksFetchStmt, conn); + + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0020, ex); + } finally { + closeStatements(linksFetchStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public List findLinksForConnector(long connectorId, Connection conn) { + PreparedStatement linkByConnectorFetchStmt = null; + try { + linkByConnectorFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_LINK_FOR_CONNECTOR_CONFIGURABLE); + linkByConnectorFetchStmt.setLong(1, connectorId); + return loadLinks(linkByConnectorFetchStmt, conn); + } catch (SQLException ex) { + logException(ex, connectorId); + throw new SqoopException(CommonRepositoryError.COMMON_0020, ex); + } finally { + closeStatements(linkByConnectorFetchStmt); + } + } + + /** + * {@inheritDoc} + */ + public void createJob(MJob job, Connection conn) { + PreparedStatement stmt = null; + int result; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB, Statement.RETURN_GENERATED_KEYS); + stmt.setString(1, job.getName()); + stmt.setLong(2, job.getLinkId(Direction.FROM)); + stmt.setLong(3, job.getLinkId(Direction.TO)); + stmt.setBoolean(4, job.getEnabled()); + stmt.setString(5, job.getCreationUser()); + stmt.setTimestamp(6, new Timestamp(job.getCreationDate().getTime())); + stmt.setString(7, job.getLastUpdateUser()); + stmt.setTimestamp(8, new Timestamp(job.getLastUpdateDate().getTime())); + + result = stmt.executeUpdate(); + if (result != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0009, + Integer.toString(result)); + } + + ResultSet rsetJobId = stmt.getGeneratedKeys(); + + if (!rsetJobId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + + long jobId = rsetJobId.getLong(1); + + // from config for the job + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB_INPUT, + jobId, + job.getJobConfig(Direction.FROM).getConfigs(), + conn); + // to config for the job + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB_INPUT, + jobId, + job.getJobConfig(Direction.TO).getConfigs(), + conn); + // driver config per job + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB_INPUT, + jobId, + job.getDriverConfig().getConfigs(), + conn); + + job.setPersistenceId(jobId); + + } catch (SQLException ex) { + logException(ex, job); + throw new SqoopException(CommonRepositoryError.COMMON_0023, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void updateJob(MJob job, Connection conn) { + PreparedStatement deleteStmt = null; + PreparedStatement updateStmt = null; + try { + // Firstly remove old values + deleteStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_JOB_INPUT); + deleteStmt.setLong(1, job.getPersistenceId()); + deleteStmt.executeUpdate(); + + // Update job table + updateStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_UPDATE_JOB); + updateStmt.setString(1, job.getName()); + updateStmt.setString(2, job.getLastUpdateUser()); + updateStmt.setTimestamp(3, new Timestamp(new Date().getTime())); + + updateStmt.setLong(4, job.getPersistenceId()); + updateStmt.executeUpdate(); + + // And reinsert new values + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB_INPUT, + job.getPersistenceId(), + job.getJobConfig(Direction.FROM).getConfigs(), + conn); + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB_INPUT, + job.getPersistenceId(), + job.getJobConfig(Direction.TO).getConfigs(), + conn); + createInputValues(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_JOB_INPUT, + job.getPersistenceId(), + job.getDriverConfig().getConfigs(), + conn); + + } catch (SQLException ex) { + logException(ex, job); + throw new SqoopException(CommonRepositoryError.COMMON_0024, ex); + } finally { + closeStatements(deleteStmt, updateStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean existsJob(long jobId, Connection conn) { + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_JOB_CHECK_BY_ID); + stmt.setLong(1, jobId); + rs = stmt.executeQuery(); + + // Should be always valid in query with count + rs.next(); + + return rs.getLong(1) == 1; + } catch (SQLException ex) { + logException(ex, jobId); + throw new SqoopException(CommonRepositoryError.COMMON_0026, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean inUseJob(long jobId, Connection conn) { + MSubmission submission = findLastSubmissionForJob(jobId, conn); + + // We have no submissions and thus job can't be in use + if (submission == null) { + return false; + } + + // We can't remove running job + if (submission.getStatus().isRunning()) { + return true; + } + + return false; + } + + /** + * {@inheritDoc} + */ + @Override + public void enableJob(long jobId, boolean enabled, Connection conn) { + PreparedStatement enableConn = null; + + try { + enableConn = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_ENABLE_JOB); + enableConn.setBoolean(1, enabled); + enableConn.setLong(2, jobId); + enableConn.executeUpdate(); + } catch (SQLException ex) { + logException(ex, jobId); + throw new SqoopException(CommonRepositoryError.COMMON_0039, ex); + } finally { + closeStatements(enableConn); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteJobInputs(long id, Connection conn) { + PreparedStatement dltInput = null; + try { + dltInput = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_JOB_INPUT); + dltInput.setLong(1, id); + dltInput.executeUpdate(); + } catch (SQLException ex) { + logException(ex, id); + throw new SqoopException(CommonRepositoryError.COMMON_0025, ex); + } finally { + closeStatements(dltInput); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteJob(long jobId, Connection conn) { + PreparedStatement dlt = null; + try { + deleteJobInputs(jobId, conn); + dlt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_JOB); + dlt.setLong(1, jobId); + dlt.executeUpdate(); + } catch (SQLException ex) { + logException(ex, jobId); + throw new SqoopException(CommonRepositoryError.COMMON_0025, ex); + } finally { + closeStatements(dlt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public MJob findJob(long jobId, Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_JOB_SINGLE_BY_ID); + stmt.setLong(1, jobId); + + List jobs = loadJobs(stmt, conn); + + if (jobs.size() != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0027, "Couldn't find" + + " job with id " + jobId); + } + + // Return the first and only one link object + return jobs.get(0); + + } catch (SQLException ex) { + logException(ex, jobId); + throw new SqoopException(CommonRepositoryError.COMMON_0028, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public MJob findJob(String name, Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_JOB_SINGLE_BY_NAME); + stmt.setString(1, name); + + List jobs = loadJobs(stmt, conn); + + if (jobs.size() != 1) { + return null; + } + + // Return the first and only one link object + return jobs.get(0); + + } catch (SQLException ex) { + logException(ex, name); + throw new SqoopException(CommonRepositoryError.COMMON_0028, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public List findJobs(Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_JOB); + + return loadJobs(stmt, conn); + + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0028, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void createSubmission(MSubmission submission, Connection conn) { + PreparedStatement stmt = null; + int result; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_SUBMISSION, + Statement.RETURN_GENERATED_KEYS); + stmt.setLong(1, submission.getJobId()); + stmt.setString(2, submission.getStatus().name()); + stmt.setString(3, submission.getCreationUser()); + stmt.setTimestamp(4, new Timestamp(submission.getCreationDate().getTime())); + stmt.setString(5, submission.getLastUpdateUser()); + stmt.setTimestamp(6, new Timestamp(submission.getLastUpdateDate().getTime())); + stmt.setString(7, submission.getExternalId()); + stmt.setString(8, submission.getExternalLink()); + stmt.setString(9, submission.getExceptionInfo()); + stmt.setString(10, submission.getExceptionStackTrace()); + + result = stmt.executeUpdate(); + if (result != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0009, + Integer.toString(result)); + } + + ResultSet rsetSubmissionId = stmt.getGeneratedKeys(); + + if (!rsetSubmissionId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + + long submissionId = rsetSubmissionId.getLong(1); + + if(submission.getCounters() != null) { + createSubmissionCounters(submissionId, submission.getCounters(), conn); + } + + // Save created persistence id + submission.setPersistenceId(submissionId); + + } catch (SQLException ex) { + logException(ex, submission); + throw new SqoopException(CommonRepositoryError.COMMON_0031, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean existsSubmission(long submissionId, Connection conn) { + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SUBMISSION_CHECK); + stmt.setLong(1, submissionId); + rs = stmt.executeQuery(); + + // Should be always valid in query with count + rs.next(); + + return rs.getLong(1) == 1; + } catch (SQLException ex) { + logException(ex, submissionId); + throw new SqoopException(CommonRepositoryError.COMMON_0030, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void updateSubmission(MSubmission submission, Connection conn) { + PreparedStatement stmt = null; + PreparedStatement deleteStmt = null; + try { + // Update properties in main table + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_UPDATE_SUBMISSION); + stmt.setString(1, submission.getStatus().name()); + stmt.setString(2, submission.getLastUpdateUser()); + stmt.setTimestamp(3, new Timestamp(submission.getLastUpdateDate().getTime())); + stmt.setString(4, submission.getExceptionInfo()); + stmt.setString(5, submission.getExceptionStackTrace()); + + stmt.setLong(6, submission.getPersistenceId()); + stmt.executeUpdate(); + + // Delete previous counters + deleteStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_DELETE_COUNTER_SUBMISSION); + deleteStmt.setLong(1, submission.getPersistenceId()); + deleteStmt.executeUpdate(); + + // Reinsert new counters if needed + if(submission.getCounters() != null) { + createSubmissionCounters(submission.getPersistenceId(), submission.getCounters(), conn); + } + + } catch (SQLException ex) { + logException(ex, submission); + throw new SqoopException(CommonRepositoryError.COMMON_0032, ex); + } finally { + closeStatements(stmt, deleteStmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void purgeSubmissions(Date threshold, Connection conn) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_PURGE_SUBMISSIONS); + stmt.setTimestamp(1, new Timestamp(threshold.getTime())); + stmt.executeUpdate(); + + } catch (SQLException ex) { + logException(ex, threshold); + throw new SqoopException(CommonRepositoryError.COMMON_0033, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * {@inheritDoc} + */ + @Override + public List findUnfinishedSubmissions(Connection conn) { + List submissions = new LinkedList(); + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SUBMISSION_UNFINISHED); + + for(SubmissionStatus status : SubmissionStatus.unfinished()) { + stmt.setString(1, status.name()); + rs = stmt.executeQuery(); + + while(rs.next()) { + submissions.add(loadSubmission(rs, conn)); + } + + rs.close(); + rs = null; + } + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0034, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + + return submissions; + } + + /** + * {@inheritDoc} + */ + @Override + public List findSubmissions(Connection conn) { + List submissions = new LinkedList(); + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SUBMISSIONS); + rs = stmt.executeQuery(); + + while(rs.next()) { + submissions.add(loadSubmission(rs, conn)); + } + + rs.close(); + rs = null; + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0036, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + + return submissions; + } + + /** + * {@inheritDoc} + */ + @Override + public List findSubmissionsForJob(long jobId, Connection conn) { + List submissions = new LinkedList(); + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SUBMISSIONS_FOR_JOB); + stmt.setLong(1, jobId); + rs = stmt.executeQuery(); + + while(rs.next()) { + submissions.add(loadSubmission(rs, conn)); + } + + rs.close(); + rs = null; + } catch (SQLException ex) { + logException(ex); + throw new SqoopException(CommonRepositoryError.COMMON_0037, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + + return submissions; + } + + /** + * {@inheritDoc} + */ + @Override + public MSubmission findLastSubmissionForJob(long jobId, Connection conn) { + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SUBMISSIONS_FOR_JOB); + stmt.setLong(1, jobId); + stmt.setMaxRows(1); + rs = stmt.executeQuery(); + + if(!rs.next()) { + return null; + } + + return loadSubmission(rs, conn); + } catch (SQLException ex) { + logException(ex, jobId); + throw new SqoopException(CommonRepositoryError.COMMON_0037, ex); + } finally { + closeResultSets(rs); + closeStatements(stmt); + } + } + + private void insertConnectorDirection(Long connectorId, Direction direction, Connection conn) + throws SQLException { + PreparedStatement stmt = null; + + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_SQ_CONNECTOR_DIRECTIONS); + stmt.setLong(1, connectorId); + stmt.setLong(2, getDirection(direction, conn)); + + if (stmt.executeUpdate() != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0043); + } + } finally { + closeStatements(stmt); + } + } + + private void insertConnectorDirections(Long connectorId, SupportedDirections directions, Connection conn) + throws SQLException { + if (directions.isDirectionSupported(Direction.FROM)) { + insertConnectorDirection(connectorId, Direction.FROM, conn); + } + + if (directions.isDirectionSupported(Direction.TO)) { + insertConnectorDirection(connectorId, Direction.TO, conn); + } + } + + private long insertAndGetConnectorId(MConnector mc, Connection conn) { + PreparedStatement baseConnectorStmt = null; + try { + baseConnectorStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_CONFIGURABLE, + Statement.RETURN_GENERATED_KEYS); + baseConnectorStmt.setString(1, mc.getUniqueName()); + baseConnectorStmt.setString(2, mc.getClassName()); + baseConnectorStmt.setString(3, mc.getVersion()); + baseConnectorStmt.setString(4, mc.getType().name()); + + int baseConnectorCount = baseConnectorStmt.executeUpdate(); + if (baseConnectorCount != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0009, + Integer.toString(baseConnectorCount)); + } + + ResultSet rsetConnectorId = baseConnectorStmt.getGeneratedKeys(); + + if (!rsetConnectorId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + // connector configurable also have directions + insertConnectorDirections(rsetConnectorId.getLong(1), mc.getSupportedDirections(), conn); + return rsetConnectorId.getLong(1); + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0011, mc.toString(), ex); + } finally { + closeStatements(baseConnectorStmt); + } + } + + /** + * Helper method to insert the configs from the MConnector into the + * repository. The job and connector configs within mc will get + * updated with the id of the configs when this function returns. + * @param mc The connector to use for updating configs + * @param conn JDBC connection to use for inserting the configs + */ + private void insertConfigsForConnector(MConnector mc, Connection conn) { + long connectorId = mc.getPersistenceId(); + PreparedStatement baseConfigStmt = null; + PreparedStatement baseInputStmt = null; + try{ + baseConfigStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_CONFIG, + Statement.RETURN_GENERATED_KEYS); + + baseInputStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_INPUT, + Statement.RETURN_GENERATED_KEYS); + + // Register link type config + registerConfigs(connectorId, null /* No direction for LINK type config*/, mc.getLinkConfig().getConfigs(), + MConfigType.LINK.name(), baseConfigStmt, baseInputStmt, conn); + + // Register both from/to job type config for connector + if (mc.getSupportedDirections().isDirectionSupported(Direction.FROM)) { + registerConfigs(connectorId, Direction.FROM, mc.getFromConfig().getConfigs(), + MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); + } + if (mc.getSupportedDirections().isDirectionSupported(Direction.TO)) { + registerConfigs(connectorId, Direction.TO, mc.getToConfig().getConfigs(), + MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); + } + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0011, + mc.toString(), ex); + } finally { + closeStatements(baseConfigStmt, baseInputStmt); + } + } + + private long insertAndGetDriverId(MDriver mDriver, Connection conn) { + PreparedStatement baseDriverStmt = null; + try { + baseDriverStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_CONFIGURABLE, + Statement.RETURN_GENERATED_KEYS); + baseDriverStmt.setString(1, mDriver.getUniqueName()); + baseDriverStmt.setString(2, Driver.getClassName()); + baseDriverStmt.setString(3, mDriver.getVersion()); + baseDriverStmt.setString(4, mDriver.getType().name()); + + int baseDriverCount = baseDriverStmt.executeUpdate(); + if (baseDriverCount != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0009, Integer.toString(baseDriverCount)); + } + + ResultSet rsetDriverId = baseDriverStmt.getGeneratedKeys(); + + if (!rsetDriverId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + return rsetDriverId.getLong(1); + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0044, mDriver.toString(), ex); + } finally { + closeStatements(baseDriverStmt); + } + } + + private void insertConfigsforDriver(MDriver mDriver, Connection conn) { + PreparedStatement baseConfigStmt = null; + PreparedStatement baseInputStmt = null; + try { + baseConfigStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_CONFIG, + Statement.RETURN_GENERATED_KEYS); + baseInputStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_INTO_INPUT, + Statement.RETURN_GENERATED_KEYS); + + // Register a driver config as a job type with no owner/connector and direction + registerConfigs(mDriver.getPersistenceId(), null /* no direction*/, mDriver.getDriverConfig().getConfigs(), + MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); + + } catch (SQLException ex) { + logException(ex, mDriver); + throw new SqoopException(CommonRepositoryError.COMMON_0011, ex); + } finally { + closeStatements(baseConfigStmt, baseInputStmt); + } + } + + /** + * Stores counters for given submission in repository. + * + * @param submissionId Submission id + * @param counters Counters that should be stored + * @param conn Connection to derby repository + * @throws java.sql.SQLException + */ + private void createSubmissionCounters(long submissionId, Counters counters, Connection conn) throws SQLException { + PreparedStatement stmt = null; + + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_COUNTER_SUBMISSION); + + for(CounterGroup group : counters) { + long groupId = getCounterGroupId(group, conn); + + for(Counter counter: group) { + long counterId = getCounterId(counter, conn); + + stmt.setLong(1, groupId); + stmt.setLong(2, counterId); + stmt.setLong(3, submissionId); + stmt.setLong(4, counter.getValue()); + + stmt.executeUpdate(); + } + } + } finally { + closeStatements(stmt); + } + } + + /** + * Resolves counter group database id. + * + * @param group Given group + * @param conn Connection to database + * @return Id + * @throws java.sql.SQLException + */ + private long getCounterGroupId(CounterGroup group, Connection conn) throws SQLException { + PreparedStatement select = null; + PreparedStatement insert = null; + ResultSet rsSelect = null; + ResultSet rsInsert = null; + + try { + select = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_COUNTER_GROUP); + select.setString(1, group.getName()); + + rsSelect = select.executeQuery(); + + if(rsSelect.next()) { + return rsSelect.getLong(1); + } + + insert = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_COUNTER_GROUP, Statement.RETURN_GENERATED_KEYS); + insert.setString(1, group.getName()); + insert.executeUpdate(); + + rsInsert = insert.getGeneratedKeys(); + + if (!rsInsert.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + + return rsInsert.getLong(1); + } finally { + closeResultSets(rsSelect, rsInsert); + closeStatements(select, insert); + } + } + + /** + * Resolves counter id. + * + * @param counter Given counter + * @param conn Connection to database + * @return Id + * @throws java.sql.SQLException + */ + private long getCounterId(Counter counter, Connection conn) throws SQLException { + PreparedStatement select = null; + PreparedStatement insert = null; + ResultSet rsSelect = null; + ResultSet rsInsert = null; + + try { + select = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_COUNTER); + select.setString(1, counter.getName()); + + rsSelect = select.executeQuery(); + + if(rsSelect.next()) { + return rsSelect.getLong(1); + } + + insert = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_COUNTER, Statement.RETURN_GENERATED_KEYS); + insert.setString(1, counter.getName()); + insert.executeUpdate(); + + rsInsert = insert.getGeneratedKeys(); + + if (!rsInsert.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0010); + } + + return rsInsert.getLong(1); + } finally { + closeResultSets(rsSelect, rsInsert); + closeStatements(select, insert); + } + } + + /** + * Create MSubmission structure from result set. + * + * @param rs Result set, only active row will be fetched + * @param conn Connection to database + * @return Created MSubmission structure + * @throws java.sql.SQLException + */ + private MSubmission loadSubmission(ResultSet rs, Connection conn) throws SQLException { + MSubmission submission = new MSubmission(); + + submission.setPersistenceId(rs.getLong(1)); + submission.setJobId(rs.getLong(2)); + submission.setStatus(SubmissionStatus.valueOf(rs.getString(3))); + submission.setCreationUser(rs.getString(4)); + submission.setCreationDate(rs.getTimestamp(5)); + submission.setLastUpdateUser(rs.getString(6)); + submission.setLastUpdateDate(rs.getTimestamp(7)); + submission.setExternalId(rs.getString(8)); + submission.setExternalLink(rs.getString(9)); + submission.setExceptionInfo(rs.getString(10)); + submission.setExceptionStackTrace(rs.getString(11)); + + Counters counters = loadCountersSubmission(rs.getLong(1), conn); + submission.setCounters(counters); + + return submission; + } + + private Counters loadCountersSubmission(long submissionId, Connection conn) throws SQLException { + PreparedStatement stmt = null; + ResultSet rs = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_COUNTER_SUBMISSION); + stmt.setLong(1, submissionId); + rs = stmt.executeQuery(); + + Counters counters = new Counters(); + + while (rs.next()) { + String groupName = rs.getString(1); + String counterName = rs.getString(2); + long value = rs.getLong(3); + + CounterGroup group = counters.getCounterGroup(groupName); + if (group == null) { + group = new CounterGroup(groupName); + counters.addCounterGroup(group); + } + + group.addCounter(new Counter(counterName, value)); + } + + if (counters.isEmpty()) { + return null; + } else { + return counters; + } + } finally { + closeStatements(stmt); + closeResultSets(rs); + } + } + + private Long getDirection(Direction direction, Connection conn) throws SQLException { + PreparedStatement directionStmt = null; + ResultSet rs = null; + + try { + directionStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SQD_ID_BY_SQD_NAME); + directionStmt.setString(1, direction.toString()); + rs = directionStmt.executeQuery(); + + rs.next(); + return rs.getLong(1); + } finally { + if (rs != null) { + closeResultSets(rs); + } + if (directionStmt != null) { + closeStatements(directionStmt); + } + } + } + + private Direction getDirection(long directionId, Connection conn) throws SQLException { + PreparedStatement directionStmt = null; + ResultSet rs = null; + + try { + directionStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SQD_NAME_BY_SQD_ID); + directionStmt.setLong(1, directionId); + rs = directionStmt.executeQuery(); + + rs.next(); + return Direction.valueOf(rs.getString(1)); + } finally { + if (rs != null) { + closeResultSets(rs); + } + if (directionStmt != null) { + closeStatements(directionStmt); + } + } + } + + private SupportedDirections findConnectorSupportedDirections(long connectorId, Connection conn) throws SQLException { + PreparedStatement connectorDirectionsStmt = null; + ResultSet rs = null; + + boolean from = false, to = false; + + try { + connectorDirectionsStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SQ_CONNECTOR_DIRECTIONS); + connectorDirectionsStmt.setLong(1, connectorId); + rs = connectorDirectionsStmt.executeQuery(); + + while(rs.next()) { + switch(getDirection(rs.getLong(2), conn)) { + case FROM: + from = true; + break; + + case TO: + to = true; + break; + } + } + } finally { + if (rs != null) { + closeResultSets(rs); + } + if (connectorDirectionsStmt != null) { + closeStatements(connectorDirectionsStmt); + } + } + + return new SupportedDirections(from, to); + } + + private List loadConnectors(PreparedStatement stmt, Connection conn) throws SQLException { + List connectors = new ArrayList(); + ResultSet rsConnectors = null; + PreparedStatement connectorConfigFetchStmt = null; + PreparedStatement connectorConfigInputFetchStmt = null; + + try { + rsConnectors = stmt.executeQuery(); + connectorConfigFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIG_FOR_CONFIGURABLE); + connectorConfigInputFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_INPUT); + + while(rsConnectors.next()) { + long connectorId = rsConnectors.getLong(1); + String connectorName = rsConnectors.getString(2); + String connectorClassName = rsConnectors.getString(3); + String connectorVersion = rsConnectors.getString(4); + + connectorConfigFetchStmt.setLong(1, connectorId); + + List linkConfig = new ArrayList(); + List fromConfig = new ArrayList(); + List toConfig = new ArrayList(); + + loadConnectorConfigTypes(linkConfig, fromConfig, toConfig, connectorConfigFetchStmt, + connectorConfigInputFetchStmt, 1, conn); + + SupportedDirections supportedDirections + = findConnectorSupportedDirections(connectorId, conn); + MFromConfig fromJobConfig = null; + MToConfig toJobConfig = null; + if (supportedDirections.isDirectionSupported(Direction.FROM)) { + fromJobConfig = new MFromConfig(fromConfig); + } + if (supportedDirections.isDirectionSupported(Direction.TO)) { + toJobConfig = new MToConfig(toConfig); + } + MConnector mc = new MConnector(connectorName, connectorClassName, connectorVersion, + new MLinkConfig(linkConfig), fromJobConfig, toJobConfig); + mc.setPersistenceId(connectorId); + + connectors.add(mc); + } + } finally { + closeResultSets(rsConnectors); + closeStatements(connectorConfigFetchStmt, connectorConfigInputFetchStmt); + } + return connectors; + } + + private List loadLinks(PreparedStatement stmt, + Connection conn) + throws SQLException { + List links = new ArrayList(); + ResultSet rsConnection = null; + PreparedStatement connectorConfigFetchStatement = null; + PreparedStatement connectorConfigInputStatement = null; + + try { + rsConnection = stmt.executeQuery(); + + // + connectorConfigFetchStatement = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIG_FOR_CONFIGURABLE); + connectorConfigInputStatement = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_FETCH_LINK_INPUT); + + while(rsConnection.next()) { + long id = rsConnection.getLong(1); + String name = rsConnection.getString(2); + long connectorId = rsConnection.getLong(3); + boolean enabled = rsConnection.getBoolean(4); + String creationUser = rsConnection.getString(5); + Date creationDate = rsConnection.getTimestamp(6); + String updateUser = rsConnection.getString(7); + Date lastUpdateDate = rsConnection.getTimestamp(8); + + connectorConfigFetchStatement.setLong(1, connectorId); + connectorConfigInputStatement.setLong(1, id); + connectorConfigInputStatement.setLong(3, id); + + List connectorLinkConfig = new ArrayList(); + List fromConfig = new ArrayList(); + List toConfig = new ArrayList(); + + loadConnectorConfigTypes(connectorLinkConfig, fromConfig, toConfig, connectorConfigFetchStatement, + connectorConfigInputStatement, 2, conn); + MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig)); + + link.setPersistenceId(id); + link.setName(name); + link.setCreationUser(creationUser); + link.setCreationDate(creationDate); + link.setLastUpdateUser(updateUser); + link.setLastUpdateDate(lastUpdateDate); + link.setEnabled(enabled); + + links.add(link); + } + } finally { + closeResultSets(rsConnection); + closeStatements(connectorConfigFetchStatement, connectorConfigInputStatement); + } + + return links; + } + + private List loadJobs(PreparedStatement stmt, + Connection conn) + throws SQLException { + List jobs = new ArrayList(); + ResultSet rsJob = null; + PreparedStatement fromConfigFetchStmt = null; + PreparedStatement toConfigFetchStmt = null; + PreparedStatement driverConfigfetchStmt = null; + PreparedStatement jobInputFetchStmt = null; + + try { + rsJob = stmt.executeQuery(); + // Note: Job does not hold a explicit reference to the driver since every + // job has the same driver + long driverId = this.findDriver(MDriver.DRIVER_NAME, conn).getPersistenceId(); + fromConfigFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIG_FOR_CONFIGURABLE); + toConfigFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIG_FOR_CONFIGURABLE); + driverConfigfetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_CONFIG_FOR_CONFIGURABLE); + jobInputFetchStmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_FETCH_JOB_INPUT); + + while(rsJob.next()) { + long fromConnectorId = rsJob.getLong(1); + long toConnectorId = rsJob.getLong(2); + long id = rsJob.getLong(3); + String name = rsJob.getString(4); + long fromLinkId = rsJob.getLong(5); + long toLinkId = rsJob.getLong(6); + boolean enabled = rsJob.getBoolean(7); + String createBy = rsJob.getString(8); + Date creationDate = rsJob.getTimestamp(9); + String updateBy = rsJob.getString(10); + Date lastUpdateDate = rsJob.getTimestamp(11); + + fromConfigFetchStmt.setLong(1, fromConnectorId); + toConfigFetchStmt.setLong(1,toConnectorId); + driverConfigfetchStmt.setLong(1, driverId); + + jobInputFetchStmt.setLong(1, id); + jobInputFetchStmt.setLong(3, id); + + // FROM entity configs + List fromConnectorLinkConfig = new ArrayList(); + List fromConnectorFromJobConfig = new ArrayList(); + List fromConnectorToJobConfig = new ArrayList(); + + loadConnectorConfigTypes(fromConnectorLinkConfig, fromConnectorFromJobConfig, fromConnectorToJobConfig, + fromConfigFetchStmt, jobInputFetchStmt, 2, conn); + + // TO entity configs + List toConnectorLinkConfig = new ArrayList(); + List toConnectorFromJobConfig = new ArrayList(); + List toConnectorToJobConfig = new ArrayList(); + + // ?? dont we need 2 different driver configs for the from/to? + List driverConfig = new ArrayList(); + + loadConnectorConfigTypes(toConnectorLinkConfig, toConnectorFromJobConfig, toConnectorToJobConfig, + toConfigFetchStmt, jobInputFetchStmt, 2, conn); + + loadDriverConfigs(driverConfig, driverConfigfetchStmt, jobInputFetchStmt, 2); + + MJob job = new MJob( + fromConnectorId, toConnectorId, + fromLinkId, toLinkId, + new MFromConfig(fromConnectorFromJobConfig), + new MToConfig(toConnectorToJobConfig), + new MDriverConfig(driverConfig)); + + job.setPersistenceId(id); + job.setName(name); + job.setCreationUser(createBy); + job.setCreationDate(creationDate); + job.setLastUpdateUser(updateBy); + job.setLastUpdateDate(lastUpdateDate); + job.setEnabled(enabled); + + jobs.add(job); + } + } finally { + closeResultSets(rsJob); + closeStatements(fromConfigFetchStmt, toConfigFetchStmt, driverConfigfetchStmt, jobInputFetchStmt); + } + + return jobs; + } + + private void registerConfigDirection(Long configId, Direction direction, Connection conn) + throws SQLException { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_INSERT_SQ_CONFIG_DIRECTIONS); + stmt.setLong(1, configId); + stmt.setLong(2, getDirection(direction, conn)); + if (stmt.executeUpdate() != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0042); + } + } finally { + closeStatements(stmt); + } + } + + /** + * Register configs in derby database. This method will insert the ids + * generated by the repository into the configs passed in itself. + * + * Use given prepared statements to create entire config structure in database. + * + * @param configurableId + * @param configs + * @param type + * @param baseConfigStmt + * @param baseInputStmt + * @param conn + * @return short number of configs registered. + * @throws java.sql.SQLException + */ + private short registerConfigs(Long configurableId, Direction direction, + List configs, String type, PreparedStatement baseConfigStmt, + PreparedStatement baseInputStmt, Connection conn) + throws SQLException { + short configIndex = 0; + + for (MConfig config : configs) { + if (configurableId == null) { + baseConfigStmt.setNull(1, Types.BIGINT); + } else { + baseConfigStmt.setLong(1, configurableId); + } + + baseConfigStmt.setString(2, config.getName()); + baseConfigStmt.setString(3, type); + baseConfigStmt.setShort(4, configIndex++); + + int baseConfigCount = baseConfigStmt.executeUpdate(); + if (baseConfigCount != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0012, + Integer.toString(baseConfigCount)); + } + ResultSet rsetConfigId = baseConfigStmt.getGeneratedKeys(); + if (!rsetConfigId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0013); + } + + long configId = rsetConfigId.getLong(1); + config.setPersistenceId(configId); + + if (direction != null) { + registerConfigDirection(configId, direction, conn); + } + + // Insert all the inputs + List> inputs = config.getInputs(); + registerConfigInputs(configId, inputs, baseInputStmt); + } + return configIndex; + } + + /** + * Save given inputs to the database. + * + * Use given prepare statement to save all inputs into repository. + * + * @param configId Identifier for corresponding config + * @param inputs List of inputs that needs to be saved + * @param baseInputStmt Statement that we can utilize + * @throws java.sql.SQLException In case of any failure on Derby side + */ + private void registerConfigInputs(long configId, List> inputs, + PreparedStatement baseInputStmt) throws SQLException { + short inputIndex = 0; + for (MInput input : inputs) { + baseInputStmt.setString(1, input.getName()); + baseInputStmt.setLong(2, configId); + baseInputStmt.setShort(3, inputIndex++); + baseInputStmt.setString(4, input.getType().name()); + baseInputStmt.setBoolean(5, input.isSensitive()); + // String specific column(s) + if (input.getType().equals(MInputType.STRING)) { + MStringInput strInput = (MStringInput) input; + baseInputStmt.setShort(6, strInput.getMaxLength()); + } else { + baseInputStmt.setNull(6, Types.INTEGER); + } + // Enum specific column(s) + if(input.getType() == MInputType.ENUM) { + baseInputStmt.setString(7, StringUtils.join(((MEnumInput) input).getValues(), ",")); + } else { + baseInputStmt.setNull(7, Types.VARCHAR); + } + + int baseInputCount = baseInputStmt.executeUpdate(); + if (baseInputCount != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0014, + Integer.toString(baseInputCount)); + } + + ResultSet rsetInputId = baseInputStmt.getGeneratedKeys(); + if (!rsetInputId.next()) { + throw new SqoopException(CommonRepositoryError.COMMON_0015); + } + + long inputId = rsetInputId.getLong(1); + input.setPersistenceId(inputId); + } + } + + /** + * Load configs and corresponding inputs from Derby database. + * + * Use given prepared statements to load all configs and corresponding inputs + * from Derby. + * + * @param driverConfig List of driver configs that will be filled up + * @param configFetchStatement Prepared statement for fetching configs + * @param inputFetchStmt Prepare statement for fetching inputs + * @param configPosition position of the config + * @throws java.sql.SQLException In case of any failure on Derby side + */ + public void loadDriverConfigs(List driverConfig, + PreparedStatement configFetchStatement, + PreparedStatement inputFetchStmt, + int configPosition) throws SQLException { + + // Get list of structures from database + ResultSet rsetConfig = configFetchStatement.executeQuery(); + while (rsetConfig.next()) { + long configId = rsetConfig.getLong(1); + Long fromConnectorId = rsetConfig.getLong(2); + String configName = rsetConfig.getString(3); + String configTYpe = rsetConfig.getString(4); + int configIndex = rsetConfig.getInt(5); + List> configInputs = new ArrayList>(); + + MConfig mDriverConfig = new MConfig(configName, configInputs); + mDriverConfig.setPersistenceId(configId); + + inputFetchStmt.setLong(configPosition, configId); + + ResultSet rsetInput = inputFetchStmt.executeQuery(); + while (rsetInput.next()) { + long inputId = rsetInput.getLong(1); + String inputName = rsetInput.getString(2); + long inputConfig = rsetInput.getLong(3); + short inputIndex = rsetInput.getShort(4); + String inputType = rsetInput.getString(5); + boolean inputSensitivity = rsetInput.getBoolean(6); + short inputStrLength = rsetInput.getShort(7); + String inputEnumValues = rsetInput.getString(8); + String value = rsetInput.getString(9); + + MInputType mit = MInputType.valueOf(inputType); + + MInput input = null; + switch (mit) { + case STRING: + input = new MStringInput(inputName, inputSensitivity, inputStrLength); + break; + case MAP: + input = new MMapInput(inputName, inputSensitivity); + break; + case BOOLEAN: + input = new MBooleanInput(inputName, inputSensitivity); + break; + case INTEGER: + input = new MIntegerInput(inputName, inputSensitivity); + break; + case ENUM: + input = new MEnumInput(inputName, inputSensitivity, inputEnumValues.split(",")); + break; + default: + throw new SqoopException(CommonRepositoryError.COMMON_0003, + "input-" + inputName + ":" + inputId + ":" + + "config-" + inputConfig + ":" + mit.name()); + } + + // Set persistent ID + input.setPersistenceId(inputId); + + // Set value + if(value == null) { + input.setEmpty(); + } else { + input.restoreFromUrlSafeValueString(value); + } + + if (mDriverConfig.getInputs().size() != inputIndex) { + throw new SqoopException(CommonRepositoryError.COMMON_0006, + "config: " + mDriverConfig + + "; input: " + input + + "; index: " + inputIndex + + "; expected: " + mDriverConfig.getInputs().size() + ); + } + + mDriverConfig.getInputs().add(input); + } + + if (mDriverConfig.getInputs().size() == 0) { + throw new SqoopException(CommonRepositoryError.COMMON_0005, + "owner-" + fromConnectorId + + "; config: " + mDriverConfig + ); + } + + MConfigType configType = MConfigType.valueOf(configTYpe); + switch (configType) { + case JOB: + if (driverConfig.size() != configIndex) { + throw new SqoopException(CommonRepositoryError.COMMON_0007, + "owner-" + fromConnectorId + + "; config: " + configType + + "; index: " + configIndex + + "; expected: " + driverConfig.size() + ); + } + driverConfig.add(mDriverConfig); + break; + default: + throw new SqoopException(CommonRepositoryError.COMMON_0004, + "connector-" + fromConnectorId + ":" + configType); + } + } + } + + private Direction findConfigDirection(long configId, Connection conn) throws SQLException { + PreparedStatement stmt = null; + ResultSet rs = null; + + try { + stmt = conn.prepareStatement(CommonRepositoryInsertUpdateDeleteSelectQuery.STMT_SELECT_SQ_CONFIG_DIRECTIONS); + stmt.setLong(1, configId); + rs = stmt.executeQuery(); + rs.next(); + return getDirection(rs.getLong(2), conn); + } finally { + if (rs != null) { + closeResultSets(rs); + } + if (stmt != null) { + closeStatements(stmt); + } + } + } + + /** + * Load configs and corresponding inputs related to a connector. + * + * Use given prepared statements to load all configs and corresponding inputs + * from Derby. + * + * @param linkConfig List of link configs that will be filled up + * @param fromConfig FROM job configs that will be filled up + * @param toConfig TO job configs that will be filled up + * @param configFetchStmt Prepared statement for fetching configs + * @param inputFetchStmt Prepare statement for fetching inputs + * @param conn Connection object that is used to find config direction. + * @throws java.sql.SQLException In case of any failure on Derby side + */ + public void loadConnectorConfigTypes(List linkConfig, List fromConfig, + List toConfig, PreparedStatement configFetchStmt, PreparedStatement inputFetchStmt, + int configPosition, Connection conn) throws SQLException { + + // Get list of structures from database + ResultSet rsetConfig = configFetchStmt.executeQuery(); + while (rsetConfig.next()) { + long configId = rsetConfig.getLong(1); + Long configConnectorId = rsetConfig.getLong(2); + String configName = rsetConfig.getString(3); + String configType = rsetConfig.getString(4); + int configIndex = rsetConfig.getInt(5); + List> configInputs = new ArrayList>(); + + MConfig config = new MConfig(configName, configInputs); + config.setPersistenceId(configId); + + inputFetchStmt.setLong(configPosition, configId); + + ResultSet rsetInput = inputFetchStmt.executeQuery(); + while (rsetInput.next()) { + long inputId = rsetInput.getLong(1); + String inputName = rsetInput.getString(2); + long inputConfig = rsetInput.getLong(3); + short inputIndex = rsetInput.getShort(4); + String inputType = rsetInput.getString(5); + boolean inputSensitivity = rsetInput.getBoolean(6); + short inputStrLength = rsetInput.getShort(7); + String inputEnumValues = rsetInput.getString(8); + String value = rsetInput.getString(9); + + MInputType mit = MInputType.valueOf(inputType); + + MInput input = null; + switch (mit) { + case STRING: + input = new MStringInput(inputName, inputSensitivity, inputStrLength); + break; + case MAP: + input = new MMapInput(inputName, inputSensitivity); + break; + case BOOLEAN: + input = new MBooleanInput(inputName, inputSensitivity); + break; + case INTEGER: + input = new MIntegerInput(inputName, inputSensitivity); + break; + case ENUM: + input = new MEnumInput(inputName, inputSensitivity, inputEnumValues.split(",")); + break; + default: + throw new SqoopException(CommonRepositoryError.COMMON_0003, + "input-" + inputName + ":" + inputId + ":" + + "config-" + inputConfig + ":" + mit.name()); + } + + // Set persistent ID + input.setPersistenceId(inputId); + + // Set value + if(value == null) { + input.setEmpty(); + } else { + input.restoreFromUrlSafeValueString(value); + } + + if (config.getInputs().size() != inputIndex) { + throw new SqoopException(CommonRepositoryError.COMMON_0006, + "config: " + config + + "; input: " + input + + "; index: " + inputIndex + + "; expected: " + config.getInputs().size() + ); + } + + config.getInputs().add(input); + } + + if (config.getInputs().size() == 0) { + throw new SqoopException(CommonRepositoryError.COMMON_0005, + "connector-" + configConnectorId + + "; config: " + config + ); + } + + MConfigType mConfigType = MConfigType.valueOf(configType); + switch (mConfigType) { + case LINK: + if (linkConfig.size() != configIndex) { + throw new SqoopException(CommonRepositoryError.COMMON_0007, + "connector-" + configConnectorId + + "; config: " + config + + "; index: " + configIndex + + "; expected: " + linkConfig.size() + ); + } + linkConfig.add(config); + break; + case JOB: + Direction type = findConfigDirection(configId, conn); + List jobConfigs; + switch(type) { + case FROM: + jobConfigs = fromConfig; + break; + + case TO: + jobConfigs = toConfig; + break; + + default: + throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type); + } + + if (jobConfigs.size() != configIndex) { + throw new SqoopException(CommonRepositoryError.COMMON_0007, + "connector-" + configConnectorId + + "; config: " + config + + "; index: " + configIndex + + "; expected: " + jobConfigs.size() + ); + } + + jobConfigs.add(config); + break; + default: + throw new SqoopException(CommonRepositoryError.COMMON_0004, + "connector-" + configConnectorId + ":" + config); + } + } + } + + private void createInputValues(String query, + long id, + List configs, + Connection conn) throws SQLException { + PreparedStatement stmt = null; + int result; + + try { + stmt = conn.prepareStatement(query); + + for (MConfig config : configs) { + for (MInput input : config.getInputs()) { + // Skip empty values as we're not interested in storing those in db + if (input.isEmpty()) { + continue; + } + stmt.setLong(1, id); + stmt.setLong(2, input.getPersistenceId()); + stmt.setString(3, input.getUrlSafeValueString()); + + result = stmt.executeUpdate(); + if (result != 1) { + throw new SqoopException(CommonRepositoryError.COMMON_0017, + Integer.toString(result)); + } + } + } + } finally { + closeStatements(stmt); + } + } + + /** + * Execute given query on database. + * + * @param query Query that should be executed + */ + protected void runQuery(String query, Connection conn, Object... args) { + PreparedStatement stmt = null; + try { + stmt = conn.prepareStatement(query); + + for (int i = 0; i < args.length; ++i) { + if (args[i] instanceof String) { + stmt.setString(i + 1, (String)args[i]); + } else if (args[i] instanceof Long) { + stmt.setLong(i + 1, (Long) args[i]); + } else { + stmt.setObject(i + 1, args[i]); + } + } + + if (stmt.execute()) { + ResultSet rset = stmt.getResultSet(); + int count = 0; + while (rset.next()) { + count++; + } + LOG.info("QUERY(" + query + ") produced unused resultset with "+ count + " rows"); + } else { + int updateCount = stmt.getUpdateCount(); + LOG.info("QUERY(" + query + ") Update count: " + updateCount); + } + } catch (SQLException ex) { + throw new SqoopException(CommonRepositoryError.COMMON_0000, query, ex); + } finally { + closeStatements(stmt); + } + } + + /** + * Close all given Results set. + * + * Any occurring exception is silently ignored and logged. + * + * @param resultSets Result sets to close + */ + protected void closeResultSets(ResultSet ... resultSets) { + if(resultSets == null) { + return; + } + for (ResultSet rs : resultSets) { + if(rs != null) { + try { + rs.close(); + } catch(SQLException ex) { + LOG.error("Exception during closing result set", ex); + } + } + } + } + + /** + * Close all given statements. + * + * Any occurring exception is silently ignored and logged. + * + * @param stmts Statements to close + */ + protected void closeStatements(Statement... stmts) { + if(stmts == null) { + return; + } + for (Statement stmt : stmts) { + if(stmt != null) { + try { + stmt.close(); + } catch (SQLException ex) { + LOG.error("Exception during closing statement", ex); + } + } + } + } + + /** + * Log exception and all String variant of arbitrary number of objects. + * + * This method is useful to log SQLException with all objects that were + * used in the query generation to see where is the issue. + * + * @param throwable Arbitrary throwable object + * @param objects Arbitrary array of associated objects + */ + protected void logException(Throwable throwable, Object ...objects) { + LOG.error("Exception in repository operation", throwable); + LOG.error("Using database " + name()); + LOG.error("Associated objects: "+ objects.length); + for(Object object : objects) { + LOG.error("\t" + object.getClass().getSimpleName() + ": " + object.toString()); + } + } +} diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java new file mode 100644 index 00000000..921f12f9 --- /dev/null +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java @@ -0,0 +1,550 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository.common; + +import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.*; + +public class CommonRepositoryInsertUpdateDeleteSelectQuery { + /*******DIRECTION TABLE **************/ + public static final String STMT_SELECT_SQD_ID_BY_SQD_NAME = + "SELECT " + COLUMN_SQD_ID + " FROM " + TABLE_SQ_DIRECTION + + " WHERE " + COLUMN_SQD_NAME + "=?"; + + public static final String STMT_SELECT_SQD_NAME_BY_SQD_ID = + "SELECT " + COLUMN_SQD_NAME + " FROM " + TABLE_SQ_DIRECTION + + " WHERE " + COLUMN_SQD_ID + "=?"; + + /*********CONFIGURABLE TABLE ***************/ + //DML: Get configurable by given name + public static final String STMT_SELECT_FROM_CONFIGURABLE = + "SELECT " + + COLUMN_SQC_ID + ", " + + COLUMN_SQC_NAME + ", " + + COLUMN_SQC_CLASS + ", " + + COLUMN_SQC_VERSION + + " FROM " + TABLE_SQ_CONFIGURABLE + + " WHERE " + COLUMN_SQC_NAME + " = ?"; + + //DML: Get all configurables for a given type + public static final String STMT_SELECT_CONFIGURABLE_ALL_FOR_TYPE = + "SELECT " + + COLUMN_SQC_ID + ", " + + COLUMN_SQC_NAME + ", " + + COLUMN_SQC_CLASS + ", " + + COLUMN_SQC_VERSION + + " FROM " + TABLE_SQ_CONFIGURABLE + + " WHERE " + COLUMN_SQC_TYPE + " = ?"; + + //DML: Insert into configurable + public static final String STMT_INSERT_INTO_CONFIGURABLE = + "INSERT INTO " + TABLE_SQ_CONFIGURABLE + " (" + + COLUMN_SQC_NAME + ", " + + COLUMN_SQC_CLASS + ", " + + COLUMN_SQC_VERSION + ", " + + COLUMN_SQC_TYPE + + ") VALUES (?, ?, ?, ?)"; + + //Delete all configs for a given configurable + public static final String STMT_DELETE_CONFIGS_FOR_CONFIGURABLE = + "DELETE FROM " + TABLE_SQ_CONFIG + + " WHERE " + COLUMN_SQ_CFG_CONFIGURABLE + " = ?"; + + //Delete all inputs for a given configurable + public static final String STMT_DELETE_INPUTS_FOR_CONFIGURABLE = + "DELETE FROM " + TABLE_SQ_INPUT + + " WHERE " + + COLUMN_SQI_CONFIG + + " IN (SELECT " + + COLUMN_SQ_CFG_ID + + " FROM " + TABLE_SQ_CONFIG + + " WHERE " + + COLUMN_SQ_CFG_CONFIGURABLE + " = ?)"; + + //Update the configurable + public static final String STMT_UPDATE_CONFIGURABLE = + "UPDATE " + TABLE_SQ_CONFIGURABLE + + " SET " + COLUMN_SQC_NAME + " = ?, " + + COLUMN_SQC_CLASS + " = ?, " + + COLUMN_SQC_VERSION + " = ?, " + + COLUMN_SQC_TYPE + " = ? " + + " WHERE " + COLUMN_SQC_ID + " = ?"; + + /**********CONFIG TABLE **************/ + //DML: Get all configs for a given configurable + public static final String STMT_SELECT_CONFIG_FOR_CONFIGURABLE = + "SELECT " + + COLUMN_SQ_CFG_ID + ", " + + COLUMN_SQ_CFG_CONFIGURABLE + ", " + + COLUMN_SQ_CFG_NAME + ", " + + COLUMN_SQ_CFG_TYPE + ", " + + COLUMN_SQ_CFG_INDEX + + " FROM " + TABLE_SQ_CONFIG + + " WHERE " + COLUMN_SQ_CFG_CONFIGURABLE + " = ? " + + " ORDER BY " + COLUMN_SQ_CFG_INDEX; + + + //DML: Insert into config + public static final String STMT_INSERT_INTO_CONFIG = + "INSERT INTO " + TABLE_SQ_CONFIG + " (" + + COLUMN_SQ_CFG_CONFIGURABLE + ", " + + COLUMN_SQ_CFG_NAME + ", " + + COLUMN_SQ_CFG_TYPE + ", " + + COLUMN_SQ_CFG_INDEX + + ") VALUES ( ?, ?, ?, ?)"; + + /********** INPUT TABLE **************/ + // DML: Get inputs for a given config + public static final String STMT_SELECT_INPUT = + "SELECT " + + COLUMN_SQI_ID + ", " + + COLUMN_SQI_NAME + ", " + + COLUMN_SQI_CONFIG + ", " + + COLUMN_SQI_INDEX + ", " + + COLUMN_SQI_TYPE + ", " + + COLUMN_SQI_STRMASK + ", " + + COLUMN_SQI_STRLENGTH + ", " + + COLUMN_SQI_ENUMVALS + ", " + + "cast(null as varchar(100))" + + " FROM " + TABLE_SQ_INPUT + + " WHERE " + COLUMN_SQI_CONFIG + " = ?" + + " ORDER BY " + COLUMN_SQI_INDEX; + + // DML: Insert into config input + public static final String STMT_INSERT_INTO_INPUT = + "INSERT INTO " + TABLE_SQ_INPUT + " (" + + COLUMN_SQI_NAME + ", " + + COLUMN_SQI_CONFIG + ", " + + COLUMN_SQI_INDEX + ", " + + COLUMN_SQI_TYPE + ", " + + COLUMN_SQI_STRMASK + ", " + + COLUMN_SQI_STRLENGTH + ", " + + COLUMN_SQI_ENUMVALS + + ") VALUES (?, ?, ?, ?, ?, ?, ?)"; + + /**********LINK INPUT TABLE **************/ + //DML: Get inputs and values for a given link + public static final String STMT_FETCH_LINK_INPUT = + "SELECT " + + COLUMN_SQI_ID + ", " + + COLUMN_SQI_NAME + ", " + + COLUMN_SQI_CONFIG + ", " + + COLUMN_SQI_INDEX + ", " + + COLUMN_SQI_TYPE + ", " + + COLUMN_SQI_STRMASK + ", " + + COLUMN_SQI_STRLENGTH + "," + + COLUMN_SQI_ENUMVALS + ", " + + COLUMN_SQ_LNKI_VALUE + + " FROM " + TABLE_SQ_INPUT + + " LEFT OUTER JOIN " + TABLE_SQ_LINK_INPUT + + " ON " + COLUMN_SQ_LNKI_INPUT + " = " + COLUMN_SQI_ID + + " AND " + COLUMN_SQ_LNKI_LINK + " = ?" + + " WHERE " + COLUMN_SQI_CONFIG + " = ?" + + " AND (" + COLUMN_SQ_LNKI_LINK + " = ?" + " OR " + COLUMN_SQ_LNKI_LINK + " IS NULL)" + + " ORDER BY " + COLUMN_SQI_INDEX; + + /**********JOB INPUT TABLE **************/ + //DML: Fetch inputs and values for a given job + public static final String STMT_FETCH_JOB_INPUT = + "SELECT " + + COLUMN_SQI_ID + ", " + + COLUMN_SQI_NAME + ", " + + COLUMN_SQI_CONFIG + ", " + + COLUMN_SQI_INDEX + ", " + + COLUMN_SQI_TYPE + ", " + + COLUMN_SQI_STRMASK + ", " + + COLUMN_SQI_STRLENGTH + ", " + + COLUMN_SQI_ENUMVALS + ", " + + COLUMN_SQBI_VALUE + + " FROM " + TABLE_SQ_INPUT + + " LEFT OUTER JOIN " + TABLE_SQ_JOB_INPUT + + " ON " + COLUMN_SQBI_INPUT + " = " + COLUMN_SQI_ID + + " AND " + COLUMN_SQBI_JOB + " = ?" + + " WHERE " + COLUMN_SQI_CONFIG + " = ?" + + " AND (" + COLUMN_SQBI_JOB + " = ? OR " + COLUMN_SQBI_JOB + " IS NULL)" + + " ORDER BY " + COLUMN_SQI_INDEX; + + /**********LINK TABLE **************/ + // DML: Insert new link + public static final String STMT_INSERT_LINK = + "INSERT INTO " + TABLE_SQ_LINK + " (" + + COLUMN_SQ_LNK_NAME + ", " + + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + COLUMN_SQ_LNK_ENABLED + ", " + + COLUMN_SQ_LNK_CREATION_USER + ", " + + COLUMN_SQ_LNK_CREATION_DATE + ", " + + COLUMN_SQ_LNK_UPDATE_USER + ", " + + COLUMN_SQ_LNK_UPDATE_DATE + + ") VALUES (?, ?, ?, ?, ?, ?, ?)"; + + // DML: Insert new link inputs + public static final String STMT_INSERT_LINK_INPUT = + "INSERT INTO " + TABLE_SQ_LINK_INPUT + " (" + + COLUMN_SQ_LNKI_LINK + ", " + + COLUMN_SQ_LNKI_INPUT + ", " + + COLUMN_SQ_LNKI_VALUE + + ") VALUES (?, ?, ?)"; + + // DML: Update link + public static final String STMT_UPDATE_LINK = + "UPDATE " + TABLE_SQ_LINK + " SET " + + COLUMN_SQ_LNK_NAME + " = ?, " + + COLUMN_SQ_LNK_UPDATE_USER + " = ?, " + + COLUMN_SQ_LNK_UPDATE_DATE + " = ? " + + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; + + // DML: Enable or disable link + public static final String STMT_ENABLE_LINK = + "UPDATE " + TABLE_SQ_LINK + " SET " + + COLUMN_SQ_LNK_ENABLED + " = ? " + + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; + + // DML: Delete rows from link input table + public static final String STMT_DELETE_LINK_INPUT = + "DELETE FROM " + TABLE_SQ_LINK_INPUT + + " WHERE " + COLUMN_SQ_LNKI_LINK + " = ?"; + + // DML: Delete row from link table + public static final String STMT_DELETE_LINK = + "DELETE FROM " + TABLE_SQ_LINK + + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; + + // DML: Select one specific link + public static final String STMT_SELECT_LINK_SINGLE = + "SELECT " + + COLUMN_SQ_LNK_ID + ", " + + COLUMN_SQ_LNK_NAME + ", " + + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + COLUMN_SQ_LNK_ENABLED + ", " + + COLUMN_SQ_LNK_CREATION_USER + ", " + + COLUMN_SQ_LNK_CREATION_DATE + ", " + + COLUMN_SQ_LNK_UPDATE_USER + ", " + + COLUMN_SQ_LNK_UPDATE_DATE + + " FROM " + TABLE_SQ_LINK + + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; + + // DML: Select one specific link by name + public static final String STMT_SELECT_LINK_SINGLE_BY_NAME = + "SELECT " + + COLUMN_SQ_LNK_ID + ", " + + COLUMN_SQ_LNK_NAME + ", " + + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + COLUMN_SQ_LNK_ENABLED + ", " + + COLUMN_SQ_LNK_CREATION_USER + ", " + + COLUMN_SQ_LNK_CREATION_DATE + ", " + + COLUMN_SQ_LNK_UPDATE_USER + ", " + + COLUMN_SQ_LNK_UPDATE_DATE + + " FROM " + TABLE_SQ_LINK + + " WHERE " + COLUMN_SQ_LNK_NAME + " = ?"; + + // DML: Select all links + public static final String STMT_SELECT_LINK_ALL = + "SELECT " + + COLUMN_SQ_LNK_ID + ", " + + COLUMN_SQ_LNK_NAME + ", " + + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + COLUMN_SQ_LNK_ENABLED + ", " + + COLUMN_SQ_LNK_CREATION_USER + ", " + + COLUMN_SQ_LNK_CREATION_DATE + ", " + + COLUMN_SQ_LNK_UPDATE_USER + ", " + + COLUMN_SQ_LNK_UPDATE_DATE + + " FROM " + TABLE_SQ_LINK; + + // DML: Select all links for a specific connector. + public static final String STMT_SELECT_LINK_FOR_CONNECTOR_CONFIGURABLE = + "SELECT " + + COLUMN_SQ_LNK_ID + ", " + + COLUMN_SQ_LNK_NAME + ", " + + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + COLUMN_SQ_LNK_ENABLED + ", " + + COLUMN_SQ_LNK_CREATION_USER + ", " + + COLUMN_SQ_LNK_CREATION_DATE + ", " + + COLUMN_SQ_LNK_UPDATE_USER + ", " + + COLUMN_SQ_LNK_UPDATE_DATE + + " FROM " + TABLE_SQ_LINK + + " WHERE " + COLUMN_SQ_LNK_CONFIGURABLE + " = ?"; + + // DML: Check if given link exists + public static final String STMT_SELECT_LINK_CHECK_BY_ID = + "SELECT count(*) FROM " + TABLE_SQ_LINK + + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; + + /**********JOB TABLE **************/ + // DML: Insert new job + public static final String STMT_INSERT_JOB = + "INSERT INTO " + TABLE_SQ_JOB + " (" + + COLUMN_SQB_NAME + ", " + + COLUMN_SQB_FROM_LINK + ", " + + COLUMN_SQB_TO_LINK + ", " + + COLUMN_SQB_ENABLED + ", " + + COLUMN_SQB_CREATION_USER + ", " + + COLUMN_SQB_CREATION_DATE + ", " + + COLUMN_SQB_UPDATE_USER + ", " + + COLUMN_SQB_UPDATE_DATE + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + + // DML: Insert new job inputs + public static final String STMT_INSERT_JOB_INPUT = + "INSERT INTO " + TABLE_SQ_JOB_INPUT + " (" + + COLUMN_SQBI_JOB + ", " + + COLUMN_SQBI_INPUT + ", " + + COLUMN_SQBI_VALUE + + ") VALUES (?, ?, ?)"; + + public static final String STMT_UPDATE_JOB = + "UPDATE " + TABLE_SQ_JOB + " SET " + + COLUMN_SQB_NAME + " = ?, " + + COLUMN_SQB_UPDATE_USER + " = ?, " + + COLUMN_SQB_UPDATE_DATE + " = ? " + + " WHERE " + COLUMN_SQB_ID + " = ?"; + + // DML: Enable or disable job + public static final String STMT_ENABLE_JOB = + "UPDATE " + TABLE_SQ_JOB + " SET " + + COLUMN_SQB_ENABLED + " = ? " + + " WHERE " + COLUMN_SQB_ID + " = ?"; + + // DML: Delete rows from job input table + public static final String STMT_DELETE_JOB_INPUT = + "DELETE FROM " + TABLE_SQ_JOB_INPUT + + " WHERE " + COLUMN_SQBI_JOB + " = ?"; + + // DML: Delete row from job table + public static final String STMT_DELETE_JOB = + "DELETE FROM " + TABLE_SQ_JOB + + " WHERE " + COLUMN_SQB_ID + " = ?"; + + // DML: Check if given job exists + public static final String STMT_SELECT_JOB_CHECK_BY_ID = + "SELECT count(*) FROM " + TABLE_SQ_JOB + + " WHERE " + COLUMN_SQB_ID + " = ?"; + + // DML: Check if there are jobs for given link + public static final String STMT_SELECT_JOBS_FOR_LINK_CHECK = + "SELECT" + + " count(*)" + + " FROM " + TABLE_SQ_JOB + + " JOIN " + TABLE_SQ_LINK + + " ON " + COLUMN_SQB_FROM_LINK + " = " + COLUMN_SQ_LNK_ID + + " WHERE " + COLUMN_SQ_LNK_ID + " = ? "; + + //DML: Select all jobs + public static final String STMT_SELECT_JOB = + "SELECT " + + "FROM_CONNECTOR." + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + "TO_CONNECTOR." + COLUMN_SQ_LNK_CONFIGURABLE + ", " + + "JOB." + COLUMN_SQB_ID + ", " + + "JOB." + COLUMN_SQB_NAME + ", " + + "JOB." + COLUMN_SQB_FROM_LINK + ", " + + "JOB." + COLUMN_SQB_TO_LINK + ", " + + "JOB." + COLUMN_SQB_ENABLED + ", " + + "JOB." + COLUMN_SQB_CREATION_USER + ", " + + "JOB." + COLUMN_SQB_CREATION_DATE + ", " + + "JOB." + COLUMN_SQB_UPDATE_USER + ", " + + "JOB." + COLUMN_SQB_UPDATE_DATE + + " FROM " + TABLE_SQ_JOB + " JOB" + + " LEFT JOIN " + TABLE_SQ_LINK + " FROM_CONNECTOR" + + " ON " + COLUMN_SQB_FROM_LINK + " = FROM_CONNECTOR." + COLUMN_SQ_LNK_ID + + " LEFT JOIN " + TABLE_SQ_LINK + " TO_CONNECTOR" + + " ON " + COLUMN_SQB_TO_LINK + " = TO_CONNECTOR." + COLUMN_SQ_LNK_ID; + + // DML: Select one specific job + public static final String STMT_SELECT_JOB_SINGLE_BY_ID = + STMT_SELECT_JOB + " WHERE " + COLUMN_SQB_ID + " = ?"; + + // DML: Select one specific job + public static final String STMT_SELECT_JOB_SINGLE_BY_NAME = + STMT_SELECT_JOB + " WHERE " + COLUMN_SQB_NAME + " = ?"; + + // DML: Select all jobs for a Connector + public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR_CONFIGURABLE = + STMT_SELECT_JOB + + " WHERE FROM_LINK." + COLUMN_SQ_LNK_CONFIGURABLE + " = ? OR TO_LINK." + + COLUMN_SQ_LNK_CONFIGURABLE + " = ?"; + + /**********SUBMISSION TABLE **************/ + // DML: Insert new submission + public static final String STMT_INSERT_SUBMISSION = + "INSERT INTO " + TABLE_SQ_SUBMISSION + "(" + + COLUMN_SQS_JOB + ", " + + COLUMN_SQS_STATUS + ", " + + COLUMN_SQS_CREATION_USER + ", " + + COLUMN_SQS_CREATION_DATE + ", " + + COLUMN_SQS_UPDATE_USER + ", " + + COLUMN_SQS_UPDATE_DATE + ", " + + COLUMN_SQS_EXTERNAL_ID + ", " + + COLUMN_SQS_EXTERNAL_LINK + ", " + + COLUMN_SQS_EXCEPTION + ", " + + COLUMN_SQS_EXCEPTION_TRACE + ") " + + " VALUES(?, ?, ?, ?, ?, ?, ?, substr(?, 1, 150) , substr(?, 1, 150), substr(?, 1, 750))"; + + // DML: Update existing submission + public static final String STMT_UPDATE_SUBMISSION = + "UPDATE " + TABLE_SQ_SUBMISSION + " SET " + + COLUMN_SQS_STATUS + " = ?, " + + COLUMN_SQS_UPDATE_USER + " = ?, " + + COLUMN_SQS_UPDATE_DATE + " = ?, " + + COLUMN_SQS_EXCEPTION + " = ?, " + + COLUMN_SQS_EXCEPTION_TRACE + " = ?" + + " WHERE " + COLUMN_SQS_ID + " = ?"; + + // DML: Check if given submission exists + public static final String STMT_SELECT_SUBMISSION_CHECK = + "SELECT" + + " count(*)" + + " FROM " + TABLE_SQ_SUBMISSION + + " WHERE " + COLUMN_SQS_ID + " = ?"; + + // DML: Purge old entries + public static final String STMT_PURGE_SUBMISSIONS = + "DELETE FROM " + TABLE_SQ_SUBMISSION + + " WHERE " + COLUMN_SQS_UPDATE_DATE + " < ?"; + + // DML: Get unfinished + public static final String STMT_SELECT_SUBMISSION_UNFINISHED = + "SELECT " + + COLUMN_SQS_ID + ", " + + COLUMN_SQS_JOB + ", " + + COLUMN_SQS_STATUS + ", " + + COLUMN_SQS_CREATION_USER + ", " + + COLUMN_SQS_CREATION_DATE + ", " + + COLUMN_SQS_UPDATE_USER + ", " + + COLUMN_SQS_UPDATE_DATE + ", " + + COLUMN_SQS_EXTERNAL_ID + ", " + + COLUMN_SQS_EXTERNAL_LINK + ", " + + COLUMN_SQS_EXCEPTION + ", " + + COLUMN_SQS_EXCEPTION_TRACE + + " FROM " + TABLE_SQ_SUBMISSION + + " WHERE " + COLUMN_SQS_STATUS + " = ?"; + + // DML : Get all submissions + public static final String STMT_SELECT_SUBMISSIONS = + "SELECT " + + COLUMN_SQS_ID + ", " + + COLUMN_SQS_JOB + ", " + + COLUMN_SQS_STATUS + ", " + + COLUMN_SQS_CREATION_USER + ", " + + COLUMN_SQS_CREATION_DATE + ", " + + COLUMN_SQS_UPDATE_USER + ", " + + COLUMN_SQS_UPDATE_DATE + ", " + + COLUMN_SQS_EXTERNAL_ID + ", " + + COLUMN_SQS_EXTERNAL_LINK + ", " + + COLUMN_SQS_EXCEPTION + ", " + + COLUMN_SQS_EXCEPTION_TRACE + + " FROM " + TABLE_SQ_SUBMISSION + + " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC"; + + // DML: Get submissions for a job + public static final String STMT_SELECT_SUBMISSIONS_FOR_JOB = + "SELECT " + + COLUMN_SQS_ID + ", " + + COLUMN_SQS_JOB + ", " + + COLUMN_SQS_STATUS + ", " + + COLUMN_SQS_CREATION_USER + ", " + + COLUMN_SQS_CREATION_DATE + ", " + + COLUMN_SQS_UPDATE_USER + ", " + + COLUMN_SQS_UPDATE_DATE + ", " + + COLUMN_SQS_EXTERNAL_ID + ", " + + COLUMN_SQS_EXTERNAL_LINK + ", " + + COLUMN_SQS_EXCEPTION + ", " + + COLUMN_SQS_EXCEPTION_TRACE + + " FROM " + TABLE_SQ_SUBMISSION + + " WHERE " + COLUMN_SQS_JOB + " = ?" + + " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC"; + + // DML: Select counter group + public static final String STMT_SELECT_COUNTER_GROUP = + "SELECT " + + COLUMN_SQG_ID + ", " + + COLUMN_SQG_NAME + " " + + "FROM " + TABLE_SQ_COUNTER_GROUP + " " + + "WHERE " + COLUMN_SQG_NAME + " = substr(?, 1, 75)"; + + // DML: Insert new counter group + public static final String STMT_INSERT_COUNTER_GROUP = + "INSERT INTO " + TABLE_SQ_COUNTER_GROUP + " (" + + COLUMN_SQG_NAME + ") " + + "VALUES (substr(?, 1, 75))"; + + // DML: Select counter + public static final String STMT_SELECT_COUNTER = + "SELECT " + + COLUMN_SQR_ID + ", " + + COLUMN_SQR_NAME + " " + + "FROM " + TABLE_SQ_COUNTER + " " + + "WHERE " + COLUMN_SQR_NAME + " = substr(?, 1, 75)"; + + // DML: Insert new counter + public static final String STMT_INSERT_COUNTER = + "INSERT INTO " + TABLE_SQ_COUNTER + " (" + + COLUMN_SQR_NAME + ") " + + "VALUES (substr(?, 1, 75))"; + + // DML: Insert new counter submission + public static final String STMT_INSERT_COUNTER_SUBMISSION = + "INSERT INTO " + TABLE_SQ_COUNTER_SUBMISSION + " (" + + COLUMN_SQRS_GROUP + ", " + + COLUMN_SQRS_COUNTER + ", " + + COLUMN_SQRS_SUBMISSION + ", " + + COLUMN_SQRS_VALUE + ") " + + "VALUES (?, ?, ?, ?)"; + + // DML: Select counter submission + public static final String STMT_SELECT_COUNTER_SUBMISSION = + "SELECT " + + COLUMN_SQG_NAME + ", " + + COLUMN_SQR_NAME + ", " + + COLUMN_SQRS_VALUE + " " + + "FROM " + TABLE_SQ_COUNTER_SUBMISSION + " " + + "LEFT JOIN " + TABLE_SQ_COUNTER_GROUP + + " ON " + COLUMN_SQRS_GROUP + " = " + COLUMN_SQG_ID + " " + + "LEFT JOIN " + TABLE_SQ_COUNTER + + " ON " + COLUMN_SQRS_COUNTER + " = " + COLUMN_SQR_ID + " " + + "WHERE " + COLUMN_SQRS_SUBMISSION + " = ? "; + + // DML: Delete rows from counter submission table + public static final String STMT_DELETE_COUNTER_SUBMISSION = + "DELETE FROM " + TABLE_SQ_COUNTER_SUBMISSION + + " WHERE " + COLUMN_SQRS_SUBMISSION + " = ?"; + + /******* CONFIG and CONNECTOR DIRECTIONS ****/ + public static final String STMT_INSERT_SQ_CONNECTOR_DIRECTIONS = + "INSERT INTO " + TABLE_SQ_CONNECTOR_DIRECTIONS + " " + + "(" + COLUMN_SQCD_CONNECTOR + ", " + COLUMN_SQCD_DIRECTION + ")" + + " VALUES (?, ?)"; + + public static final String STMT_INSERT_SQ_CONFIG_DIRECTIONS = + "INSERT INTO " + TABLE_SQ_CONFIG_DIRECTIONS + " " + + "(" + COLUMN_SQ_CFG_DIR_CONFIG + ", " + COLUMN_SQ_CFG_DIR_DIRECTION + ")" + + " VALUES (?, ?)"; + + public static final String STMT_SELECT_SQ_CONNECTOR_DIRECTIONS_ALL = + "SELECT " + COLUMN_SQCD_CONNECTOR + ", " + COLUMN_SQCD_DIRECTION + + " FROM " + TABLE_SQ_CONNECTOR_DIRECTIONS; + + public static final String STMT_SELECT_SQ_CONNECTOR_DIRECTIONS = + STMT_SELECT_SQ_CONNECTOR_DIRECTIONS_ALL + " WHERE " + + COLUMN_SQCD_CONNECTOR + " = ?"; + + public static final String STMT_SELECT_SQ_CONFIG_DIRECTIONS_ALL = + "SELECT " + COLUMN_SQ_CFG_DIR_CONFIG + ", " + COLUMN_SQ_CFG_DIR_DIRECTION + + " FROM " + TABLE_SQ_CONFIG_DIRECTIONS; + + public static final String STMT_SELECT_SQ_CONFIG_DIRECTIONS = + STMT_SELECT_SQ_CONFIG_DIRECTIONS_ALL + " WHERE " + + COLUMN_SQ_CFG_DIR_CONFIG + " = ?"; +} diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java new file mode 100644 index 00000000..173dcb8f --- /dev/null +++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java @@ -0,0 +1,248 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.repository.common; + +public final class CommonRepositorySchemaConstants { + + public static final String SCHEMA_SQOOP = "SQOOP"; + + private static final String SCHEMA_PREFIX = SCHEMA_SQOOP + "."; + + // SQ_DIRECTION + + public static final String TABLE_SQ_DIRECTION_NAME = "SQ_DIRECTION"; + + public static final String TABLE_SQ_DIRECTION = SCHEMA_PREFIX + TABLE_SQ_DIRECTION_NAME; + + public static final String COLUMN_SQD_ID = "SQD_ID"; + + public static final String COLUMN_SQD_NAME = "SQD_NAME"; + + // SQ_CONFIGURABLE + + public static final String TABLE_SQ_CONFIGURABLE_NAME = "SQ_CONFIGURABLE"; + + public static final String TABLE_SQ_CONFIGURABLE = SCHEMA_PREFIX + TABLE_SQ_CONFIGURABLE_NAME; + + public static final String COLUMN_SQC_ID = "SQC_ID"; + + public static final String COLUMN_SQC_NAME = "SQC_NAME"; + + public static final String COLUMN_SQC_CLASS = "SQC_CLASS"; + + public static final String COLUMN_SQC_VERSION = "SQC_VERSION"; + + public static final String COLUMN_SQC_TYPE = "SQC_TYPE"; + + // SQ_CONNECTOR_DIRECTIONS + + public static final String TABLE_SQ_CONNECTOR_DIRECTIONS_NAME = "SQ_CONNECTOR_DIRECTIONS"; + + public static final String TABLE_SQ_CONNECTOR_DIRECTIONS = SCHEMA_PREFIX + + TABLE_SQ_CONNECTOR_DIRECTIONS_NAME; + + public static final String COLUMN_SQCD_CONNECTOR = "SQCD_CONNECTOR"; + + public static final String COLUMN_SQCD_DIRECTION = "SQCD_DIRECTION"; + + // SQ_CONFIG + + public static final String TABLE_SQ_CONFIG_NAME = "SQ_CONFIG"; + + public static final String TABLE_SQ_CONFIG = SCHEMA_PREFIX + TABLE_SQ_CONFIG_NAME; + + public static final String COLUMN_SQ_CFG_ID = "SQ_CFG_ID"; + + public static final String COLUMN_SQ_CFG_CONFIGURABLE = "SQ_CFG_CONFIGURABLE"; + + public static final String COLUMN_SQ_CFG_DIRECTION = "SQ_CFG_DIRECTION"; + + public static final String COLUMN_SQ_CFG_NAME = "SQ_CFG_NAME"; + + public static final String COLUMN_SQ_CFG_TYPE = "SQ_CFG_TYPE"; + + public static final String COLUMN_SQ_CFG_INDEX = "SQ_CFG_INDEX"; + + // SQ_CONFIG_DIRECTIONS + + public static final String TABLE_SQ_CONFIG_DIRECTIONS_NAME = "SQ_CONFIG_DIRECTIONS"; + + public static final String TABLE_SQ_CONFIG_DIRECTIONS = SCHEMA_PREFIX + + TABLE_SQ_CONFIG_DIRECTIONS_NAME; + + public static final String COLUMN_SQ_CFG_DIR_CONFIG = "SQ_CFG_DIR_CONFIG"; + + public static final String COLUMN_SQ_CFG_DIR_DIRECTION = "SQ_CFG_DIR_DIRECTION"; + + // SQ_INPUT + + public static final String TABLE_SQ_INPUT_NAME = "SQ_INPUT"; + + public static final String TABLE_SQ_INPUT = SCHEMA_PREFIX + TABLE_SQ_INPUT_NAME; + + public static final String COLUMN_SQI_ID = "SQI_ID"; + + public static final String COLUMN_SQI_NAME = "SQI_NAME"; + + public static final String COLUMN_SQI_CONFIG = "SQI_CONFIG"; + + public static final String COLUMN_SQI_INDEX = "SQI_INDEX"; + + public static final String COLUMN_SQI_TYPE = "SQI_TYPE"; + + public static final String COLUMN_SQI_STRMASK = "SQI_STRMASK"; + + public static final String COLUMN_SQI_STRLENGTH = "SQI_STRLENGTH"; + + public static final String COLUMN_SQI_ENUMVALS = "SQI_ENUMVALS"; + + public static final String TABLE_SQ_LINK_NAME = "SQ_LINK"; + + public static final String TABLE_SQ_LINK = SCHEMA_PREFIX + TABLE_SQ_LINK_NAME; + + public static final String COLUMN_SQ_LNK_ID = "SQ_LNK_ID"; + + public static final String COLUMN_SQ_LNK_NAME = "SQ_LNK_NAME"; + + public static final String COLUMN_SQ_LNK_CONFIGURABLE = "SQ_LNK_CONFIGURABLE"; + + public static final String COLUMN_SQ_LNK_CREATION_USER = "SQ_LNK_CREATION_USER"; + + public static final String COLUMN_SQ_LNK_CREATION_DATE = "SQ_LNK_CREATION_DATE"; + + public static final String COLUMN_SQ_LNK_UPDATE_USER = "SQ_LNK_UPDATE_USER"; + + public static final String COLUMN_SQ_LNK_UPDATE_DATE = "SQ_LNK_UPDATE_DATE"; + + public static final String COLUMN_SQ_LNK_ENABLED = "SQ_LNK_ENABLED"; + + // SQ_JOB + + public static final String TABLE_SQ_JOB_NAME = "SQ_JOB"; + + public static final String TABLE_SQ_JOB = SCHEMA_PREFIX + TABLE_SQ_JOB_NAME; + + public static final String COLUMN_SQB_ID = "SQB_ID"; + + public static final String COLUMN_SQB_NAME = "SQB_NAME"; + + public static final String COLUMN_SQB_FROM_LINK = "SQB_FROM_LINK"; + + public static final String COLUMN_SQB_TO_LINK = "SQB_TO_LINK"; + + public static final String COLUMN_SQB_CREATION_USER = "SQB_CREATION_USER"; + + public static final String COLUMN_SQB_CREATION_DATE = "SQB_CREATION_DATE"; + + public static final String COLUMN_SQB_UPDATE_USER = "SQB_UPDATE_USER"; + + public static final String COLUMN_SQB_UPDATE_DATE = "SQB_UPDATE_DATE"; + + public static final String COLUMN_SQB_ENABLED = "SQB_ENABLED"; + + // SQ_LINK_INPUT + + public static final String TABLE_SQ_LINK_INPUT_NAME = "SQ_LINK_INPUT"; + + public static final String TABLE_SQ_LINK_INPUT = SCHEMA_PREFIX + TABLE_SQ_LINK_INPUT_NAME; + + public static final String COLUMN_SQ_LNKI_LINK = "SQ_LNKI_LINK"; + + public static final String COLUMN_SQ_LNKI_INPUT = "SQ_LNKI_INPUT"; + + public static final String COLUMN_SQ_LNKI_VALUE = "SQ_LNKI_VALUE"; + + // SQ_JOB_INPUT + + public static final String TABLE_SQ_JOB_INPUT_NAME = "SQ_JOB_INPUT"; + + public static final String TABLE_SQ_JOB_INPUT = SCHEMA_PREFIX + TABLE_SQ_JOB_INPUT_NAME; + + public static final String COLUMN_SQBI_JOB = "SQBI_JOB"; + + public static final String COLUMN_SQBI_INPUT = "SQBI_INPUT"; + + public static final String COLUMN_SQBI_VALUE = "SQBI_VALUE"; + + // SQ_SUBMISSION + + public static final String TABLE_SQ_SUBMISSION_NAME = "SQ_SUBMISSION"; + + public static final String TABLE_SQ_SUBMISSION = SCHEMA_PREFIX + TABLE_SQ_SUBMISSION_NAME; + + public static final String COLUMN_SQS_ID = "SQS_ID"; + + public static final String COLUMN_SQS_JOB = "SQS_JOB"; + + public static final String COLUMN_SQS_STATUS = "SQS_STATUS"; + + public static final String COLUMN_SQS_CREATION_USER = "SQS_CREATION_USER"; + + public static final String COLUMN_SQS_CREATION_DATE = "SQS_CREATION_DATE"; + + public static final String COLUMN_SQS_UPDATE_USER = "SQS_UPDATE_USER"; + + public static final String COLUMN_SQS_UPDATE_DATE = "SQS_UPDATE_DATE"; + + public static final String COLUMN_SQS_EXTERNAL_ID = "SQS_EXTERNAL_ID"; + + public static final String COLUMN_SQS_EXTERNAL_LINK = "SQS_EXTERNAL_LINK"; + + public static final String COLUMN_SQS_EXCEPTION = "SQS_EXCEPTION"; + + public static final String COLUMN_SQS_EXCEPTION_TRACE = "SQS_EXCEPTION_TRACE"; + + // SQ_COUNTER_GROUP + + public static final String TABLE_SQ_COUNTER_GROUP_NAME = "SQ_COUNTER_GROUP"; + + public static final String TABLE_SQ_COUNTER_GROUP = SCHEMA_PREFIX + TABLE_SQ_COUNTER_GROUP_NAME; + + public static final String COLUMN_SQG_ID = "SQG_ID"; + + public static final String COLUMN_SQG_NAME = "SQG_NAME"; + + // SQ_COUNTER_GROUP + + public static final String TABLE_SQ_COUNTER_NAME = "SQ_COUNTER"; + + public static final String TABLE_SQ_COUNTER = SCHEMA_PREFIX + TABLE_SQ_COUNTER_NAME; + + public static final String COLUMN_SQR_ID = "SQR_ID"; + + public static final String COLUMN_SQR_NAME = "SQR_NAME"; + + // SQ_COUNTER_SUBMISSION + + public static final String TABLE_SQ_COUNTER_SUBMISSION_NAME = "SQ_COUNTER_SUBMISSION"; + + public static final String TABLE_SQ_COUNTER_SUBMISSION = SCHEMA_PREFIX + + TABLE_SQ_COUNTER_SUBMISSION_NAME; + + public static final String COLUMN_SQRS_GROUP = "SQRS_GROUP"; + + public static final String COLUMN_SQRS_COUNTER = "SQRS_COUNTER"; + + public static final String COLUMN_SQRS_SUBMISSION = "SQRS_SUBMISSION"; + + public static final String COLUMN_SQRS_VALUE = "SQRS_VALUE"; + + private CommonRepositorySchemaConstants() { + // Disable explicit object creation + } +} \ No newline at end of file diff --git a/repository/repository-derby/pom.xml b/repository/repository-derby/pom.xml index 6ad6d64d..9be96db3 100644 --- a/repository/repository-derby/pom.xml +++ b/repository/repository-derby/pom.xml @@ -37,6 +37,11 @@ limitations under the License. sqoop-core + + org.apache.sqoop.repository + sqoop-repository-common + + org.apache.derby derby diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java index 8fbf47fc..d869cb7a 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoConstants.java @@ -24,7 +24,6 @@ public final class DerbyRepoConstants { @Deprecated // use only for the upgrade code public static final String SYSKEY_VERSION = "version"; - public static final String SYSKEY_DERBY_REPOSITORY_VERSION = "repository.version"; /** * Expected version of the repository structures. diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java index aad219eb..769544bb 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepoError.java @@ -24,174 +24,34 @@ public enum DerbyRepoError implements ErrorCode { /** An unknown error has occurred. */ DERBYREPO_0000("An unknown error has occurred"), - /** The Derby Repository handler was unable to determine if schema exists.*/ - DERBYREPO_0001("Unable to determine if schema exists"), - /** The system was unable to shutdown embedded derby repository server. */ - DERBYREPO_0002("Unable to shutdown embedded Derby instance"), + DERBYREPO_0001("Unable to shutdown embedded Derby instance"), /** The system was unable to run the specified query. */ - DERBYREPO_0003("Unable to run specified query"), - - /** The system was unable to query the repository for given entity */ - DERBYREPO_0004("Unable to retrieve entity data"), - - /** The repository contains more than one connector with same name */ - DERBYREPO_0005("Invalid entity state - multiple connectors with name"), - - /** The system does not support the given input type.*/ - DERBYREPO_0006("Unknown input type encountered"), - - /** The system does not support the given config type.*/ - DERBYREPO_0007("Unknown config type encountered"), - - /** No input was found for the given config. */ - DERBYREPO_0008("The config contains no input"), - - /** The system could not load the config due to unexpected position of input.*/ - DERBYREPO_0009("The config input retrieved does not match expected position"), - - /** - * The system could not load the connector due to unexpected position - * of config. - */ - DERBYREPO_0010("The config retrieved does not match expected position"), - - /** - * The system was not able to register entity due to a pre-assigned - * persistence identifier. - */ - DERBYREPO_0011("Entity cannot have preassigned persistence id"), + DERBYREPO_0002("Unable to run specified query"), /** * The system was unable to register various entities. */ - DERBYREPO_0012("Unexpected update count when registering entity"), + DERBYREPO_0003("Unexpected update count when registering entity"), /** * The system was unable to register entity due to a failure to retrieve * the generated identifier. */ - DERBYREPO_0013("Unable to retrieve generated identifier"), - - /** - * The system was unable to register connector due to a server - * error. - */ - DERBYREPO_0014("Registration of connector failed"), - - /** - * The system was not able to register connector due to an unexpected - * update count. - */ - DERBYREPO_0015("Unexpected update count on config registration"), - - /** - * The system was unable to register connector due to a failure to - * retrieve the generated identifier for a config. - */ - DERBYREPO_0016("Unable to retrieve generated identifier for config"), - - /** - * The system was unable to register connector due to an unexpected - * update count for config input registration. - */ - DERBYREPO_0017("Unexpected update count for config input"), - - /** - * The system was unable to register connector due to a failure to - * retrieve the generated identifier for a config input. - */ - DERBYREPO_0018("Unable to retrieve generated identifier for config input"), + DERBYREPO_0004("Unable to retrieve generated identifier"), /** We cant create new link in repository **/ - DERBYREPO_0019("Unable to create new link data"), - - /** We can't save values for input to repository **/ - DERBYREPO_0020("Unable to save input values to the repository"), - - /** We can't update link in repository **/ - DERBYREPO_0021("Unable to update link in repository"), - - /** We can't delete link in repository **/ - DERBYREPO_0022("Unable to delete link in repository"), - - /** We can't restore link from repository **/ - DERBYREPO_0023("Unable to load link from repository"), - - /** We can't restore specific link from repository **/ - DERBYREPO_0024("Unable to load specific link from repository"), - - /** We're unable to check if given link already exists */ - DERBYREPO_0025("Unable to check if given link exists"), - - /** We cant create new job in repository **/ - DERBYREPO_0026("Unable to create new job data"), - - /** We can't update job in repository **/ - DERBYREPO_0027("Unable to update job in repository"), - - /** We can't delete job in repository **/ - DERBYREPO_0028("Unable to delete job in repository"), - - /** We're unable to check if given job already exists */ - DERBYREPO_0029("Unable to check if given job exists"), - - /** We can't restore specific job from repository **/ - DERBYREPO_0030("Unable to load specific job from repository"), - - /** We can't restore job from repository **/ - DERBYREPO_0031("Unable to load job from repository"), - - /** Can't verify if link is referenced from somewhere **/ - DERBYREPO_0032("Unable to check if link is in use"), - - /** We're unable to check if given submission already exists */ - DERBYREPO_0033("Unable to check if given submission exists"), - - /** We cant create new submission in repository **/ - DERBYREPO_0034("Unable to create new submission data"), - - /** We can't update submission in repository **/ - DERBYREPO_0035("Unable to update submission in the repository"), - - /** Can't purge old submissions **/ - DERBYREPO_0036("Unable to purge old submissions"), - - /** Can't retrieve unfinished submissions **/ - DERBYREPO_0037("Can't retrieve unfinished submissions"), - - DERBYREPO_0038("Update of connector failed"), - - /** Can't retrieve all submissions **/ - DERBYREPO_0039("Can't retrieve all submissions"), - - /** Can't retrieve submissions for a job **/ - DERBYREPO_0040("Can't retrieve submissions for a job"), + DERBYREPO_0005("Unable to create new link data"), /** Can't detect version of the database structures **/ - DERBYREPO_0041("Can't detect version of repository storage"), + DERBYREPO_0006("Can't detect version of repository storage"), - /** Can't enable/disable link **/ - DERBYREPO_0042("Can't enable/disable link"), + /** Can't add directions **/ + DERBYREPO_0007("Could not add directions"), - /** Can't enable/disable job **/ - DERBYREPO_0043("Can't enable/disable job"), - - DERBYREPO_0044("Update of driver config failed"), - - DERBYREPO_0045("Can't retrieve all connectors"), - - DERBYREPO_0046("Could not add directions"), - - DERBYREPO_0047("Could not get ID of recently added direction"), - - DERBYREPO_0048("Could not register config direction"), - - DERBYREPO_0049("Could not set connector direction"), - - /** The system was unable to register driver due to a server error **/ - DERBYREPO_0050("Registration of driver failed"), + /** Can't get ID of direction **/ + DERBYREPO_0008("Could not get ID of recently added direction"), ; diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java index 059a8275..4d99a52a 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java @@ -29,8 +29,6 @@ import java.sql.Statement; import java.sql.Timestamp; import java.sql.Types; -import java.util.ArrayList; -import java.util.Date; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -38,47 +36,21 @@ import java.util.Set; import java.util.TreeMap; -import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.sqoop.common.Direction; -import org.apache.sqoop.common.DirectionError; import org.apache.sqoop.common.SqoopException; -import org.apache.sqoop.common.SupportedDirections; import org.apache.sqoop.connector.ConnectorHandler; import org.apache.sqoop.connector.ConnectorManagerUtils; -import org.apache.sqoop.driver.Driver; -import org.apache.sqoop.model.MBooleanInput; -import org.apache.sqoop.model.MConfig; -import org.apache.sqoop.model.MConfigType; -import org.apache.sqoop.model.MConfigurableType; -import org.apache.sqoop.model.MConnector; -import org.apache.sqoop.model.MDriver; -import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MEnumInput; -import org.apache.sqoop.model.MFromConfig; -import org.apache.sqoop.model.MInput; -import org.apache.sqoop.model.MInputType; -import org.apache.sqoop.model.MIntegerInput; -import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MLink; -import org.apache.sqoop.model.MLinkConfig; -import org.apache.sqoop.model.MMapInput; -import org.apache.sqoop.model.MStringInput; -import org.apache.sqoop.model.MSubmission; -import org.apache.sqoop.model.MToConfig; import org.apache.sqoop.repository.JdbcRepositoryContext; -import org.apache.sqoop.repository.JdbcRepositoryHandler; -import org.apache.sqoop.submission.SubmissionStatus; -import org.apache.sqoop.submission.counter.Counter; -import org.apache.sqoop.submission.counter.CounterGroup; -import org.apache.sqoop.submission.counter.Counters; +import org.apache.sqoop.repository.common.CommonRepoConstants; +import org.apache.sqoop.repository.common.CommonRepositoryHandler; /** * JDBC based repository handler for Derby database. * * Repository implementation for Derby database. */ -public class DerbyRepositoryHandler extends JdbcRepositoryHandler { +public class DerbyRepositoryHandler extends CommonRepositoryHandler { private static final Logger LOG = Logger.getLogger(DerbyRepositoryHandler.class); @@ -100,167 +72,8 @@ public class DerbyRepositoryHandler extends JdbcRepositoryHandler { * {@inheritDoc} */ @Override - public void registerConnector(MConnector mc, Connection conn) { - if (mc.hasPersistenceId()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0011, - mc.getUniqueName()); - } - mc.setPersistenceId(insertAndGetConnectorId(mc, conn)); - insertConfigsForConnector(mc, conn); - } - - /** - * Helper method to insert the configs from the into the - * repository. - * @param mDriver The driver instance to use to upgrade. - * @param conn JDBC link to use for updating the configs - */ - private void insertConfigsForDriver(MDriver mDriver, Connection conn) { - PreparedStatement baseConfigStmt = null; - PreparedStatement baseInputStmt = null; - try{ - baseConfigStmt = conn.prepareStatement(STMT_INSERT_INTO_CONFIG, - Statement.RETURN_GENERATED_KEYS); - - baseInputStmt = conn.prepareStatement(STMT_INSERT_INTO_INPUT, - Statement.RETURN_GENERATED_KEYS); - - // Register the job config type, since driver config is per job - registerConfigs(null, null, mDriver.getDriverConfig().getConfigs(), - MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); - - } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mDriver.toString(), ex); - } finally { - closeStatements(baseConfigStmt, baseInputStmt); - } - } - - /** - * Helper method to insert the configs from the MConnector into the - * repository. The job and connector configs within mc will get - * updated with the id of the configs when this function returns. - * @param mc The connector to use for updating configs - * @param conn JDBC connection to use for inserting the configs - */ - private void insertConfigsForConnector (MConnector mc, Connection conn) { - long connectorId = mc.getPersistenceId(); - PreparedStatement baseConfigStmt = null; - PreparedStatement baseInputStmt = null; - try{ - baseConfigStmt = conn.prepareStatement(STMT_INSERT_INTO_CONFIG, - Statement.RETURN_GENERATED_KEYS); - - baseInputStmt = conn.prepareStatement(STMT_INSERT_INTO_INPUT, - Statement.RETURN_GENERATED_KEYS); - - // Register link type config - registerConfigs(connectorId, null /* No direction for LINK type config*/, mc.getLinkConfig().getConfigs(), - MConfigType.LINK.name(), baseConfigStmt, baseInputStmt, conn); - - // Register both from/to job type config for connector - if (mc.getSupportedDirections().isDirectionSupported(Direction.FROM)) { - registerConfigs(connectorId, Direction.FROM, mc.getFromConfig().getConfigs(), - MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); - } - if (mc.getSupportedDirections().isDirectionSupported(Direction.TO)) { - registerConfigs(connectorId, Direction.TO, mc.getToConfig().getConfigs(), - MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); - } - } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0014, - mc.toString(), ex); - } finally { - closeStatements(baseConfigStmt, baseInputStmt); - } - - } - - private void insertConnectorDirection(Long connectorId, Direction direction, Connection conn) - throws SQLException { - PreparedStatement stmt = null; - - try { - stmt = conn.prepareStatement(STMT_INSERT_SQ_CONNECTOR_DIRECTIONS); - stmt.setLong(1, connectorId); - stmt.setLong(2, getDirection(direction, conn)); - - if (stmt.executeUpdate() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0049); - } - } finally { - closeStatements(stmt); - } - } - - private void insertConnectorDirections(Long connectorId, SupportedDirections directions, Connection conn) - throws SQLException { - if (directions.isDirectionSupported(Direction.FROM)) { - insertConnectorDirection(connectorId, Direction.FROM, conn); - } - - if (directions.isDirectionSupported(Direction.TO)) { - insertConnectorDirection(connectorId, Direction.TO, conn); - } - } - - private long insertAndGetConnectorId(MConnector mc, Connection conn) { - PreparedStatement baseConnectorStmt = null; - try { - baseConnectorStmt = conn.prepareStatement(STMT_INSERT_INTO_CONFIGURABLE, - Statement.RETURN_GENERATED_KEYS); - baseConnectorStmt.setString(1, mc.getUniqueName()); - baseConnectorStmt.setString(2, mc.getClassName()); - baseConnectorStmt.setString(3, mc.getVersion()); - baseConnectorStmt.setString(4, mc.getType().name()); - - int baseConnectorCount = baseConnectorStmt.executeUpdate(); - if (baseConnectorCount != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0012, - Integer.toString(baseConnectorCount)); - } - - ResultSet rsetConnectorId = baseConnectorStmt.getGeneratedKeys(); - - if (!rsetConnectorId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - // connector configurable also have directions - insertConnectorDirections(rsetConnectorId.getLong(1), mc.getSupportedDirections(), conn); - return rsetConnectorId.getLong(1); - } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0014, mc.toString(), ex); - } finally { - closeStatements(baseConnectorStmt); - } - } - - private long insertAndGetDriverId(MDriver mDriver, Connection conn) { - PreparedStatement baseDriverStmt = null; - try { - baseDriverStmt = conn.prepareStatement(STMT_INSERT_INTO_CONFIGURABLE, - Statement.RETURN_GENERATED_KEYS); - baseDriverStmt.setString(1, mDriver.getUniqueName()); - baseDriverStmt.setString(2, Driver.getClassName()); - baseDriverStmt.setString(3, mDriver.getVersion()); - baseDriverStmt.setString(4, mDriver.getType().name()); - - int baseDriverCount = baseDriverStmt.executeUpdate(); - if (baseDriverCount != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0012, Integer.toString(baseDriverCount)); - } - - ResultSet rsetDriverId = baseDriverStmt.getGeneratedKeys(); - - if (!rsetDriverId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - return rsetDriverId.getLong(1); - } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0050, mDriver.toString(), ex); - } finally { - closeStatements(baseDriverStmt); - } + public String name() { + return "Derby"; } /** @@ -301,7 +114,7 @@ public synchronized void shutdown() { // Shutdown for one db instance is expected to raise SQL STATE 45000 if (ex.getErrorCode() != 45000) { throw new SqoopException( - DerbyRepoError.DERBYREPO_0002, shutDownUrl, ex); + DerbyRepoError.DERBYREPO_0001, shutDownUrl, ex); } LOG.info("Embedded Derby shutdown raised SQL STATE " + "45000 as expected."); @@ -350,7 +163,7 @@ public int detectRepositoryVersion(Connection conn) { } } catch (SQLException e) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0041, e); + throw new SqoopException(DerbyRepoError.DERBYREPO_0006, e); } finally { closeResultSets(rs); } @@ -360,7 +173,7 @@ public int detectRepositoryVersion(Connection conn) { // NOTE: Since we can different types of version stored in system table, we renamed the // key name for the repository version from "version" to "repository.version" for clarity stmt = conn.prepareStatement(STMT_SELECT_DEPRECATED_OR_NEW_SYSTEM_VERSION); - stmt.setString(1, DerbyRepoConstants.SYSKEY_DERBY_REPOSITORY_VERSION); + stmt.setString(1, CommonRepoConstants.SYSKEY_VERSION); stmt.setString(2, DerbyRepoConstants.SYSKEY_VERSION); rs = stmt.executeQuery(); @@ -537,8 +350,8 @@ private void renameConnectorToConfigurable(Connection conn) { private void upgradeRepositoryVersion(Connection conn) { // remove the deprecated sys version runQuery(STMT_DELETE_SYSTEM, conn, DerbyRepoConstants.SYSKEY_VERSION); - runQuery(STMT_DELETE_SYSTEM, conn, DerbyRepoConstants.SYSKEY_DERBY_REPOSITORY_VERSION); - runQuery(STMT_INSERT_SYSTEM, conn, DerbyRepoConstants.SYSKEY_DERBY_REPOSITORY_VERSION, "" + runQuery(STMT_DELETE_SYSTEM, conn, CommonRepoConstants.SYSKEY_VERSION); + runQuery(STMT_INSERT_SYSTEM, conn, CommonRepoConstants.SYSKEY_VERSION, "" + DerbyRepoConstants.LATEST_DERBY_REPOSITORY_VERSION); } /** @@ -556,7 +369,7 @@ protected Map insertDirections(Connection conn) { insertDirectionStmt = conn.prepareStatement(STMT_INSERT_DIRECTION, Statement.RETURN_GENERATED_KEYS); insertDirectionStmt.setString(1, direction.toString()); if (insertDirectionStmt.executeUpdate() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0046, "Could not add directions FROM and TO."); + throw new SqoopException(DerbyRepoError.DERBYREPO_0007, "Could not add directions FROM and TO."); } ResultSet directionId = insertDirectionStmt.getGeneratedKeys(); @@ -567,7 +380,7 @@ protected Map insertDirections(Connection conn) { directionMap.put(direction, directionId.getLong(1)); } else { - throw new SqoopException(DerbyRepoError.DERBYREPO_0047, "Could not get ID of direction " + direction); + throw new SqoopException(DerbyRepoError.DERBYREPO_0008, "Could not get ID of direction " + direction); } } } catch (SQLException e) { @@ -779,7 +592,7 @@ protected long registerHdfsConnector(Connection conn) { } } } catch (SQLException e) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); + throw new SqoopException(DerbyRepoError.DERBYREPO_0004); } break; @@ -817,13 +630,13 @@ private Long createHdfsConnection(Connection conn, Long connectorId) { result = stmt.executeUpdate(); if (result != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0012, + throw new SqoopException(DerbyRepoError.DERBYREPO_0003, Integer.toString(result)); } ResultSet rsetConnectionId = stmt.getGeneratedKeys(); if (!rsetConnectionId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); + throw new SqoopException(DerbyRepoError.DERBYREPO_0004); } if (LOG.isTraceEnabled()) { @@ -832,174 +645,12 @@ private Long createHdfsConnection(Connection conn, Long connectorId) { return rsetConnectionId.getLong(1); } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0019, ex); + throw new SqoopException(DerbyRepoError.DERBYREPO_0005, ex); } finally { closeStatements(stmt); } } - /** - * {@inheritDoc} - */ - @Override - public boolean isRespositorySuitableForUse(Connection conn) { - // TODO(jarcec): Verify that all structures are present (e.g. something like corruption validation) - // NOTE: At this point is is just checking if the repo version matches the version - // in the upgraded code - return detectRepositoryVersion(conn) == DerbyRepoConstants.LATEST_DERBY_REPOSITORY_VERSION; - } - - /** - * {@inheritDoc} - */ - @Override - public MConnector findConnector(String shortName, Connection conn) { - if (LOG.isDebugEnabled()) { - LOG.debug("Looking up connector: " + shortName); - } - MConnector mc = null; - PreparedStatement connectorFetchStmt = null; - try { - connectorFetchStmt = conn.prepareStatement(STMT_SELECT_FROM_CONFIGURABLE); - connectorFetchStmt.setString(1, shortName); - - List connectors = loadConnectors(connectorFetchStmt, conn); - - if (connectors.size() == 0) { - LOG.debug("No connector found by name: " + shortName); - return null; - } else if (connectors.size() == 1) { - LOG.debug("Looking up connector: " + shortName + ", found: " + mc); - return connectors.get(0); - } else { - throw new SqoopException(DerbyRepoError.DERBYREPO_0005, shortName); - } - - } catch (SQLException ex) { - logException(ex, shortName); - throw new SqoopException(DerbyRepoError.DERBYREPO_0004, shortName, ex); - } finally { - closeStatements(connectorFetchStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List findConnectors(Connection conn) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_CONFIGURABLE_ALL_FOR_TYPE); - stmt.setString(1, MConfigurableType.CONNECTOR.name()); - - return loadConnectors(stmt,conn); - } catch (SQLException ex) { - logException(ex); - throw new SqoopException(DerbyRepoError.DERBYREPO_0045, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void registerDriver(MDriver mDriver, Connection conn) { - if (mDriver.hasPersistenceId()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0011, mDriver.getUniqueName()); - } - mDriver.setPersistenceId(insertAndGetDriverId(mDriver, conn)); - insertConfigsforDriver(mDriver, conn); - } - - private void insertConfigsforDriver(MDriver mDriver, Connection conn) { - PreparedStatement baseConfigStmt = null; - PreparedStatement baseInputStmt = null; - try { - baseConfigStmt = conn.prepareStatement(STMT_INSERT_INTO_CONFIG, - Statement.RETURN_GENERATED_KEYS); - baseInputStmt = conn.prepareStatement(STMT_INSERT_INTO_INPUT, - Statement.RETURN_GENERATED_KEYS); - - // Register a driver config as a job type with no owner/connector and direction - registerConfigs(mDriver.getPersistenceId(), null /* no direction*/, mDriver.getDriverConfig().getConfigs(), - MConfigType.JOB.name(), baseConfigStmt, baseInputStmt, conn); - - } catch (SQLException ex) { - logException(ex, mDriver); - throw new SqoopException(DerbyRepoError.DERBYREPO_0014, ex); - } finally { - closeStatements(baseConfigStmt, baseInputStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public MDriver findDriver(String shortName, Connection conn) { - LOG.debug("Looking up Driver and config "); - PreparedStatement driverFetchStmt = null; - PreparedStatement driverConfigFetchStmt = null; - PreparedStatement driverConfigInputFetchStmt = null; - - MDriver mDriver; - try { - driverFetchStmt = conn.prepareStatement(STMT_SELECT_FROM_CONFIGURABLE); - driverFetchStmt.setString(1, shortName); - - ResultSet rsDriverSet = driverFetchStmt.executeQuery(); - if (!rsDriverSet.next()) { - return null; - } - Long driverId = rsDriverSet.getLong(1); - String driverVersion = rsDriverSet.getString(4); - - driverConfigFetchStmt = conn.prepareStatement(STMT_SELECT_CONFIG_FOR_CONFIGURABLE); - driverConfigFetchStmt.setLong(1, driverId); - - driverConfigInputFetchStmt = conn.prepareStatement(STMT_SELECT_INPUT); - List driverConfigs = new ArrayList(); - loadDriverConfigs(driverConfigs, driverConfigFetchStmt, driverConfigInputFetchStmt, 1); - - if (driverConfigs.isEmpty()) { - return null; - } - mDriver = new MDriver(new MDriverConfig(driverConfigs), driverVersion); - mDriver.setPersistenceId(driverId); - - } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0004, "Driver", ex); - } finally { - if (driverConfigFetchStmt != null) { - try { - driverConfigFetchStmt.close(); - } catch (SQLException ex) { - LOG.error("Unable to close driver config fetch statement", ex); - } - } - if (driverConfigInputFetchStmt != null) { - try { - driverConfigInputFetchStmt.close(); - } catch (SQLException ex) { - LOG.error("Unable to close driver input fetch statement", ex); - } - } - if (driverFetchStmt != null) { - try { - driverFetchStmt.close(); - } catch (SQLException ex) { - LOG.error("Unable to close driver fetch statement", ex); - } - } - } - - LOG.debug("Looked up Driver and config"); - return mDriver; - } - /** * {@inheritDoc} */ @@ -1012,1871 +663,10 @@ public String validationQuery() { * {@inheritDoc} */ @Override - public void createLink(MLink link, Connection conn) { - PreparedStatement stmt = null; - int result; - try { - stmt = conn.prepareStatement(STMT_INSERT_LINK, - Statement.RETURN_GENERATED_KEYS); - stmt.setString(1, link.getName()); - stmt.setLong(2, link.getConnectorId()); - stmt.setBoolean(3, link.getEnabled()); - stmt.setString(4, link.getCreationUser()); - stmt.setTimestamp(5, new Timestamp(link.getCreationDate().getTime())); - stmt.setString(6, link.getLastUpdateUser()); - stmt.setTimestamp(7, new Timestamp(link.getLastUpdateDate().getTime())); - - result = stmt.executeUpdate(); - if (result != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0012, - Integer.toString(result)); - } - - ResultSet rsetConnectionId = stmt.getGeneratedKeys(); - - if (!rsetConnectionId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - - long connectionId = rsetConnectionId.getLong(1); - - createInputValues(STMT_INSERT_LINK_INPUT, - connectionId, - link.getConnectorLinkConfig().getConfigs(), - conn); - link.setPersistenceId(connectionId); - - } catch (SQLException ex) { - logException(ex, link); - throw new SqoopException(DerbyRepoError.DERBYREPO_0019, ex); - } finally { - closeStatements(stmt); - } - } - /** - * {@inheritDoc} - */ - @Override - public void updateLink(MLink link, Connection conn) { - PreparedStatement deleteStmt = null; - PreparedStatement updateStmt = null; - try { - // Firstly remove old values - deleteStmt = conn.prepareStatement(STMT_DELETE_LINK_INPUT); - deleteStmt.setLong(1, link.getPersistenceId()); - deleteStmt.executeUpdate(); - - // Update LINK_CONFIG table - updateStmt = conn.prepareStatement(STMT_UPDATE_LINK); - updateStmt.setString(1, link.getName()); - updateStmt.setString(2, link.getLastUpdateUser()); - updateStmt.setTimestamp(3, new Timestamp(new Date().getTime())); - - updateStmt.setLong(4, link.getPersistenceId()); - updateStmt.executeUpdate(); - - // And reinsert new values - createInputValues(STMT_INSERT_LINK_INPUT, - link.getPersistenceId(), - link.getConnectorLinkConfig().getConfigs(), - conn); - - } catch (SQLException ex) { - logException(ex, link); - throw new SqoopException(DerbyRepoError.DERBYREPO_0021, ex); - } finally { - closeStatements(deleteStmt, updateStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean existsLink(long id, Connection conn) { - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_LINK_CHECK_BY_ID); - stmt.setLong(1, id); - rs = stmt.executeQuery(); - - // Should be always valid in query with count - rs.next(); - - return rs.getLong(1) == 1; - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0025, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - } - - @Override - public boolean inUseLink(long connectionId, Connection conn) { - PreparedStatement stmt = null; - ResultSet rs = null; - - try { - stmt = conn.prepareStatement(STMT_SELECT_JOBS_FOR_LINK_CHECK); - stmt.setLong(1, connectionId); - rs = stmt.executeQuery(); - - // Should be always valid in case of count(*) query - rs.next(); - - return rs.getLong(1) != 0; - - } catch (SQLException e) { - logException(e, connectionId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0032, e); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - } - - @Override - public void enableLink(long connectionId, boolean enabled, Connection conn) { - PreparedStatement enableConn = null; - - try { - enableConn = conn.prepareStatement(STMT_ENABLE_LINK); - enableConn.setBoolean(1, enabled); - enableConn.setLong(2, connectionId); - enableConn.executeUpdate(); - } catch (SQLException ex) { - logException(ex, connectionId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0042, ex); - } finally { - closeStatements(enableConn); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteLink(long id, Connection conn) { - PreparedStatement dltConn = null; - - try { - deleteLinkInputs(id, conn); - dltConn = conn.prepareStatement(STMT_DELETE_LINK); - dltConn.setLong(1, id); - dltConn.executeUpdate(); - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0022, ex); - } finally { - closeStatements(dltConn); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteLinkInputs(long id, Connection conn) { - PreparedStatement dltConnInput = null; - try { - dltConnInput = conn.prepareStatement(STMT_DELETE_LINK_INPUT); - dltConnInput.setLong(1, id); - dltConnInput.executeUpdate(); - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0022, ex); - } finally { - closeStatements(dltConnInput); - } - } - - /** - * {@inheritDoc} - */ - @Override - public MLink findLink(long linkId, Connection conn) { - PreparedStatement linkFetchStmt = null; - try { - linkFetchStmt = conn.prepareStatement(STMT_SELECT_LINK_SINGLE); - linkFetchStmt.setLong(1, linkId); - - List links = loadLinks(linkFetchStmt, conn); - - if (links.size() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0024, "Couldn't find link with id " - + linkId); - } - - // Return the first and only one link object with the given id - return links.get(0); - - } catch (SQLException ex) { - logException(ex, linkId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0023, ex); - } finally { - closeStatements(linkFetchStmt); - } - } - - @Override - public MLink findLink(String linkName, Connection conn) { - PreparedStatement linkFetchStmt = null; - try { - linkFetchStmt = conn.prepareStatement(STMT_SELECT_LINK_SINGLE_BY_NAME); - linkFetchStmt.setString(1, linkName); - - List links = loadLinks(linkFetchStmt, conn); - - if (links.size() != 1) { - return null; - } - - // Return the first and only one link object with the given name - return links.get(0); - - } catch (SQLException ex) { - logException(ex, linkName); - throw new SqoopException(DerbyRepoError.DERBYREPO_0023, ex); - } finally { - closeStatements(linkFetchStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List findLinks(Connection conn) { - PreparedStatement linksFetchStmt = null; - try { - linksFetchStmt = conn.prepareStatement(STMT_SELECT_LINK_ALL); - - return loadLinks(linksFetchStmt, conn); - - } catch (SQLException ex) { - logException(ex); - throw new SqoopException(DerbyRepoError.DERBYREPO_0023, ex); - } finally { - closeStatements(linksFetchStmt); - } - } - - - /** - * - * {@inheritDoc} - * - */ - @Override - public List findLinksForConnector(long connectorId, Connection conn) { - PreparedStatement linkByConnectorFetchStmt = null; - try { - linkByConnectorFetchStmt = conn.prepareStatement(STMT_SELECT_LINK_FOR_CONNECTOR_CONFIGURABLE); - linkByConnectorFetchStmt.setLong(1, connectorId); - return loadLinks(linkByConnectorFetchStmt, conn); - } catch (SQLException ex) { - logException(ex, connectorId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0023, ex); - } finally { - closeStatements(linkByConnectorFetchStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void upgradeConnectorAndConfigs(MConnector mConnector, Connection conn) { - updateConnectorAndDeleteConfigs(mConnector, conn); - insertConfigsForConnector(mConnector, conn); - } - - private void updateConnectorAndDeleteConfigs(MConnector mConnector, Connection conn) { - PreparedStatement updateConnectorStatement = null; - PreparedStatement deleteConfig = null; - PreparedStatement deleteInput = null; - try { - updateConnectorStatement = conn.prepareStatement(STMT_UPDATE_CONFIGURABLE); - deleteInput = conn.prepareStatement(STMT_DELETE_INPUTS_FOR_CONFIGURABLE); - deleteConfig = conn.prepareStatement(STMT_DELETE_CONFIGS_FOR_CONFIGURABLE); - updateConnectorStatement.setString(1, mConnector.getUniqueName()); - updateConnectorStatement.setString(2, mConnector.getClassName()); - updateConnectorStatement.setString(3, mConnector.getVersion()); - updateConnectorStatement.setString(4, mConnector.getType().name()); - updateConnectorStatement.setLong(5, mConnector.getPersistenceId()); - - if (updateConnectorStatement.executeUpdate() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0038); - } - deleteInput.setLong(1, mConnector.getPersistenceId()); - deleteConfig.setLong(1, mConnector.getPersistenceId()); - deleteInput.executeUpdate(); - deleteConfig.executeUpdate(); - - } catch (SQLException e) { - logException(e, mConnector); - throw new SqoopException(DerbyRepoError.DERBYREPO_0038, e); - } finally { - closeStatements(updateConnectorStatement, deleteConfig, deleteInput); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void upgradeDriverAndConfigs(MDriver mDriver, Connection conn) { - updateDriverAndDeleteConfigs(mDriver, conn); - insertConfigsForDriver(mDriver, conn); - } - - private void updateDriverAndDeleteConfigs(MDriver mDriver, Connection conn) { - PreparedStatement updateDriverStatement = null; - PreparedStatement deleteConfig = null; - PreparedStatement deleteInput = null; - try { - updateDriverStatement = conn.prepareStatement(STMT_UPDATE_CONFIGURABLE); - deleteInput = conn.prepareStatement(STMT_DELETE_INPUTS_FOR_CONFIGURABLE); - deleteConfig = conn.prepareStatement(STMT_DELETE_CONFIGS_FOR_CONFIGURABLE); - updateDriverStatement.setString(1, mDriver.getUniqueName()); - updateDriverStatement.setString(2, Driver.getClassName()); - updateDriverStatement.setString(3, mDriver.getVersion()); - updateDriverStatement.setString(4, mDriver.getType().name()); - updateDriverStatement.setLong(5, mDriver.getPersistenceId()); - - if (updateDriverStatement.executeUpdate() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0038); - } - deleteInput.setLong(1, mDriver.getPersistenceId()); - deleteConfig.setLong(1, mDriver.getPersistenceId()); - deleteInput.executeUpdate(); - deleteConfig.executeUpdate(); - - } catch (SQLException e) { - logException(e, mDriver); - throw new SqoopException(DerbyRepoError.DERBYREPO_0044, e); - } finally { - closeStatements(updateDriverStatement, deleteConfig, deleteInput); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void createJob(MJob job, Connection conn) { - PreparedStatement stmt = null; - int result; - try { - stmt = conn.prepareStatement(STMT_INSERT_JOB, Statement.RETURN_GENERATED_KEYS); - stmt.setString(1, job.getName()); - stmt.setLong(2, job.getLinkId(Direction.FROM)); - stmt.setLong(3, job.getLinkId(Direction.TO)); - stmt.setBoolean(4, job.getEnabled()); - stmt.setString(5, job.getCreationUser()); - stmt.setTimestamp(6, new Timestamp(job.getCreationDate().getTime())); - stmt.setString(7, job.getLastUpdateUser()); - stmt.setTimestamp(8, new Timestamp(job.getLastUpdateDate().getTime())); - - result = stmt.executeUpdate(); - if (result != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0012, - Integer.toString(result)); - } - - ResultSet rsetJobId = stmt.getGeneratedKeys(); - - if (!rsetJobId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - - long jobId = rsetJobId.getLong(1); - - // from config for the job - createInputValues(STMT_INSERT_JOB_INPUT, - jobId, - job.getJobConfig(Direction.FROM).getConfigs(), - conn); - // to config for the job - createInputValues(STMT_INSERT_JOB_INPUT, - jobId, - job.getJobConfig(Direction.TO).getConfigs(), - conn); - // driver config per job - createInputValues(STMT_INSERT_JOB_INPUT, - jobId, - job.getDriverConfig().getConfigs(), - conn); - - job.setPersistenceId(jobId); - - } catch (SQLException ex) { - logException(ex, job); - throw new SqoopException(DerbyRepoError.DERBYREPO_0026, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void updateJob(MJob job, Connection conn) { - PreparedStatement deleteStmt = null; - PreparedStatement updateStmt = null; - try { - // Firstly remove old values - deleteStmt = conn.prepareStatement(STMT_DELETE_JOB_INPUT); - deleteStmt.setLong(1, job.getPersistenceId()); - deleteStmt.executeUpdate(); - - // Update job table - updateStmt = conn.prepareStatement(STMT_UPDATE_JOB); - updateStmt.setString(1, job.getName()); - updateStmt.setString(2, job.getLastUpdateUser()); - updateStmt.setTimestamp(3, new Timestamp(new Date().getTime())); - - updateStmt.setLong(4, job.getPersistenceId()); - updateStmt.executeUpdate(); - - // And reinsert new values - createInputValues(STMT_INSERT_JOB_INPUT, - job.getPersistenceId(), - job.getJobConfig(Direction.FROM).getConfigs(), - conn); - createInputValues(STMT_INSERT_JOB_INPUT, - job.getPersistenceId(), - job.getJobConfig(Direction.TO).getConfigs(), - conn); - createInputValues(STMT_INSERT_JOB_INPUT, - job.getPersistenceId(), - job.getDriverConfig().getConfigs(), - conn); - - } catch (SQLException ex) { - logException(ex, job); - throw new SqoopException(DerbyRepoError.DERBYREPO_0027, ex); - } finally { - closeStatements(deleteStmt, updateStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean existsJob(long id, Connection conn) { - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_JOB_CHECK_BY_ID); - stmt.setLong(1, id); - rs = stmt.executeQuery(); - - // Should be always valid in query with count - rs.next(); - - return rs.getLong(1) == 1; - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0029, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - } - - @Override - public boolean inUseJob(long jobId, Connection conn) { - MSubmission submission = findLastSubmissionForJob(jobId, conn); - - // We have no submissions and thus job can't be in use - if(submission == null) { - return false; - } - - // We can't remove running job - if(submission.getStatus().isRunning()) { - return true; - } - - return false; - } - - @Override - public void enableJob(long jobId, boolean enabled, Connection conn) { - PreparedStatement enableConn = null; - - try { - enableConn = conn.prepareStatement(STMT_ENABLE_JOB); - enableConn.setBoolean(1, enabled); - enableConn.setLong(2, jobId); - enableConn.executeUpdate(); - } catch (SQLException ex) { - logException(ex, jobId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0043, ex); - } finally { - closeStatements(enableConn); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteJob(long id, Connection conn) { - PreparedStatement dlt = null; - try { - deleteJobInputs(id, conn); - dlt = conn.prepareStatement(STMT_DELETE_JOB); - dlt.setLong(1, id); - dlt.executeUpdate(); - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0028, ex); - } finally { - closeStatements(dlt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void deleteJobInputs(long id, Connection conn) { - PreparedStatement dltInput = null; - try { - dltInput = conn.prepareStatement(STMT_DELETE_JOB_INPUT); - dltInput.setLong(1, id); - dltInput.executeUpdate(); - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0028, ex); - } finally { - closeStatements(dltInput); - } - } - - /** - * {@inheritDoc} - */ - @Override - public MJob findJob(long id, Connection conn) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_JOB_SINGLE_BY_ID); - stmt.setLong(1, id); - - List jobs = loadJobs(stmt, conn); - - if (jobs.size() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0030, "Couldn't find job with id " - + id); - } - - // Return the first and only one link object - return jobs.get(0); - - } catch (SQLException ex) { - logException(ex, id); - throw new SqoopException(DerbyRepoError.DERBYREPO_0031, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public MJob findJob(String name, Connection conn) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_JOB_SINGLE_BY_NAME); - stmt.setString(1, name); - - List jobs = loadJobs(stmt, conn); - - if (jobs.size() != 1) { - return null; - } - - // Return the first and only one link object - return jobs.get(0); - - } catch (SQLException ex) { - logException(ex, name); - throw new SqoopException(DerbyRepoError.DERBYREPO_0031, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List findJobs(Connection conn) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_JOB); - - return loadJobs(stmt, conn); - - } catch (SQLException ex) { - logException(ex); - throw new SqoopException(DerbyRepoError.DERBYREPO_0031, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List findJobsForConnector(long connectorId, Connection conn) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_ALL_JOBS_FOR_CONNECTOR_CONFIGURABLE); - stmt.setLong(1, connectorId); - stmt.setLong(2, connectorId); - return loadJobs(stmt, conn); - - } catch (SQLException ex) { - logException(ex, connectorId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0031, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void createSubmission(MSubmission submission, Connection conn) { - PreparedStatement stmt = null; - int result; - try { - stmt = conn.prepareStatement(STMT_INSERT_SUBMISSION, - Statement.RETURN_GENERATED_KEYS); - stmt.setLong(1, submission.getJobId()); - stmt.setString(2, submission.getStatus().name()); - stmt.setString(3, submission.getCreationUser()); - stmt.setTimestamp(4, new Timestamp(submission.getCreationDate().getTime())); - stmt.setString(5, submission.getLastUpdateUser()); - stmt.setTimestamp(6, new Timestamp(submission.getLastUpdateDate().getTime())); - stmt.setString(7, submission.getExternalId()); - stmt.setString(8, submission.getExternalLink()); - stmt.setString(9, submission.getExceptionInfo()); - stmt.setString(10, submission.getExceptionStackTrace()); - - result = stmt.executeUpdate(); - if (result != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0012, - Integer.toString(result)); - } - - ResultSet rsetSubmissionId = stmt.getGeneratedKeys(); - - if (!rsetSubmissionId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - - long submissionId = rsetSubmissionId.getLong(1); - - if(submission.getCounters() != null) { - createSubmissionCounters(submissionId, submission.getCounters(), conn); - } - - // Save created persistence id - submission.setPersistenceId(submissionId); - - } catch (SQLException ex) { - logException(ex, submission); - throw new SqoopException(DerbyRepoError.DERBYREPO_0034, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public boolean existsSubmission(long submissionId, Connection conn) { - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_SUBMISSION_CHECK); - stmt.setLong(1, submissionId); - rs = stmt.executeQuery(); - - // Should be always valid in query with count - rs.next(); - - return rs.getLong(1) == 1; - } catch (SQLException ex) { - logException(ex, submissionId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0033, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void updateSubmission(MSubmission submission, Connection conn) { - PreparedStatement stmt = null; - PreparedStatement deleteStmt = null; - try { - // Update properties in main table - stmt = conn.prepareStatement(STMT_UPDATE_SUBMISSION); - stmt.setString(1, submission.getStatus().name()); - stmt.setString(2, submission.getLastUpdateUser()); - stmt.setTimestamp(3, new Timestamp(submission.getLastUpdateDate().getTime())); - stmt.setString(4, submission.getExceptionInfo()); - stmt.setString(5, submission.getExceptionStackTrace()); - - stmt.setLong(6, submission.getPersistenceId()); - stmt.executeUpdate(); - - // Delete previous counters - deleteStmt = conn.prepareStatement(STMT_DELETE_COUNTER_SUBMISSION); - deleteStmt.setLong(1, submission.getPersistenceId()); - deleteStmt.executeUpdate(); - - // Reinsert new counters if needed - if(submission.getCounters() != null) { - createSubmissionCounters(submission.getPersistenceId(), submission.getCounters(), conn); - } - - } catch (SQLException ex) { - logException(ex, submission); - throw new SqoopException(DerbyRepoError.DERBYREPO_0035, ex); - } finally { - closeStatements(stmt, deleteStmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public void purgeSubmissions(Date threshold, Connection conn) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_PURGE_SUBMISSIONS); - stmt.setTimestamp(1, new Timestamp(threshold.getTime())); - stmt.executeUpdate(); - - } catch (SQLException ex) { - logException(ex, threshold); - throw new SqoopException(DerbyRepoError.DERBYREPO_0036, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List findUnfinishedSubmissions(Connection conn) { - List submissions = new LinkedList(); - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_SUBMISSION_UNFINISHED); - - for(SubmissionStatus status : SubmissionStatus.unfinished()) { - stmt.setString(1, status.name()); - rs = stmt.executeQuery(); - - while(rs.next()) { - submissions.add(loadSubmission(rs, conn)); - } - - rs.close(); - rs = null; - } - } catch (SQLException ex) { - logException(ex); - throw new SqoopException(DerbyRepoError.DERBYREPO_0037, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - - return submissions; - } - - /** - * {@inheritDoc} - */ - @Override - public List findSubmissions(Connection conn) { - List submissions = new LinkedList(); - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_SUBMISSIONS); - rs = stmt.executeQuery(); - - while(rs.next()) { - submissions.add(loadSubmission(rs, conn)); - } - - rs.close(); - rs = null; - } catch (SQLException ex) { - logException(ex); - throw new SqoopException(DerbyRepoError.DERBYREPO_0039, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - - return submissions; - } - - @Override - public List findSubmissionsForJob(long jobId, Connection conn) { - List submissions = new LinkedList(); - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_SUBMISSIONS_FOR_JOB); - stmt.setLong(1, jobId); - rs = stmt.executeQuery(); - - while(rs.next()) { - submissions.add(loadSubmission(rs, conn)); - } - - rs.close(); - rs = null; - } catch (SQLException ex) { - logException(ex); - throw new SqoopException(DerbyRepoError.DERBYREPO_0040, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - - return submissions; - } - - /** - * {@inheritDoc} - */ - @Override - public MSubmission findLastSubmissionForJob(long jobId, Connection conn) { - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_SUBMISSIONS_FOR_JOB); - stmt.setLong(1, jobId); - stmt.setMaxRows(1); - rs = stmt.executeQuery(); - - if(!rs.next()) { - return null; - } - - return loadSubmission(rs, conn); - } catch (SQLException ex) { - logException(ex, jobId); - throw new SqoopException(DerbyRepoError.DERBYREPO_0040, ex); - } finally { - closeResultSets(rs); - closeStatements(stmt); - } - } - - /** - * Stores counters for given submission in repository. - * - * @param submissionId Submission id - * @param counters Counters that should be stored - * @param conn Connection to derby repository - * @throws SQLException - */ - private void createSubmissionCounters(long submissionId, Counters counters, Connection conn) throws SQLException { - PreparedStatement stmt = null; - - try { - stmt = conn.prepareStatement(STMT_INSERT_COUNTER_SUBMISSION); - - for(CounterGroup group : counters) { - long groupId = getCounterGroupId(group, conn); - - for(Counter counter: group) { - long counterId = getCounterId(counter, conn); - - stmt.setLong(1, groupId); - stmt.setLong(2, counterId); - stmt.setLong(3, submissionId); - stmt.setLong(4, counter.getValue()); - - stmt.executeUpdate(); - } - } - } finally { - closeStatements(stmt); - } - } - - /** - * Resolves counter group database id. - * - * @param group Given group - * @param conn Connection to database - * @return Id - * @throws SQLException - */ - private long getCounterGroupId(CounterGroup group, Connection conn) throws SQLException { - PreparedStatement select = null; - PreparedStatement insert = null; - ResultSet rsSelect = null; - ResultSet rsInsert = null; - - try { - select = conn.prepareStatement(STMT_SELECT_COUNTER_GROUP); - select.setString(1, group.getName()); - - rsSelect = select.executeQuery(); - - if(rsSelect.next()) { - return rsSelect.getLong(1); - } - - insert = conn.prepareStatement(STMT_INSERT_COUNTER_GROUP, Statement.RETURN_GENERATED_KEYS); - insert.setString(1, group.getName()); - insert.executeUpdate(); - - rsInsert = insert.getGeneratedKeys(); - - if (!rsInsert.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - - return rsInsert.getLong(1); - } finally { - closeResultSets(rsSelect, rsInsert); - closeStatements(select, insert); - } - } - - /** - * Resolves counter id. - * - * @param counter Given counter - * @param conn Connection to database - * @return Id - * @throws SQLException - */ - private long getCounterId(Counter counter, Connection conn) throws SQLException { - PreparedStatement select = null; - PreparedStatement insert = null; - ResultSet rsSelect = null; - ResultSet rsInsert = null; - - try { - select = conn.prepareStatement(STMT_SELECT_COUNTER); - select.setString(1, counter.getName()); - - rsSelect = select.executeQuery(); - - if(rsSelect.next()) { - return rsSelect.getLong(1); - } - - insert = conn.prepareStatement(STMT_INSERT_COUNTER, Statement.RETURN_GENERATED_KEYS); - insert.setString(1, counter.getName()); - insert.executeUpdate(); - - rsInsert = insert.getGeneratedKeys(); - - if (!rsInsert.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0013); - } - - return rsInsert.getLong(1); - } finally { - closeResultSets(rsSelect, rsInsert); - closeStatements(select, insert); - } - } - - /** - * Create MSubmission structure from result set. - * - * @param rs Result set, only active row will be fetched - * @param conn Connection to database - * @return Created MSubmission structure - * @throws SQLException - */ - private MSubmission loadSubmission(ResultSet rs, Connection conn) throws SQLException { - MSubmission submission = new MSubmission(); - - submission.setPersistenceId(rs.getLong(1)); - submission.setJobId(rs.getLong(2)); - submission.setStatus(SubmissionStatus.valueOf(rs.getString(3))); - submission.setCreationUser(rs.getString(4)); - submission.setCreationDate(rs.getTimestamp(5)); - submission.setLastUpdateUser(rs.getString(6)); - submission.setLastUpdateDate(rs.getTimestamp(7)); - submission.setExternalId(rs.getString(8)); - submission.setExternalLink(rs.getString(9)); - submission.setExceptionInfo(rs.getString(10)); - submission.setExceptionStackTrace(rs.getString(11)); - - Counters counters = loadCountersSubmission(rs.getLong(1), conn); - submission.setCounters(counters); - - return submission; - } - - private Counters loadCountersSubmission(long submissionId, Connection conn) throws SQLException { - PreparedStatement stmt = null; - ResultSet rs = null; - try { - stmt = conn.prepareStatement(STMT_SELECT_COUNTER_SUBMISSION); - stmt.setLong(1, submissionId); - rs = stmt.executeQuery(); - - Counters counters = new Counters(); - - while (rs.next()) { - String groupName = rs.getString(1); - String counterName = rs.getString(2); - long value = rs.getLong(3); - - CounterGroup group = counters.getCounterGroup(groupName); - if (group == null) { - group = new CounterGroup(groupName); - counters.addCounterGroup(group); - } - - group.addCounter(new Counter(counterName, value)); - } - - if (counters.isEmpty()) { - return null; - } else { - return counters; - } - } finally { - closeStatements(stmt); - closeResultSets(rs); - } - } - - private Long getDirection(Direction direction, Connection conn) throws SQLException { - PreparedStatement directionStmt = null; - ResultSet rs = null; - - try { - directionStmt = conn.prepareStatement(STMT_SELECT_SQD_ID_BY_SQD_NAME); - directionStmt.setString(1, direction.toString()); - rs = directionStmt.executeQuery(); - - rs.next(); - return rs.getLong(1); - } finally { - if (rs != null) { - closeResultSets(rs); - } - if (directionStmt != null) { - closeStatements(directionStmt); - } - } - } - - private Direction getDirection(long directionId, Connection conn) throws SQLException { - PreparedStatement directionStmt = null; - ResultSet rs = null; - - try { - directionStmt = conn.prepareStatement(STMT_SELECT_SQD_NAME_BY_SQD_ID); - directionStmt.setLong(1, directionId); - rs = directionStmt.executeQuery(); - - rs.next(); - return Direction.valueOf(rs.getString(1)); - } finally { - if (rs != null) { - closeResultSets(rs); - } - if (directionStmt != null) { - closeStatements(directionStmt); - } - } - } - - private SupportedDirections findConnectorSupportedDirections(long connectorId, Connection conn) throws SQLException { - PreparedStatement connectorDirectionsStmt = null; - ResultSet rs = null; - - boolean from = false, to = false; - - try { - connectorDirectionsStmt = conn.prepareStatement(STMT_SELECT_SQ_CONNECTOR_DIRECTIONS); - connectorDirectionsStmt.setLong(1, connectorId); - rs = connectorDirectionsStmt.executeQuery(); - - while(rs.next()) { - switch(getDirection(rs.getLong(2), conn)) { - case FROM: - from = true; - break; - - case TO: - to = true; - break; - } - } - } finally { - if (rs != null) { - closeResultSets(rs); - } - if (connectorDirectionsStmt != null) { - closeStatements(connectorDirectionsStmt); - } - } - - return new SupportedDirections(from, to); - } - - private List loadConnectors(PreparedStatement stmt, Connection conn) throws SQLException { - List connectors = new ArrayList(); - ResultSet rsConnectors = null; - PreparedStatement connectorConfigFetchStmt = null; - PreparedStatement connectorConfigInputFetchStmt = null; - - try { - rsConnectors = stmt.executeQuery(); - connectorConfigFetchStmt = conn.prepareStatement(STMT_SELECT_CONFIG_FOR_CONFIGURABLE); - connectorConfigInputFetchStmt = conn.prepareStatement(STMT_SELECT_INPUT); - - while(rsConnectors.next()) { - long connectorId = rsConnectors.getLong(1); - String connectorName = rsConnectors.getString(2); - String connectorClassName = rsConnectors.getString(3); - String connectorVersion = rsConnectors.getString(4); - - connectorConfigFetchStmt.setLong(1, connectorId); - - List linkConfig = new ArrayList(); - List fromConfig = new ArrayList(); - List toConfig = new ArrayList(); - - loadConnectorConfigTypes(linkConfig, fromConfig, toConfig, connectorConfigFetchStmt, - connectorConfigInputFetchStmt, 1, conn); - - SupportedDirections supportedDirections - = findConnectorSupportedDirections(connectorId, conn); - MFromConfig fromJobConfig = null; - MToConfig toJobConfig = null; - if (supportedDirections.isDirectionSupported(Direction.FROM)) { - fromJobConfig = new MFromConfig(fromConfig); - } - if (supportedDirections.isDirectionSupported(Direction.TO)) { - toJobConfig = new MToConfig(toConfig); - } - MConnector mc = new MConnector(connectorName, connectorClassName, connectorVersion, - new MLinkConfig(linkConfig), fromJobConfig, toJobConfig); - mc.setPersistenceId(connectorId); - - connectors.add(mc); - } - } finally { - closeResultSets(rsConnectors); - closeStatements(connectorConfigFetchStmt, connectorConfigInputFetchStmt); - } - return connectors; - } - - private List loadLinks(PreparedStatement linkFetchStmt, Connection conn) throws SQLException { - List links = new ArrayList(); - ResultSet rsConnection = null; - PreparedStatement connectorLinkConfigFetchStatement = null; - PreparedStatement connectorLinkConfigInputStatement = null; - - try { - rsConnection = linkFetchStmt.executeQuery(); - connectorLinkConfigFetchStatement = conn.prepareStatement(STMT_SELECT_CONFIG_FOR_CONFIGURABLE); - connectorLinkConfigInputStatement = conn.prepareStatement(STMT_FETCH_LINK_INPUT); - - while(rsConnection.next()) { - long id = rsConnection.getLong(1); - String name = rsConnection.getString(2); - long connectorId = rsConnection.getLong(3); - boolean enabled = rsConnection.getBoolean(4); - String creationUser = rsConnection.getString(5); - Date creationDate = rsConnection.getTimestamp(6); - String updateUser = rsConnection.getString(7); - Date lastUpdateDate = rsConnection.getTimestamp(8); - - connectorLinkConfigFetchStatement.setLong(1, connectorId); - connectorLinkConfigInputStatement.setLong(1, id); - connectorLinkConfigInputStatement.setLong(3, id); - - List connectorLinkConfig = new ArrayList(); - List fromConfig = new ArrayList(); - List toConfig = new ArrayList(); - - loadConnectorConfigTypes(connectorLinkConfig, fromConfig, toConfig, connectorLinkConfigFetchStatement, - connectorLinkConfigInputStatement, 2, conn); - MLink link = new MLink(connectorId, new MLinkConfig(connectorLinkConfig)); - - link.setPersistenceId(id); - link.setName(name); - link.setCreationUser(creationUser); - link.setCreationDate(creationDate); - link.setLastUpdateUser(updateUser); - link.setLastUpdateDate(lastUpdateDate); - link.setEnabled(enabled); - - links.add(link); - } - } finally { - closeResultSets(rsConnection); - closeStatements(connectorLinkConfigFetchStatement, connectorLinkConfigInputStatement); - } - - return links; - } - - private List loadJobs(PreparedStatement stmt, - Connection conn) - throws SQLException { - List jobs = new ArrayList(); - ResultSet rsJob = null; - PreparedStatement fromConfigFetchStmt = null; - PreparedStatement toConfigFetchStmt = null; - PreparedStatement driverConfigfetchStmt = null; - PreparedStatement jobInputFetchStmt = null; - - try { - rsJob = stmt.executeQuery(); - // Note: Job does not hold a explicit reference to the driver since every - // job has the same driver - long driverId = this.findDriver(MDriver.DRIVER_NAME, conn).getPersistenceId(); - fromConfigFetchStmt = conn.prepareStatement(STMT_SELECT_CONFIG_FOR_CONFIGURABLE); - toConfigFetchStmt = conn.prepareStatement(STMT_SELECT_CONFIG_FOR_CONFIGURABLE); - driverConfigfetchStmt = conn.prepareStatement(STMT_SELECT_CONFIG_FOR_CONFIGURABLE); - jobInputFetchStmt = conn.prepareStatement(STMT_FETCH_JOB_INPUT); - - while(rsJob.next()) { - long fromConnectorId = rsJob.getLong(1); - long toConnectorId = rsJob.getLong(2); - long id = rsJob.getLong(3); - String name = rsJob.getString(4); - long fromLinkId = rsJob.getLong(5); - long toLinkId = rsJob.getLong(6); - boolean enabled = rsJob.getBoolean(7); - String createBy = rsJob.getString(8); - Date creationDate = rsJob.getTimestamp(9); - String updateBy = rsJob.getString(10); - Date lastUpdateDate = rsJob.getTimestamp(11); - - fromConfigFetchStmt.setLong(1, fromConnectorId); - toConfigFetchStmt.setLong(1,toConnectorId); - driverConfigfetchStmt.setLong(1, driverId); - - jobInputFetchStmt.setLong(1, id); - jobInputFetchStmt.setLong(3, id); - - // FROM entity configs - List fromConnectorLinkConfig = new ArrayList(); - List fromConnectorFromJobConfig = new ArrayList(); - List fromConnectorToJobConfig = new ArrayList(); - - loadConnectorConfigTypes(fromConnectorLinkConfig, fromConnectorFromJobConfig, fromConnectorToJobConfig, - fromConfigFetchStmt, jobInputFetchStmt, 2, conn); - - // TO entity configs - List toConnectorLinkConfig = new ArrayList(); - List toConnectorFromJobConfig = new ArrayList(); - List toConnectorToJobConfig = new ArrayList(); - - // ?? dont we need 2 different driver configs for the from/to? - List driverConfig = new ArrayList(); - - loadConnectorConfigTypes(toConnectorLinkConfig, toConnectorFromJobConfig, toConnectorToJobConfig, - toConfigFetchStmt, jobInputFetchStmt, 2, conn); - - loadDriverConfigs(driverConfig, driverConfigfetchStmt, jobInputFetchStmt, 2); - - MJob job = new MJob( - fromConnectorId, toConnectorId, - fromLinkId, toLinkId, - new MFromConfig(fromConnectorFromJobConfig), - new MToConfig(toConnectorToJobConfig), - new MDriverConfig(driverConfig)); - - job.setPersistenceId(id); - job.setName(name); - job.setCreationUser(createBy); - job.setCreationDate(creationDate); - job.setLastUpdateUser(updateBy); - job.setLastUpdateDate(lastUpdateDate); - job.setEnabled(enabled); - - jobs.add(job); - } - } finally { - closeResultSets(rsJob); - closeStatements(fromConfigFetchStmt, toConfigFetchStmt, driverConfigfetchStmt, jobInputFetchStmt); - } - - return jobs; - } - - private void registerConfigDirection(Long configId, Direction direction, Connection conn) - throws SQLException { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(STMT_INSERT_SQ_CONFIG_DIRECTIONS); - stmt.setLong(1, configId); - stmt.setLong(2, getDirection(direction, conn)); - if (stmt.executeUpdate() != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0048); - } - } finally { - closeStatements(stmt); - } - } - - /** - * Register configs in derby database. This method will insert the ids - * generated by the repository into the configs passed in itself. - * - * Use given prepared statements to create entire config structure in database. - * - * @param configurableId - * @param configs - * @param type - * @param baseConfigStmt - * @param baseInputStmt - * @param conn - * @return short number of configs registered. - * @throws SQLException - */ - private short registerConfigs(Long configurableId, Direction direction, - List configs, String type, PreparedStatement baseConfigStmt, - PreparedStatement baseInputStmt, Connection conn) - throws SQLException { - short configIndex = 0; - - for (MConfig config : configs) { - if (configurableId == null) { - baseConfigStmt.setNull(1, Types.BIGINT); - } else { - baseConfigStmt.setLong(1, configurableId); - } - - baseConfigStmt.setString(2, config.getName()); - baseConfigStmt.setString(3, type); - baseConfigStmt.setShort(4, configIndex++); - - int baseConfigCount = baseConfigStmt.executeUpdate(); - if (baseConfigCount != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0015, - Integer.toString(baseConfigCount)); - } - ResultSet rsetConfigId = baseConfigStmt.getGeneratedKeys(); - if (!rsetConfigId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0016); - } - - long configId = rsetConfigId.getLong(1); - config.setPersistenceId(configId); - - if (direction != null) { - registerConfigDirection(configId, direction, conn); - } - - // Insert all the inputs - List> inputs = config.getInputs(); - registerConfigInputs(configId, inputs, baseInputStmt); - } - return configIndex; - } - - /** - * Save given inputs to the database. - * - * Use given prepare statement to save all inputs into repository. - * - * @param configId Identifier for corresponding config - * @param inputs List of inputs that needs to be saved - * @param baseInputStmt Statement that we can utilize - * @throws SQLException In case of any failure on Derby side - */ - private void registerConfigInputs(long configId, List> inputs, - PreparedStatement baseInputStmt) throws SQLException { - short inputIndex = 0; - for (MInput input : inputs) { - baseInputStmt.setString(1, input.getName()); - baseInputStmt.setLong(2, configId); - baseInputStmt.setShort(3, inputIndex++); - baseInputStmt.setString(4, input.getType().name()); - baseInputStmt.setBoolean(5, input.isSensitive()); - // String specific column(s) - if (input.getType().equals(MInputType.STRING)) { - MStringInput strInput = (MStringInput) input; - baseInputStmt.setShort(6, strInput.getMaxLength()); - } else { - baseInputStmt.setNull(6, Types.INTEGER); - } - // Enum specific column(s) - if(input.getType() == MInputType.ENUM) { - baseInputStmt.setString(7, StringUtils.join(((MEnumInput)input).getValues(), ",")); - } else { - baseInputStmt.setNull(7, Types.VARCHAR); - } - - int baseInputCount = baseInputStmt.executeUpdate(); - if (baseInputCount != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0017, - Integer.toString(baseInputCount)); - } - - ResultSet rsetInputId = baseInputStmt.getGeneratedKeys(); - if (!rsetInputId.next()) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0018); - } - - long inputId = rsetInputId.getLong(1); - input.setPersistenceId(inputId); - } - } - - /** - * Execute given query on database. - * - * @param query Query that should be executed - */ - private void runQuery(String query, Connection conn, Object... args) { - PreparedStatement stmt = null; - try { - stmt = conn.prepareStatement(query); - - for (int i = 0; i < args.length; ++i) { - if (args[i] instanceof String) { - stmt.setString(i + 1, (String)args[i]); - } else if (args[i] instanceof Long) { - stmt.setLong(i + 1, (Long) args[i]); - } else { - stmt.setObject(i + 1, args[i]); - } - } - - if (stmt.execute()) { - ResultSet rset = stmt.getResultSet(); - int count = 0; - while (rset.next()) { - count++; - } - LOG.info("QUERY(" + query + ") produced unused resultset with "+ count + " rows"); - } else { - int updateCount = stmt.getUpdateCount(); - LOG.info("QUERY(" + query + ") Update count: " + updateCount); - } - } catch (SQLException ex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0003, query, ex); - } finally { - closeStatements(stmt); - } - } - - /** - * Load configs and corresponding inputs from Derby database. - * - * Use given prepared statements to load all configs and corresponding inputs - * from Derby. - * - * @param driverConfig List of driver configs that will be filled up - * @param configFetchStatement Prepared statement for fetching configs - * @param inputFetchStmt Prepare statement for fetching inputs - * @param configPosition position of the config - * @throws SQLException In case of any failure on Derby side - */ - public void loadDriverConfigs(List driverConfig, - PreparedStatement configFetchStatement, - PreparedStatement inputFetchStmt, - int configPosition) throws SQLException { - - // Get list of structures from database - ResultSet rsetConfig = configFetchStatement.executeQuery(); - while (rsetConfig.next()) { - long configId = rsetConfig.getLong(1); - Long fromConnectorId = rsetConfig.getLong(2); - String configName = rsetConfig.getString(3); - String configTYpe = rsetConfig.getString(4); - int configIndex = rsetConfig.getInt(5); - List> configInputs = new ArrayList>(); - - MConfig mDriverConfig = new MConfig(configName, configInputs); - mDriverConfig.setPersistenceId(configId); - - inputFetchStmt.setLong(configPosition, configId); - - ResultSet rsetInput = inputFetchStmt.executeQuery(); - while (rsetInput.next()) { - long inputId = rsetInput.getLong(1); - String inputName = rsetInput.getString(2); - long inputConfig = rsetInput.getLong(3); - short inputIndex = rsetInput.getShort(4); - String inputType = rsetInput.getString(5); - boolean inputSensitivity = rsetInput.getBoolean(6); - short inputStrLength = rsetInput.getShort(7); - String inputEnumValues = rsetInput.getString(8); - String value = rsetInput.getString(9); - - MInputType mit = MInputType.valueOf(inputType); - - MInput input = null; - switch (mit) { - case STRING: - input = new MStringInput(inputName, inputSensitivity, inputStrLength); - break; - case MAP: - input = new MMapInput(inputName, inputSensitivity); - break; - case BOOLEAN: - input = new MBooleanInput(inputName, inputSensitivity); - break; - case INTEGER: - input = new MIntegerInput(inputName, inputSensitivity); - break; - case ENUM: - input = new MEnumInput(inputName, inputSensitivity, inputEnumValues.split(",")); - break; - default: - throw new SqoopException(DerbyRepoError.DERBYREPO_0006, - "input-" + inputName + ":" + inputId + ":" - + "config-" + inputConfig + ":" + mit.name()); - } - - // Set persistent ID - input.setPersistenceId(inputId); - - // Set value - if(value == null) { - input.setEmpty(); - } else { - input.restoreFromUrlSafeValueString(value); - } - - if (mDriverConfig.getInputs().size() != inputIndex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0009, - "config: " + mDriverConfig - + "; input: " + input - + "; index: " + inputIndex - + "; expected: " + mDriverConfig.getInputs().size() - ); - } - - mDriverConfig.getInputs().add(input); - } - - if (mDriverConfig.getInputs().size() == 0) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0008, - "owner-" + fromConnectorId - + "; config: " + mDriverConfig - ); - } - - MConfigType configType = MConfigType.valueOf(configTYpe); - switch (configType) { - case JOB: - if (driverConfig.size() != configIndex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0010, - "owner-" + fromConnectorId - + "; config: " + configType - + "; index: " + configIndex - + "; expected: " + driverConfig.size() - ); - } - driverConfig.add(mDriverConfig); - break; - default: - throw new SqoopException(DerbyRepoError.DERBYREPO_0007, - "connector-" + fromConnectorId + ":" + configType); - } - } - } - - private Direction findConfigDirection(long configId, Connection conn) throws SQLException { - PreparedStatement stmt = null; - ResultSet rs = null; - - try { - stmt = conn.prepareStatement(STMT_SELECT_SQ_CONFIG_DIRECTIONS); - stmt.setLong(1, configId); - rs = stmt.executeQuery(); - rs.next(); - return getDirection(rs.getLong(2), conn); - } finally { - if (rs != null) { - closeResultSets(rs); - } - if (stmt != null) { - closeStatements(stmt); - } - } - } - - /** - * Load configs and corresponding inputs related to a connector - * - * Use given prepared statements to load all configs and corresponding inputs - * from Derby. - * - * @param linkConfig List of link configs that will be filled up - * @param fromConfig FROM job configs that will be filled up - * @param toConfig TO job configs that will be filled up - * @param configFetchStmt Prepared statement for fetching configs - * @param inputFetchStmt Prepare statement for fetching inputs - * @param conn Connection object that is used to find config direction. - * @throws SQLException In case of any failure on Derby side - */ - public void loadConnectorConfigTypes(List linkConfig, List fromConfig, - List toConfig, PreparedStatement configFetchStmt, PreparedStatement inputFetchStmt, - int configPosition, Connection conn) throws SQLException { - - // Get list of structures from database - ResultSet rsetConfig = configFetchStmt.executeQuery(); - while (rsetConfig.next()) { - long configId = rsetConfig.getLong(1); - Long configConnectorId = rsetConfig.getLong(2); - String configName = rsetConfig.getString(3); - String configType = rsetConfig.getString(4); - int configIndex = rsetConfig.getInt(5); - List> configInputs = new ArrayList>(); - - MConfig config = new MConfig(configName, configInputs); - config.setPersistenceId(configId); - - inputFetchStmt.setLong(configPosition, configId); - - ResultSet rsetInput = inputFetchStmt.executeQuery(); - while (rsetInput.next()) { - long inputId = rsetInput.getLong(1); - String inputName = rsetInput.getString(2); - long inputConfig = rsetInput.getLong(3); - short inputIndex = rsetInput.getShort(4); - String inputType = rsetInput.getString(5); - boolean inputSensitivity = rsetInput.getBoolean(6); - short inputStrLength = rsetInput.getShort(7); - String inputEnumValues = rsetInput.getString(8); - String value = rsetInput.getString(9); - - MInputType mit = MInputType.valueOf(inputType); - - MInput input = null; - switch (mit) { - case STRING: - input = new MStringInput(inputName, inputSensitivity, inputStrLength); - break; - case MAP: - input = new MMapInput(inputName, inputSensitivity); - break; - case BOOLEAN: - input = new MBooleanInput(inputName, inputSensitivity); - break; - case INTEGER: - input = new MIntegerInput(inputName, inputSensitivity); - break; - case ENUM: - input = new MEnumInput(inputName, inputSensitivity, inputEnumValues.split(",")); - break; - default: - throw new SqoopException(DerbyRepoError.DERBYREPO_0006, - "input-" + inputName + ":" + inputId + ":" - + "config-" + inputConfig + ":" + mit.name()); - } - - // Set persistent ID - input.setPersistenceId(inputId); - - // Set value - if(value == null) { - input.setEmpty(); - } else { - input.restoreFromUrlSafeValueString(value); - } - - if (config.getInputs().size() != inputIndex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0009, - "config: " + config - + "; input: " + input - + "; index: " + inputIndex - + "; expected: " + config.getInputs().size() - ); - } - - config.getInputs().add(input); - } - - if (config.getInputs().size() == 0) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0008, - "connector-" + configConnectorId - + "; config: " + config - ); - } - - MConfigType mConfigType = MConfigType.valueOf(configType); - switch (mConfigType) { - case LINK: - if (linkConfig.size() != configIndex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0010, - "connector-" + configConnectorId - + "; config: " + config - + "; index: " + configIndex - + "; expected: " + linkConfig.size() - ); - } - linkConfig.add(config); - break; - case JOB: - Direction type = findConfigDirection(configId, conn); - List jobConfigs; - switch(type) { - case FROM: - jobConfigs = fromConfig; - break; - - case TO: - jobConfigs = toConfig; - break; - - default: - throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type); - } - - if (jobConfigs.size() != configIndex) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0010, - "connector-" + configConnectorId - + "; config: " + config - + "; index: " + configIndex - + "; expected: " + jobConfigs.size() - + "; configIndex: " + configIndex - - ); - } - - jobConfigs.add(config); - break; - default: - throw new SqoopException(DerbyRepoError.DERBYREPO_0007, - "connector-" + configConnectorId + ":" + config); - } - } - } - - private void createInputValues(String query, - long id, - List configs, - Connection conn) throws SQLException { - PreparedStatement stmt = null; - int result; - - try { - stmt = conn.prepareStatement(query); - - for (MConfig config : configs) { - for (MInput input : config.getInputs()) { - // Skip empty values as we're not interested in storing those in db - if (input.isEmpty()) { - continue; - } - stmt.setLong(1, id); - stmt.setLong(2, input.getPersistenceId()); - stmt.setString(3, input.getUrlSafeValueString()); - - result = stmt.executeUpdate(); - if (result != 1) { - throw new SqoopException(DerbyRepoError.DERBYREPO_0020, - Integer.toString(result)); - } - } - } - } finally { - closeStatements(stmt); - } - } - - /** - * Close all given Results set. - * - * Any occurring exception is silently ignored and logged. - * - * @param resultSets Result sets to close - */ - private void closeResultSets(ResultSet ... resultSets) { - if(resultSets == null) { - return; - } - for (ResultSet rs : resultSets) { - if(rs != null) { - try { - rs.close(); - } catch(SQLException ex) { - LOG.error("Exception during closing result set", ex); - } - } - } - } - - /** - * Close all given statements. - * - * Any occurring exception is silently ignored and logged. - * - * @param stmts Statements to close - */ - private void closeStatements(Statement... stmts) { - if(stmts == null) { - return; - } - for (Statement stmt : stmts) { - if(stmt != null) { - try { - stmt.close(); - } catch (SQLException ex) { - LOG.error("Exception during closing statement", ex); - } - } - } - } - - /** - * Log exception and all String variant of arbitrary number of objects. - * - * This method is useful to log SQLException with all objects that were - * used in the query generation to see where is the issue. - * - * @param throwable Arbitrary throwable object - * @param objects Arbitrary array of associated objects - */ - private void logException(Throwable throwable, Object ...objects) { - LOG.error("Exception in repository operation", throwable); - LOG.error("Associated objects: "+ objects.length); - for(Object object : objects) { - LOG.error("\t" + object.getClass().getSimpleName() + ": " + object.toString()); - } + public boolean isRespositorySuitableForUse(Connection conn) { + // TODO(jarcec): Verify that all structures are present (e.g. something like corruption validation) + // NOTE: At this point is is just checking if the repo version matches the version + // in the upgraded code + return detectRepositoryVersion(conn) == DerbyRepoConstants.LATEST_DERBY_REPOSITORY_VERSION; } } \ No newline at end of file diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaInsertUpdateDeleteSelectQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaInsertUpdateDeleteSelectQuery.java index 6c5fad77..4cbaf5ad 100644 --- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaInsertUpdateDeleteSelectQuery.java +++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaInsertUpdateDeleteSelectQuery.java @@ -25,7 +25,7 @@ */ public final class DerbySchemaInsertUpdateDeleteSelectQuery { -/******** SYSTEM TABLE**************/ + /******** SYSTEM TABLE**************/ // DML: Get system key public static final String STMT_SELECT_SYSTEM = "SELECT " @@ -33,8 +33,8 @@ public final class DerbySchemaInsertUpdateDeleteSelectQuery { + " FROM " + TABLE_SQ_SYSTEM + " WHERE " + COLUMN_SQM_KEY + " = ?"; - //DML: Get deprecated or the new repo version system key - public static final String STMT_SELECT_DEPRECATED_OR_NEW_SYSTEM_VERSION = + //DML: Get deprecated or the new repo version system key + public static final String STMT_SELECT_DEPRECATED_OR_NEW_SYSTEM_VERSION = "SELECT " + COLUMN_SQM_VALUE + " FROM " + TABLE_SQ_SYSTEM + " WHERE ( " + COLUMN_SQM_KEY + " = ? )" @@ -52,40 +52,10 @@ public final class DerbySchemaInsertUpdateDeleteSelectQuery { + COLUMN_SQM_VALUE + ") " + "VALUES(?, ?)"; - /*******DIRECTION TABLE **************/ - - public static final String STMT_SELECT_SQD_ID_BY_SQD_NAME = - "SELECT " + COLUMN_SQD_ID + " FROM " + TABLE_SQ_DIRECTION - + " WHERE " + COLUMN_SQD_NAME + "=?"; - - public static final String STMT_SELECT_SQD_NAME_BY_SQD_ID = - "SELECT " + COLUMN_SQD_NAME + " FROM " + TABLE_SQ_DIRECTION - + " WHERE " + COLUMN_SQD_ID + "=?"; - -/*********CONFIGURABLE TABLE ***************/ - //DML: Get configurable by given name - public static final String STMT_SELECT_FROM_CONFIGURABLE = - "SELECT " - + COLUMN_SQC_ID + ", " - + COLUMN_SQC_NAME + ", " - + COLUMN_SQC_CLASS + ", " - + COLUMN_SQC_VERSION - + " FROM " + TABLE_SQ_CONFIGURABLE - + " WHERE " + COLUMN_SQC_NAME + " = ?"; - - //DML: Get all configurables for a given type - public static final String STMT_SELECT_CONFIGURABLE_ALL_FOR_TYPE = - "SELECT " - + COLUMN_SQC_ID + ", " - + COLUMN_SQC_NAME + ", " - + COLUMN_SQC_CLASS + ", " - + COLUMN_SQC_VERSION - + " FROM " + TABLE_SQ_CONFIGURABLE - + " WHERE " + COLUMN_SQC_TYPE + " = ?"; - - // DML: Select all connectors - @Deprecated // used only for upgrade logic - public static final String STMT_SELECT_CONNECTOR_ALL = + /*********CONFIGURABLE TABLE ***************/ + // DML: Select all connectors + @Deprecated // used only for upgrade logic + public static final String STMT_SELECT_CONNECTOR_ALL = "SELECT " + COLUMN_SQC_ID + ", " + COLUMN_SQC_NAME + ", " @@ -93,50 +63,17 @@ public final class DerbySchemaInsertUpdateDeleteSelectQuery { + COLUMN_SQC_VERSION + " FROM " + TABLE_SQ_CONNECTOR; - //DML: Insert into configurable - public static final String STMT_INSERT_INTO_CONFIGURABLE = - "INSERT INTO " + TABLE_SQ_CONFIGURABLE + " (" - + COLUMN_SQC_NAME + ", " - + COLUMN_SQC_CLASS + ", " - + COLUMN_SQC_VERSION + ", " - + COLUMN_SQC_TYPE - + ") VALUES (?, ?, ?, ?)"; - - @Deprecated // used only in the upgrade path - public static final String STMT_INSERT_INTO_CONFIGURABLE_WITHOUT_SUPPORTED_DIRECTIONS = + @Deprecated // used only in the upgrade path + public static final String STMT_INSERT_INTO_CONFIGURABLE_WITHOUT_SUPPORTED_DIRECTIONS = "INSERT INTO " + TABLE_SQ_CONNECTOR+ " (" + COLUMN_SQC_NAME + ", " + COLUMN_SQC_CLASS + ", " + COLUMN_SQC_VERSION + ") VALUES (?, ?, ?)"; - //Delete all configs for a given configurable - public static final String STMT_DELETE_CONFIGS_FOR_CONFIGURABLE = - "DELETE FROM " + TABLE_SQ_CONFIG - + " WHERE " + COLUMN_SQ_CFG_CONFIGURABLE + " = ?"; - //Delete all inputs for a given configurable - public static final String STMT_DELETE_INPUTS_FOR_CONFIGURABLE = - "DELETE FROM " + TABLE_SQ_INPUT - + " WHERE " - + COLUMN_SQI_CONFIG - + " IN (SELECT " - + COLUMN_SQ_CFG_ID - + " FROM " + TABLE_SQ_CONFIG - + " WHERE " - + COLUMN_SQ_CFG_CONFIGURABLE + " = ?)"; - - //Update the configurable - public static final String STMT_UPDATE_CONFIGURABLE = - "UPDATE " + TABLE_SQ_CONFIGURABLE - + " SET " + COLUMN_SQC_NAME + " = ?, " - + COLUMN_SQC_CLASS + " = ?, " - + COLUMN_SQC_VERSION + " = ?, " - + COLUMN_SQC_TYPE + " = ? " - + " WHERE " + COLUMN_SQC_ID + " = ?"; - - //DML: Insert new connection - @Deprecated // used only in upgrade path - public static final String STMT_INSERT_CONNECTION = + //DML: Insert new connection + @Deprecated // used only in upgrade path + public static final String STMT_INSERT_CONNECTION = "INSERT INTO " + TABLE_SQ_CONNECTION + " (" + COLUMN_SQN_NAME + ", " + COLUMN_SQN_CONNECTOR + "," @@ -146,271 +83,7 @@ public final class DerbySchemaInsertUpdateDeleteSelectQuery { + COLUMN_SQN_UPDATE_USER + ", " + COLUMN_SQN_UPDATE_DATE + ") VALUES (?, ?, ?, ?, ?, ?, ?)"; - - /**********CONFIG TABLE **************/ - //DML: Get all configs for a given configurable - public static final String STMT_SELECT_CONFIG_FOR_CONFIGURABLE = - "SELECT " - + COLUMN_SQ_CFG_ID + ", " - + COLUMN_SQ_CFG_CONFIGURABLE + ", " - + COLUMN_SQ_CFG_NAME + ", " - + COLUMN_SQ_CFG_TYPE + ", " - + COLUMN_SQ_CFG_INDEX - + " FROM " + TABLE_SQ_CONFIG - + " WHERE " + COLUMN_SQ_CFG_CONFIGURABLE + " = ? " - + " ORDER BY " + COLUMN_SQ_CFG_INDEX; - - - //DML: Insert into config - public static final String STMT_INSERT_INTO_CONFIG = - "INSERT INTO " + TABLE_SQ_CONFIG + " (" - + COLUMN_SQ_CFG_CONFIGURABLE + ", " - + COLUMN_SQ_CFG_NAME + ", " - + COLUMN_SQ_CFG_TYPE + ", " - + COLUMN_SQ_CFG_INDEX - + ") VALUES ( ?, ?, ?, ?)"; - - /********** INPUT TABLE **************/ - - // DML: Get inputs for a given config - public static final String STMT_SELECT_INPUT = - "SELECT " - + COLUMN_SQI_ID + ", " - + COLUMN_SQI_NAME + ", " - + COLUMN_SQI_CONFIG + ", " - + COLUMN_SQI_INDEX + ", " - + COLUMN_SQI_TYPE + ", " - + COLUMN_SQI_STRMASK + ", " - + COLUMN_SQI_STRLENGTH + ", " - + COLUMN_SQI_ENUMVALS + ", " - + "cast(null as varchar(100))" - + " FROM " + TABLE_SQ_INPUT - + " WHERE " + COLUMN_SQI_CONFIG + " = ?" - + " ORDER BY " + COLUMN_SQI_INDEX; - - // DML: Insert into config input - public static final String STMT_INSERT_INTO_INPUT = - "INSERT INTO " + TABLE_SQ_INPUT + " (" - + COLUMN_SQI_NAME + ", " - + COLUMN_SQI_CONFIG + ", " - + COLUMN_SQI_INDEX + ", " - + COLUMN_SQI_TYPE + ", " - + COLUMN_SQI_STRMASK + ", " - + COLUMN_SQI_STRLENGTH + ", " - + COLUMN_SQI_ENUMVALS - + ") VALUES (?, ?, ?, ?, ?, ?, ?)"; - - /**********LINK INPUT TABLE **************/ - - //DML: Get inputs and values for a given link - public static final String STMT_FETCH_LINK_INPUT = - "SELECT " - + COLUMN_SQI_ID + ", " - + COLUMN_SQI_NAME + ", " - + COLUMN_SQI_CONFIG + ", " - + COLUMN_SQI_INDEX + ", " - + COLUMN_SQI_TYPE + ", " - + COLUMN_SQI_STRMASK + ", " - + COLUMN_SQI_STRLENGTH + "," - + COLUMN_SQI_ENUMVALS + ", " - + COLUMN_SQ_LNKI_VALUE - + " FROM " + TABLE_SQ_INPUT - + " LEFT OUTER JOIN " + TABLE_SQ_LINK_INPUT - + " ON " + COLUMN_SQ_LNKI_INPUT + " = " + COLUMN_SQI_ID - + " AND " + COLUMN_SQ_LNKI_LINK + " = ?" - + " WHERE " + COLUMN_SQI_CONFIG + " = ?" - + " AND (" + COLUMN_SQ_LNKI_LINK + " = ?" + " OR " + COLUMN_SQ_LNKI_LINK + " IS NULL)" - + " ORDER BY " + COLUMN_SQI_INDEX; - - /**********JOB INPUT TABLE **************/ - - //DML: Fetch inputs and values for a given job - public static final String STMT_FETCH_JOB_INPUT = - "SELECT " - + COLUMN_SQI_ID + ", " - + COLUMN_SQI_NAME + ", " - + COLUMN_SQI_CONFIG + ", " - + COLUMN_SQI_INDEX + ", " - + COLUMN_SQI_TYPE + ", " - + COLUMN_SQI_STRMASK + ", " - + COLUMN_SQI_STRLENGTH + ", " - + COLUMN_SQI_ENUMVALS + ", " - + COLUMN_SQBI_VALUE - + " FROM " + TABLE_SQ_INPUT - + " LEFT OUTER JOIN " + TABLE_SQ_JOB_INPUT - + " ON " + COLUMN_SQBI_INPUT + " = " + COLUMN_SQI_ID - + " AND " + COLUMN_SQBI_JOB + " = ?" - + " WHERE " + COLUMN_SQI_CONFIG + " = ?" - + " AND (" + COLUMN_SQBI_JOB + " = ? OR " + COLUMN_SQBI_JOB + " IS NULL)" - + " ORDER BY " + COLUMN_SQI_INDEX; - - /**********LINK TABLE **************/ - - // DML: Insert new link - public static final String STMT_INSERT_LINK = - "INSERT INTO " + TABLE_SQ_LINK + " (" - + COLUMN_SQ_LNK_NAME + ", " - + COLUMN_SQ_LNK_CONFIGURABLE + ", " - + COLUMN_SQ_LNK_ENABLED + ", " - + COLUMN_SQ_LNK_CREATION_USER + ", " - + COLUMN_SQ_LNK_CREATION_DATE + ", " - + COLUMN_SQ_LNK_UPDATE_USER + ", " - + COLUMN_SQ_LNK_UPDATE_DATE - + ") VALUES (?, ?, ?, ?, ?, ?, ?)"; - - // DML: Insert new link inputs - public static final String STMT_INSERT_LINK_INPUT = - "INSERT INTO " + TABLE_SQ_LINK_INPUT + " (" - + COLUMN_SQ_LNKI_LINK + ", " - + COLUMN_SQ_LNKI_INPUT + ", " - + COLUMN_SQ_LNKI_VALUE - + ") VALUES (?, ?, ?)"; - - // DML: Update link - public static final String STMT_UPDATE_LINK = - "UPDATE " + TABLE_SQ_LINK + " SET " - + COLUMN_SQ_LNK_NAME + " = ?, " - + COLUMN_SQ_LNK_UPDATE_USER + " = ?, " - + COLUMN_SQ_LNK_UPDATE_DATE + " = ? " - + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; - - // DML: Enable or disable link - public static final String STMT_ENABLE_LINK = - "UPDATE " + TABLE_SQ_LINK + " SET " - + COLUMN_SQ_LNK_ENABLED + " = ? " - + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; - - // DML: Delete rows from link input table - public static final String STMT_DELETE_LINK_INPUT = - "DELETE FROM " + TABLE_SQ_LINK_INPUT - + " WHERE " + COLUMN_SQ_LNKI_LINK + " = ?"; - - // DML: Delete row from link table - public static final String STMT_DELETE_LINK = - "DELETE FROM " + TABLE_SQ_LINK - + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; - - // DML: Select one specific link - public static final String STMT_SELECT_LINK_SINGLE = - "SELECT " - + COLUMN_SQ_LNK_ID + ", " - + COLUMN_SQ_LNK_NAME + ", " - + COLUMN_SQ_LNK_CONFIGURABLE + ", " - + COLUMN_SQ_LNK_ENABLED + ", " - + COLUMN_SQ_LNK_CREATION_USER + ", " - + COLUMN_SQ_LNK_CREATION_DATE + ", " - + COLUMN_SQ_LNK_UPDATE_USER + ", " - + COLUMN_SQ_LNK_UPDATE_DATE - + " FROM " + TABLE_SQ_LINK - + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; - - - // DML: Select one specific link by name - public static final String STMT_SELECT_LINK_SINGLE_BY_NAME = - "SELECT " - + COLUMN_SQ_LNK_ID + ", " - + COLUMN_SQ_LNK_NAME + ", " - + COLUMN_SQ_LNK_CONFIGURABLE + ", " - + COLUMN_SQ_LNK_ENABLED + ", " - + COLUMN_SQ_LNK_CREATION_USER + ", " - + COLUMN_SQ_LNK_CREATION_DATE + ", " - + COLUMN_SQ_LNK_UPDATE_USER + ", " - + COLUMN_SQ_LNK_UPDATE_DATE - + " FROM " + TABLE_SQ_LINK - + " WHERE " + COLUMN_SQ_LNK_NAME + " = ?"; - - // DML: Select all links - public static final String STMT_SELECT_LINK_ALL = - "SELECT " - + COLUMN_SQ_LNK_ID + ", " - + COLUMN_SQ_LNK_NAME + ", " - + COLUMN_SQ_LNK_CONFIGURABLE + ", " - + COLUMN_SQ_LNK_ENABLED + ", " - + COLUMN_SQ_LNK_CREATION_USER + ", " - + COLUMN_SQ_LNK_CREATION_DATE + ", " - + COLUMN_SQ_LNK_UPDATE_USER + ", " - + COLUMN_SQ_LNK_UPDATE_DATE - + " FROM " + TABLE_SQ_LINK; - - // DML: Select all links for a specific connector. - public static final String STMT_SELECT_LINK_FOR_CONNECTOR_CONFIGURABLE = - "SELECT " - + COLUMN_SQ_LNK_ID + ", " - + COLUMN_SQ_LNK_NAME + ", " - + COLUMN_SQ_LNK_CONFIGURABLE + ", " - + COLUMN_SQ_LNK_ENABLED + ", " - + COLUMN_SQ_LNK_CREATION_USER + ", " - + COLUMN_SQ_LNK_CREATION_DATE + ", " - + COLUMN_SQ_LNK_UPDATE_USER + ", " - + COLUMN_SQ_LNK_UPDATE_DATE - + " FROM " + TABLE_SQ_LINK - + " WHERE " + COLUMN_SQ_LNK_CONFIGURABLE + " = ?"; - - // DML: Check if given link exists - public static final String STMT_SELECT_LINK_CHECK_BY_ID = - "SELECT count(*) FROM " + TABLE_SQ_LINK - + " WHERE " + COLUMN_SQ_LNK_ID + " = ?"; - - /**********JOB TABLE **************/ - - // DML: Insert new job - public static final String STMT_INSERT_JOB = - "INSERT INTO " + TABLE_SQ_JOB + " (" - + COLUMN_SQB_NAME + ", " - + COLUMN_SQB_FROM_LINK + ", " - + COLUMN_SQB_TO_LINK + ", " - + COLUMN_SQB_ENABLED + ", " - + COLUMN_SQB_CREATION_USER + ", " - + COLUMN_SQB_CREATION_DATE + ", " - + COLUMN_SQB_UPDATE_USER + ", " - + COLUMN_SQB_UPDATE_DATE - + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; - - // DML: Insert new job inputs - public static final String STMT_INSERT_JOB_INPUT = - "INSERT INTO " + TABLE_SQ_JOB_INPUT + " (" - + COLUMN_SQBI_JOB + ", " - + COLUMN_SQBI_INPUT + ", " - + COLUMN_SQBI_VALUE - + ") VALUES (?, ?, ?)"; - - public static final String STMT_UPDATE_JOB = - "UPDATE " + TABLE_SQ_JOB + " SET " - + COLUMN_SQB_NAME + " = ?, " - + COLUMN_SQB_UPDATE_USER + " = ?, " - + COLUMN_SQB_UPDATE_DATE + " = ? " - + " WHERE " + COLUMN_SQB_ID + " = ?"; - - // DML: Enable or disable job - public static final String STMT_ENABLE_JOB = - "UPDATE " + TABLE_SQ_JOB + " SET " - + COLUMN_SQB_ENABLED + " = ? " - + " WHERE " + COLUMN_SQB_ID + " = ?"; - - // DML: Delete rows from job input table - public static final String STMT_DELETE_JOB_INPUT = - "DELETE FROM " + TABLE_SQ_JOB_INPUT - + " WHERE " + COLUMN_SQBI_JOB + " = ?"; - - // DML: Delete row from job table - public static final String STMT_DELETE_JOB = - "DELETE FROM " + TABLE_SQ_JOB - + " WHERE " + COLUMN_SQB_ID + " = ?"; - - // DML: Check if given job exists - public static final String STMT_SELECT_JOB_CHECK_BY_ID = - "SELECT count(*) FROM " + TABLE_SQ_JOB - + " WHERE " + COLUMN_SQB_ID + " = ?"; - - // DML: Check if there are jobs for given link - public static final String STMT_SELECT_JOBS_FOR_LINK_CHECK = - "SELECT" - + " count(*)" - + " FROM " + TABLE_SQ_JOB - + " JOIN " + TABLE_SQ_LINK - + " ON " + COLUMN_SQB_FROM_LINK + " = " + COLUMN_SQ_LNK_ID - + " WHERE " + COLUMN_SQ_LNK_ID + " = ? "; - + /**********JOB TABLE **************/ //DML: Select all jobs public static final String STMT_SELECT_JOB = "SELECT " @@ -431,205 +104,27 @@ public final class DerbySchemaInsertUpdateDeleteSelectQuery { + " LEFT JOIN " + TABLE_SQ_LINK + " TO_CONNECTOR" + " ON " + COLUMN_SQB_TO_LINK + " = TO_CONNECTOR." + COLUMN_SQ_LNK_ID; - // DML: Select one specific job - public static final String STMT_SELECT_JOB_SINGLE_BY_ID = - STMT_SELECT_JOB + " WHERE " + COLUMN_SQB_ID + " = ?"; -// DML: Select one specific job - public static final String STMT_SELECT_JOB_SINGLE_BY_NAME = - STMT_SELECT_JOB + " WHERE " + COLUMN_SQB_NAME + " = ?"; - - // DML: Select all jobs for a Connector - public static final String STMT_SELECT_ALL_JOBS_FOR_CONNECTOR_CONFIGURABLE = - STMT_SELECT_JOB - + " WHERE FROM_LINK." + COLUMN_SQ_LNK_CONFIGURABLE + " = ? OR TO_LINK." - + COLUMN_SQ_LNK_CONFIGURABLE + " = ?"; - - /**********SUBMISSION TABLE **************/ - - // DML: Insert new submission - public static final String STMT_INSERT_SUBMISSION = - "INSERT INTO " + TABLE_SQ_SUBMISSION + "(" - + COLUMN_SQS_JOB + ", " - + COLUMN_SQS_STATUS + ", " - + COLUMN_SQS_CREATION_USER + ", " - + COLUMN_SQS_CREATION_DATE + ", " - + COLUMN_SQS_UPDATE_USER + ", " - + COLUMN_SQS_UPDATE_DATE + ", " - + COLUMN_SQS_EXTERNAL_ID + ", " - + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE + ") " - + " VALUES(?, ?, ?, ?, ?, ?, ?, substr(?, 1, 150) , substr(?, 1, 150), substr(?, 1, 750))"; - - // DML: Update existing submission - public static final String STMT_UPDATE_SUBMISSION = - "UPDATE " + TABLE_SQ_SUBMISSION + " SET " - + COLUMN_SQS_STATUS + " = ?, " - + COLUMN_SQS_UPDATE_USER + " = ?, " - + COLUMN_SQS_UPDATE_DATE + " = ?, " - + COLUMN_SQS_EXCEPTION + " = ?, " - + COLUMN_SQS_EXCEPTION_TRACE + " = ?" - + " WHERE " + COLUMN_SQS_ID + " = ?"; - - // DML: Check if given submission exists - public static final String STMT_SELECT_SUBMISSION_CHECK = - "SELECT" - + " count(*)" - + " FROM " + TABLE_SQ_SUBMISSION - + " WHERE " + COLUMN_SQS_ID + " = ?"; - - // DML: Purge old entries - public static final String STMT_PURGE_SUBMISSIONS = - "DELETE FROM " + TABLE_SQ_SUBMISSION - + " WHERE " + COLUMN_SQS_UPDATE_DATE + " < ?"; - - // DML: Get unfinished - public static final String STMT_SELECT_SUBMISSION_UNFINISHED = - "SELECT " - + COLUMN_SQS_ID + ", " - + COLUMN_SQS_JOB + ", " - + COLUMN_SQS_STATUS + ", " - + COLUMN_SQS_CREATION_USER + ", " - + COLUMN_SQS_CREATION_DATE + ", " - + COLUMN_SQS_UPDATE_USER + ", " - + COLUMN_SQS_UPDATE_DATE + ", " - + COLUMN_SQS_EXTERNAL_ID + ", " - + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE - + " FROM " + TABLE_SQ_SUBMISSION - + " WHERE " + COLUMN_SQS_STATUS + " = ?"; - - // DML : Get all submissions - public static final String STMT_SELECT_SUBMISSIONS = - "SELECT " - + COLUMN_SQS_ID + ", " - + COLUMN_SQS_JOB + ", " - + COLUMN_SQS_STATUS + ", " - + COLUMN_SQS_CREATION_USER + ", " - + COLUMN_SQS_CREATION_DATE + ", " - + COLUMN_SQS_UPDATE_USER + ", " - + COLUMN_SQS_UPDATE_DATE + ", " - + COLUMN_SQS_EXTERNAL_ID + ", " - + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE - + " FROM " + TABLE_SQ_SUBMISSION - + " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC"; - - // DML: Get submissions for a job - public static final String STMT_SELECT_SUBMISSIONS_FOR_JOB = - "SELECT " - + COLUMN_SQS_ID + ", " - + COLUMN_SQS_JOB + ", " - + COLUMN_SQS_STATUS + ", " - + COLUMN_SQS_CREATION_USER + ", " - + COLUMN_SQS_CREATION_DATE + ", " - + COLUMN_SQS_UPDATE_USER + ", " - + COLUMN_SQS_UPDATE_DATE + ", " - + COLUMN_SQS_EXTERNAL_ID + ", " - + COLUMN_SQS_EXTERNAL_LINK + ", " - + COLUMN_SQS_EXCEPTION + ", " - + COLUMN_SQS_EXCEPTION_TRACE - + " FROM " + TABLE_SQ_SUBMISSION - + " WHERE " + COLUMN_SQS_JOB + " = ?" - + " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC"; - - // DML: Select counter group - public static final String STMT_SELECT_COUNTER_GROUP = - "SELECT " - + COLUMN_SQG_ID + ", " - + COLUMN_SQG_NAME + " " - + "FROM " + TABLE_SQ_COUNTER_GROUP + " " - + "WHERE " + COLUMN_SQG_NAME + " = substr(?, 1, 75)"; - - // DML: Insert new counter group - public static final String STMT_INSERT_COUNTER_GROUP = - "INSERT INTO " + TABLE_SQ_COUNTER_GROUP + " (" - + COLUMN_SQG_NAME + ") " - + "VALUES (substr(?, 1, 75))"; - - // DML: Select counter - public static final String STMT_SELECT_COUNTER = - "SELECT " - + COLUMN_SQR_ID + ", " - + COLUMN_SQR_NAME + " " - + "FROM " + TABLE_SQ_COUNTER + " " - + "WHERE " + COLUMN_SQR_NAME + " = substr(?, 1, 75)"; - - // DML: Insert new counter - public static final String STMT_INSERT_COUNTER = - "INSERT INTO " + TABLE_SQ_COUNTER + " (" - + COLUMN_SQR_NAME + ") " - + "VALUES (substr(?, 1, 75))"; - - // DML: Insert new counter submission - public static final String STMT_INSERT_COUNTER_SUBMISSION = - "INSERT INTO " + TABLE_SQ_COUNTER_SUBMISSION + " (" - + COLUMN_SQRS_GROUP + ", " - + COLUMN_SQRS_COUNTER + ", " - + COLUMN_SQRS_SUBMISSION + ", " - + COLUMN_SQRS_VALUE + ") " - + "VALUES (?, ?, ?, ?)"; - - // DML: Select counter submission - public static final String STMT_SELECT_COUNTER_SUBMISSION = - "SELECT " - + COLUMN_SQG_NAME + ", " - + COLUMN_SQR_NAME + ", " - + COLUMN_SQRS_VALUE + " " - + "FROM " + TABLE_SQ_COUNTER_SUBMISSION + " " - + "LEFT JOIN " + TABLE_SQ_COUNTER_GROUP - + " ON " + COLUMN_SQRS_GROUP + " = " + COLUMN_SQG_ID + " " - + "LEFT JOIN " + TABLE_SQ_COUNTER - + " ON " + COLUMN_SQRS_COUNTER + " = " + COLUMN_SQR_ID + " " - + "WHERE " + COLUMN_SQRS_SUBMISSION + " = ? "; - - // DML: Delete rows from counter submission table - public static final String STMT_DELETE_COUNTER_SUBMISSION = - "DELETE FROM " + TABLE_SQ_COUNTER_SUBMISSION - + " WHERE " + COLUMN_SQRS_SUBMISSION + " = ?"; - - /******* CONFIG and CONNECTOR DIRECTIONS ****/ - - public static final String STMT_INSERT_DIRECTION = "INSERT INTO " + TABLE_SQ_DIRECTION + " " + /******* CONFIG and CONNECTOR DIRECTIONS ****/ + public static final String STMT_INSERT_DIRECTION = "INSERT INTO " + TABLE_SQ_DIRECTION + " " + "(" + COLUMN_SQD_NAME + ") VALUES (?)"; - public static final String STMT_FETCH_CONFIG_DIRECTIONS = + public static final String STMT_FETCH_CONFIG_DIRECTIONS = "SELECT " + COLUMN_SQ_CFG_ID + ", " + COLUMN_SQ_CFG_DIRECTION + " FROM " + TABLE_SQ_CONFIG; - - public static final String STMT_INSERT_SQ_CONNECTOR_DIRECTIONS = + public static final String STMT_INSERT_SQ_CONNECTOR_DIRECTIONS = "INSERT INTO " + TABLE_SQ_CONNECTOR_DIRECTIONS + " " + "(" + COLUMN_SQCD_CONNECTOR + ", " + COLUMN_SQCD_DIRECTION + ")" + " VALUES (?, ?)"; - public static final String STMT_INSERT_SQ_CONFIG_DIRECTIONS = + public static final String STMT_INSERT_SQ_CONFIG_DIRECTIONS = "INSERT INTO " + TABLE_SQ_CONFIG_DIRECTIONS + " " + "(" + COLUMN_SQ_CFG_DIR_CONFIG + ", " + COLUMN_SQ_CFG_DIR_DIRECTION + ")" + " VALUES (?, ?)"; - public static final String STMT_SELECT_SQ_CONNECTOR_DIRECTIONS_ALL = - "SELECT " + COLUMN_SQCD_CONNECTOR + ", " + COLUMN_SQCD_DIRECTION - + " FROM " + TABLE_SQ_CONNECTOR_DIRECTIONS; - - public static final String STMT_SELECT_SQ_CONNECTOR_DIRECTIONS = - STMT_SELECT_SQ_CONNECTOR_DIRECTIONS_ALL + " WHERE " - + COLUMN_SQCD_CONNECTOR + " = ?"; - - public static final String STMT_SELECT_SQ_CONFIG_DIRECTIONS_ALL = - "SELECT " + COLUMN_SQ_CFG_DIR_CONFIG + ", " + COLUMN_SQ_CFG_DIR_DIRECTION - + " FROM " + TABLE_SQ_CONFIG_DIRECTIONS; - - public static final String STMT_SELECT_SQ_CONFIG_DIRECTIONS = - STMT_SELECT_SQ_CONFIG_DIRECTIONS_ALL + " WHERE " - + COLUMN_SQ_CFG_DIR_CONFIG + " = ?"; - - private DerbySchemaInsertUpdateDeleteSelectQuery() { // Disable explicit object creation } diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java index 85140d5c..45a96ad6 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestJobHandling.java @@ -36,6 +36,7 @@ import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; +import org.apache.sqoop.repository.common.CommonRepositoryError; import org.junit.Before; import org.junit.Test; @@ -65,7 +66,7 @@ public void testFindJob() throws Exception { handler.findJob(1, derbyConnection); fail(); } catch(SqoopException ex) { - assertEquals(DerbyRepoError.DERBYREPO_0030, ex.getErrorCode()); + assertEquals(CommonRepositoryError.COMMON_0027, ex.getErrorCode()); } loadJobsForLatestVersion(); diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java index dabb08b3..8ab384a5 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestLinkHandling.java @@ -31,6 +31,7 @@ import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MMapInput; import org.apache.sqoop.model.MStringInput; +import org.apache.sqoop.repository.common.CommonRepositoryError; import org.junit.Before; import org.junit.Test; @@ -58,8 +59,8 @@ public void testFindLink() throws Exception { try { handler.findLink(1, getDerbyDatabaseConnection()); fail(); - } catch (SqoopException ex) { - assertEquals(DerbyRepoError.DERBYREPO_0024, ex.getErrorCode()); + } catch(SqoopException ex) { + assertEquals(CommonRepositoryError.COMMON_0021, ex.getErrorCode()); } // Load prepared connections into database diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestRespositorySchemaUpgrade.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestRespositorySchemaUpgrade.java index c2819017..928c34a6 100644 --- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestRespositorySchemaUpgrade.java +++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestRespositorySchemaUpgrade.java @@ -138,7 +138,7 @@ public void testUpgradeRepoVersion2ToVersion4() throws Exception { private class TestDerbyRepositoryHandler extends DerbyRepositoryHandler { protected long registerHdfsConnector(Connection conn) { try { - runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)" + TestRespositorySchemaUpgrade.this.runQuery("INSERT INTO SQOOP.SQ_CONNECTOR(SQC_NAME, SQC_CLASS, SQC_VERSION)" + "VALUES('hdfs-connector', 'org.apache.sqoop.test.B', '1.0-test')"); return 2L; } catch (Exception e) { diff --git a/server/pom.xml b/server/pom.xml index be2ee9ba..4a5eb5e4 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -59,6 +59,11 @@ limitations under the License. hadoop${hadoop.profile} + + org.apache.sqoop.repository + sqoop-repository-common + + org.apache.sqoop.repository sqoop-repository-derby diff --git a/test/pom.xml b/test/pom.xml index 956aeb72..28a1e66f 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -62,6 +62,11 @@ limitations under the License. hadoop${hadoop.profile} + + org.apache.sqoop.repository + sqoop-repository-common + + org.apache.sqoop.repository sqoop-repository-derby