Examples of DBConfiguration


Examples of co.nubetech.hiho.mapreduce.lib.db.apache.DBConfiguration

  @SuppressWarnings("unchecked")
  public static String getSelectQuery(Configuration conf, String dbProductName)
      throws HIHOException {

    StringBuilder query = new StringBuilder();
    DBConfiguration dbConf = new DBConfiguration(conf);
    String[] fieldNames = dbConf.getInputFieldNames();
    String tableName = dbConf.getInputTableName();
    String conditions = dbConf.getInputConditions();
    StringBuilder conditionClauses = new StringBuilder();

    if (dbConf.getInputQuery() == null) {
      // We need to generate the entire query.
      query.append("SELECT ");

      for (int i = 0; i < fieldNames.length; i++) {
        query.append(fieldNames[i]);
        if (i != fieldNames.length - 1) {
          query.append(", ");
        }
      }

      query.append(" FROM ").append(tableName);
      if (!dbProductName.startsWith("ORACLE")) {
        // Seems to be necessary for hsqldb? Oracle explicitly does
        // *not*
        // use this clause.
        query.append(" AS ").append(tableName);
      }

    } else {
      // User provided the query. We replace the special token with our
      // WHERE clause.
      String inputQuery = dbConf.getInputQuery();
      if (inputQuery.indexOf(DataDrivenDBInputFormat.SUBSTITUTE_TOKEN) == -1) {
        logger.error("Could not find the clause substitution token "
            + DataDrivenDBInputFormat.SUBSTITUTE_TOKEN
            + " in the query: [" + inputQuery
            + "]. Parallel splits may not work correctly.");
View Full Code Here

Examples of co.nubetech.hiho.mapreduce.lib.db.apache.DBConfiguration

  @Override
  protected RecordReader<LongWritable, GenericDBWritable> createDBRecordReader(
      DBInputSplit split, Configuration conf) throws IOException {

    DBConfiguration dbConf = getDBConf();
    @SuppressWarnings("unchecked")
    // Class<T> inputClass = (Class<T>) (dbConf.getInputClass());
    String dbProductName = getDBProductName();

    logger.debug("Creating db record reader for db product: "
        + dbProductName);
    ArrayList params = null;
    try {
      if (conf.get(HIHOConf.QUERY_PARAMS) != null) {
        logger.debug("creating stringifier in DBQueryInputFormat");
        DefaultStringifier<ArrayList> stringifier = new DefaultStringifier<ArrayList>(
            conf, ArrayList.class);
        logger.debug("created stringifier");

        params = stringifier
            .fromString(conf.get(HIHOConf.QUERY_PARAMS));
        logger.debug("created params");
      }
      // use database product name to determine appropriate record reader.
      if (dbProductName.startsWith("MYSQL")) {
        // use MySQL-specific db reader.
        return new MySQLQueryRecordReader(split, conf, getConnection(),
            dbConf, dbConf.getInputConditions(),
            dbConf.getInputFieldNames(),
            dbConf.getInputTableName(), params);
      } else {
        // Generic reader.
        return new DBQueryRecordReader(split, conf, getConnection(),
            dbConf, dbConf.getInputConditions(),
            dbConf.getInputFieldNames(),
            dbConf.getInputTableName(), dbProductName, params);
      }
    } catch (SQLException ex) {
      throw new IOException(ex.getMessage());
    }
  }
View Full Code Here

Examples of co.nubetech.hiho.mapreduce.lib.db.apache.DBConfiguration

  }

  public static void setOutput(Job job, String tableName, String columnNames)
      throws IOException{
    job.setOutputFormatClass(GenericDBOutputFormat.class);
    DBConfiguration dbConf = new DBConfiguration(job.getConfiguration());
    dbConf.setOutputTableName(tableName);
    dbConf.setOutputFieldNames(columnNames);

    String dbDriver = job.getConfiguration().get(
        DBConfiguration.DRIVER_CLASS_PROPERTY);
    String connString = job.getConfiguration().get(
        DBConfiguration.URL_PROPERTY);
View Full Code Here

Examples of com.celum.dbtool.configuration.DbConfiguration

    public void execute() throws MojoExecutionException, MojoFailureException
    {
        try {
            registerVersionFactoryFromSettings();

            DbConfiguration config =
                    DbConfiguration.createSelfInitConfigurationForTable(patchTable)
                                   .setDataSource(getDataSource())
                                   .setVariables(getParameters())
                                   .setEventListener(this);

            if (stepsPackage == null || stepsPackage.isEmpty()) {
                config.setStepsSourceForDirectory(stepsDir);
            } else {
                config.setStepsSource(new PackageResource(stepsPackage, getClassLoaderWithCompiledClasses()));
            }

            config.build().patch();

        } catch (Exception e) {
            getLog().error(e);
        }
    }
View Full Code Here

Examples of com.cloudera.sqoop.mapreduce.db.DBConfiguration

  }

  protected RecordReader<LongWritable, T> createDBRecordReader(
      DBInputSplit split, Configuration conf) throws IOException {

    DBConfiguration dbConf = getDBConf();
    @SuppressWarnings("unchecked")
    Class<T> inputClass = (Class<T>) (dbConf.getInputClass());
    String dbProductName = getDBProductName();

    LOG.debug("Creating db record reader for db product: " + dbProductName);

    try {
      return new DataDrivenDBRecordReader<T>(split, inputClass,
          conf, getConnection(), dbConf, dbConf.getInputConditions(),
          dbConf.getInputFieldNames(), dbConf.getInputTableName(),
          dbProductName);
    } catch (SQLException ex) {
      throw new IOException(ex);
    }
  }
View Full Code Here

Examples of com.cloudera.sqoop.mapreduce.db.DBConfiguration

  @Override
  /** {@inheritDoc} */
  public void setConf(Configuration conf) {

    dbConf = new DBConfiguration(conf);

    try {
      getConnection();

      DatabaseMetaData dbMeta = connection.getMetaData();
View Full Code Here

Examples of com.cloudera.sqoop.mapreduce.db.DBConfiguration

  public static void setInput(Job job,
      Class<? extends DBWritable> inputClass,
      String tableName, String conditions,
      String orderBy, String... fieldNames) {
    job.setInputFormatClass(DBInputFormat.class);
    DBConfiguration dbConf = new DBConfiguration(job.getConfiguration());
    dbConf.setInputClass(inputClass);
    dbConf.setInputTableName(tableName);
    dbConf.setInputFieldNames(fieldNames);
    dbConf.setInputConditions(conditions);
    dbConf.setInputOrderBy(orderBy);
  }
View Full Code Here

Examples of com.cloudera.sqoop.mapreduce.db.DBConfiguration

   */
  public static void setInput(Job job,
      Class<? extends DBWritable> inputClass,
      String inputQuery, String inputCountQuery) {
    job.setInputFormatClass(DBInputFormat.class);
    DBConfiguration dbConf = new DBConfiguration(job.getConfiguration());
    dbConf.setInputClass(inputClass);
    dbConf.setInputQuery(inputQuery);
    dbConf.setInputCountQuery(inputCountQuery);
  }
View Full Code Here

Examples of com.cloudera.sqoop.mapreduce.db.DBConfiguration

  /** {@inheritDoc} */
  @Override
  protected RecordReader<LongWritable, T> createDBRecordReader(
      DBInputSplit split, Configuration conf) throws IOException {

    DBConfiguration dbConf = getDBConf();
    @SuppressWarnings("unchecked")
    Class<T> inputClass = (Class<T>) (dbConf.getInputClass());

    try {
      // Use Microsoft SQL Server specific db reader
      return new SqlServerRecordReader<T>(split, inputClass,
          conf, getConnection(), dbConf, dbConf.getInputConditions(),
          dbConf.getInputFieldNames(), dbConf.getInputTableName());
    } catch (SQLException ex) {
      throw new IOException(ex);
    }
  }
View Full Code Here

Examples of com.cloudera.sqoop.mapreduce.db.DBConfiguration

    StringBuilder query = new StringBuilder();

    DataDrivenDBInputFormat.DataDrivenDBInputSplit dataSplit =
        (DataDrivenDBInputFormat.DataDrivenDBInputSplit) getSplit();

    DBConfiguration dbConf = getDBConf();
    String [] fieldNames = getFieldNames();
    String tableName = getTableName();
    String conditions = getConditions();

    // Build the WHERE clauses associated with the data split first.
    // We need them in both branches of this function.
    StringBuilder conditionClauses = new StringBuilder();
    conditionClauses.append("( ").append(dataSplit.getLowerClause());
    conditionClauses.append(" ) AND ( ").append(dataSplit.getUpperClause());
    conditionClauses.append(" )");

    if (dbConf.getInputQuery() == null) {
      // We need to generate the entire query.
      query.append("SELECT ");

      for (int i = 0; i < fieldNames.length; i++) {
        query.append(fieldNames[i]);
        if (i != fieldNames.length -1) {
          query.append(", ");
        }
      }

      query.append(" FROM ").append(tableName);

      String tableHints =
        dbConf.getConf().get(SQLServerManager.TABLE_HINTS_PROP);
      if (tableHints != null) {
        LOG.info("Using table hints: " + tableHints);
        query.append(" WITH (").append(tableHints).append(")");
      }

      query.append(" WHERE ");
      if (conditions != null && conditions.length() > 0) {
        // Put the user's conditions first.
        query.append("( ").append(conditions).append(" ) AND ");
      }

      // Now append the conditions associated with our split.
      query.append(conditionClauses.toString());

    } else {
      // User provided the query. We replace the special token with
      // our WHERE clause.
      String inputQuery = dbConf.getInputQuery();
      if (inputQuery.indexOf(DataDrivenDBInputFormat.SUBSTITUTE_TOKEN) == -1) {
        LOG.error("Could not find the clause substitution token "
            + DataDrivenDBInputFormat.SUBSTITUTE_TOKEN + " in the query: ["
            + inputQuery + "]. Parallel splits may not work correctly.");
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.