Package org.apache.sqoop.repository

Examples of org.apache.sqoop.repository.Repository


   */
  private JsonBean deleteConnection(RequestContext ctx) {
    String sxid = ctx.getLastURLElement();
    long xid = Long.valueOf(sxid);

    Repository repository = RepositoryManager.getRepository();
    repository.deleteConnection(xid);

    return JsonBean.EMPTY_BEAN;
  }
View Full Code Here


  private JsonBean getConnections(RequestContext ctx) {
    String sxid = ctx.getLastURLElement();
    ConnectionBean bean;

    Locale locale = ctx.getAcceptLanguageHeader();
    Repository repository = RepositoryManager.getRepository();

    if (sxid.equals("all")) {

      List<MConnection> connections = repository.findConnections();
      bean = new ConnectionBean(connections);

      // Add associated resources into the bean
      for( MConnection connection : connections) {
        long connectorId = connection.getConnectorId();
        if(!bean.hasConnectorBundle(connectorId)) {
          bean.addConnectorBundle(connectorId,
            ConnectorManager.getResourceBundle(connectorId, locale));
        }
      }
    } else {
      long xid = Long.valueOf(sxid);

      MConnection connection = repository.findConnection(xid);
      long connectorId = connection.getConnectorId();

      bean = new ConnectionBean(connection);

      bean.addConnectorBundle(connectorId,
View Full Code Here

   */
  private JsonBean deleteJob(RequestContext ctx) {
    String sxid = ctx.getLastURLElement();
    long jid = Long.valueOf(sxid);

    Repository repository = RepositoryManager.getRepository();
    repository.deleteJob(jid);

    return JsonBean.EMPTY_BEAN;
  }
View Full Code Here

  private JsonBean getJobs(RequestContext ctx) {
    String sjid = ctx.getLastURLElement();
    JobBean bean;

    Locale locale = ctx.getAcceptLanguageHeader();
    Repository repository = RepositoryManager.getRepository();

    if (sjid.equals("all")) {

      List<MJob> jobs = repository.findJobs();
      bean = new JobBean(jobs);

      // Add associated resources into the bean
      for( MJob job : jobs) {
        long connectorId = job.getConnectorId();
        if(!bean.hasConnectorBundle(connectorId)) {
          bean.addConnectorBundle(connectorId,
            ConnectorManager.getResourceBundle(connectorId, locale));
        }
      }
    } else {
      long jid = Long.valueOf(sjid);

      MJob job = repository.findJob(jid);
      long connectorId = job.getConnectorId();

      bean = new JobBean(job);

      bean.addConnectorBundle(connectorId,
View Full Code Here

    return ResourceBundle.getBundle(
        FrameworkConstants.RESOURCE_BUNDLE_NAME, locale);
  }

  public static MSubmission submit(long jobId) {
    Repository repository = RepositoryManager.getRepository();

    MJob job = repository.findJob(jobId);
    if(job == null) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0004,
        "Unknown job id " + jobId);
    }
    MConnection connection = repository.findConnection(job.getConnectionId());
    SqoopConnector connector =
      ConnectorManager.getConnector(job.getConnectorId());

    // Transform forms to connector specific classes
    Object connectorConnection = ClassUtils.instantiate(
      connector.getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getConnectorPart().getForms(),
      connectorConnection);

    Object connectorJob = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorJob);

    // Transform framework specific forms
    Object frameworkConnection = ClassUtils.instantiate(
      getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getFrameworkPart().getForms(),
      frameworkConnection);

    Object frameworkJob = ClassUtils.instantiate(
      getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);

    // Create request object
    MSubmission summary = new MSubmission(jobId);
    SubmissionRequest request = executionEngine.createSubmissionRequest();

    // Save important variables to the submission request
    request.setSummary(summary);
    request.setConnector(connector);
    request.setConfigConnectorConnection(connectorConnection);
    request.setConfigConnectorJob(connectorJob);
    request.setConfigFrameworkConnection(frameworkConnection);
    request.setConfigFrameworkJob(frameworkJob);
    request.setJobType(job.getType());
    request.setJobName(job.getName());
    request.setJobId(job.getPersistenceId());
    request.setNotificationUrl(notificationBaseUrl + jobId);

    // Let's register all important jars
    // sqoop-common
    request.addJarForClass(MapContext.class);
    // sqoop-core
    request.addJarForClass(FrameworkManager.class);
    // sqoop-spi
    request.addJarForClass(SqoopConnector.class);
    // Execution engine jar
    request.addJarForClass(executionEngine.getClass());
    // Connector in use
    request.addJarForClass(connector.getClass());

    // Extra libraries that Sqoop code requires
    request.addJarForClass(JSONValue.class);

    // Get connector callbacks
    switch (job.getType()) {
      case IMPORT:
        request.setConnectorCallbacks(connector.getImporter());
        break;
      case EXPORT:
        request.setConnectorCallbacks(connector.getExporter());
        break;
      default:
        throw  new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }
    LOG.debug("Using callbacks: " + request.getConnectorCallbacks());

    // Initialize submission from connector perspective
    CallbackBase baseCallbacks = request.getConnectorCallbacks();

    Class<? extends Initializer> initializerClass = baseCallbacks.getInitializer();
    Initializer initializer = (Initializer) ClassUtils.instantiate(initializerClass);

    if(initializer == null) {
      throw  new SqoopException(FrameworkError.FRAMEWORK_0006,
        "Can't create initializer instance: " + initializerClass.getName());
    }

    // Initialize submission from connector perspective
    initializer.initialize(request.getConnectorContext(),
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob());

    // Add job specific jars to
    request.addJars(initializer.getJars(request.getConnectorContext(),
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob()));

    // Bootstrap job from framework perspective
    switch (job.getType()) {
      case IMPORT:
        prepareImportSubmission(request);
        break;
      case EXPORT:
        prepareExportSubmission(request);
        break;
      default:
        throw  new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }

    // Make sure that this job id is not currently running and submit the job
    // only if it's not.
    synchronized (submissionMutex) {
      MSubmission lastSubmission = repository.findSubmissionLastForJob(jobId);
      if(lastSubmission != null && lastSubmission.getStatus().isRunning()) {
        throw new SqoopException(FrameworkError.FRAMEWORK_0002,
          "Job with id " + jobId);
      }

      // TODO(jarcec): We might need to catch all exceptions here to ensure
      // that Destroyer will be executed in all cases.
      boolean submitted = submissionEngine.submit(request);
      if(!submitted) {
        destroySubmission(request);
        summary.setStatus(SubmissionStatus.FAILURE_ON_SUBMIT);
      }

      repository.createSubmission(summary);
    }

    // Return job status most recent
    return summary;
  }
View Full Code Here

    destroyer.destroy(false, request.getConnectorContext(),
      request.getConfigConnectorConnection(), request.getConfigConnectorJob());
  }

  public static MSubmission stop(long jobId) {
    Repository repository = RepositoryManager.getRepository();
    MSubmission submission = repository.findSubmissionLastForJob(jobId);

    if(!submission.getStatus().isRunning()) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0003,
        "Job with id " + jobId + " is not running");
    }
View Full Code Here

    // Return updated structure
    return submission;
  }

  public static MSubmission status(long jobId) {
    Repository repository = RepositoryManager.getRepository();
    MSubmission submission = repository.findSubmissionLastForJob(jobId);

    if(submission == null) {
      return new MSubmission(jobId, new Date(), SubmissionStatus.NEVER_EXECUTED);
    }
View Full Code Here

      LOG.info("Connectors loaded: " + handlerMap);
    }
  }

  private static synchronized void registerConnectors() {
    Repository repository = RepositoryManager.getRepository();

    RepositoryTransaction rtx = null;
    try {
      rtx = repository.getTransaction();
      rtx.begin();
      for (String name : handlerMap.keySet()) {
        ConnectorHandler handler = handlerMap.get(name);
        MConnector connectorMetadata = handler.getMetadata();
        MConnector registeredMetadata =
            repository.registerConnector(connectorMetadata);

        // Set registered metadata instead of connector metadata as they will
        // have filled persistent ids. We should be confident at this point that
        // there are no differences between those two structures.
        handler.setMetadata(registeredMetadata);
View Full Code Here

      LOG.info("Connectors loaded: " + handlerMap);
    }
  }

  private synchronized void registerConnectors() {
    Repository repository = RepositoryManager.getInstance().getRepository();

    RepositoryTransaction rtx = null;
    try {
      rtx = repository.getTransaction();
      rtx.begin();
      for (String name : handlerMap.keySet()) {
        ConnectorHandler handler = handlerMap.get(name);
        MConnector connectorMetadata = handler.getMetadata();
        MConnector registeredMetadata =
            repository.registerConnector(connectorMetadata);

        // Set registered metadata instead of connector metadata as they will
        // have filled persistent ids. We should be confident at this point that
        // there are no differences between those two structures.
        handler.setMetadata(registeredMetadata);
View Full Code Here

    return ResourceBundle.getBundle(
        FrameworkConstants.RESOURCE_BUNDLE_NAME, locale);
  }

  public MSubmission submit(long jobId) {
    Repository repository = RepositoryManager.getInstance().getRepository();

    MJob job = repository.findJob(jobId);
    if(job == null) {
      throw new SqoopException(FrameworkError.FRAMEWORK_0004,
        "Unknown job id " + jobId);
    }
    MConnection connection = repository.findConnection(job.getConnectionId());
    SqoopConnector connector =
      ConnectorManager.getInstance().getConnector(job.getConnectorId());

    // Transform forms to connector specific classes
    Object connectorConnection = ClassUtils.instantiate(
      connector.getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getConnectorPart().getForms(),
      connectorConnection);

    Object connectorJob = ClassUtils.instantiate(
      connector.getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getConnectorPart().getForms(), connectorJob);

    // Transform framework specific forms
    Object frameworkConnection = ClassUtils.instantiate(
      getConnectionConfigurationClass());
    FormUtils.fromForms(connection.getFrameworkPart().getForms(),
      frameworkConnection);

    Object frameworkJob = ClassUtils.instantiate(
      getJobConfigurationClass(job.getType()));
    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);

    // Create request object
    MSubmission summary = new MSubmission(jobId);
    SubmissionRequest request = executionEngine.createSubmissionRequest();

    // Save important variables to the submission request
    request.setSummary(summary);
    request.setConnector(connector);
    request.setConfigConnectorConnection(connectorConnection);
    request.setConfigConnectorJob(connectorJob);
    request.setConfigFrameworkConnection(frameworkConnection);
    request.setConfigFrameworkJob(frameworkJob);
    request.setJobType(job.getType());
    request.setJobName(job.getName());
    request.setJobId(job.getPersistenceId());
    request.setNotificationUrl(notificationBaseUrl + jobId);

    // Let's register all important jars
    // sqoop-common
    request.addJarForClass(MapContext.class);
    // sqoop-core
    request.addJarForClass(FrameworkManager.class);
    // sqoop-spi
    request.addJarForClass(SqoopConnector.class);
    // Execution engine jar
    request.addJarForClass(executionEngine.getClass());
    // Connector in use
    request.addJarForClass(connector.getClass());

    // Extra libraries that Sqoop code requires
    request.addJarForClass(JSONValue.class);

    // Get connector callbacks
    switch (job.getType()) {
      case IMPORT:
        request.setConnectorCallbacks(connector.getImporter());
        break;
      case EXPORT:
        request.setConnectorCallbacks(connector.getExporter());
        break;
      default:
        throw  new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }
    LOG.debug("Using callbacks: " + request.getConnectorCallbacks());

    // Initialize submission from connector perspective
    CallbackBase baseCallbacks = request.getConnectorCallbacks();

    Class<? extends Initializer> initializerClass = baseCallbacks.getInitializer();
    Initializer initializer = (Initializer) ClassUtils.instantiate(initializerClass);

    if(initializer == null) {
      throw  new SqoopException(FrameworkError.FRAMEWORK_0006,
        "Can't create initializer instance: " + initializerClass.getName());
    }

    // Initializer context
    InitializerContext initializerContext = new InitializerContext(request.getConnectorContext());

    // Initialize submission from connector perspective
    initializer.initialize(initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob());

    // Add job specific jars to
    request.addJars(initializer.getJars(initializerContext,
      request.getConfigConnectorConnection(),
      request.getConfigConnectorJob()));

    // Bootstrap job from framework perspective
    switch (job.getType()) {
      case IMPORT:
        prepareImportSubmission(request);
        break;
      case EXPORT:
        prepareExportSubmission(request);
        break;
      default:
        throw  new SqoopException(FrameworkError.FRAMEWORK_0005,
          "Unsupported job type " + job.getType().name());
    }

    // Make sure that this job id is not currently running and submit the job
    // only if it's not.
    synchronized (submissionMutex) {
      MSubmission lastSubmission = repository.findSubmissionLastForJob(jobId);
      if(lastSubmission != null && lastSubmission.getStatus().isRunning()) {
        throw new SqoopException(FrameworkError.FRAMEWORK_0002,
          "Job with id " + jobId);
      }

      // TODO(jarcec): We might need to catch all exceptions here to ensure
      // that Destroyer will be executed in all cases.
      boolean submitted = submissionEngine.submit(request);
      if(!submitted) {
        destroySubmission(request);
        summary.setStatus(SubmissionStatus.FAILURE_ON_SUBMIT);
      }

      repository.createSubmission(summary);
    }

    // Return job status most recent
    return summary;
  }
View Full Code Here

TOP

Related Classes of org.apache.sqoop.repository.Repository

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.