Package com.google.appengine.tools.pipeline.impl.model

Examples of com.google.appengine.tools.pipeline.impl.model.JobRecord


   *         may be passed in to further invocations of {@code futureCall()} in
   *         order to specify a data dependency.
   */
  protected <T> FutureValue<T> futureCallUnchecked(JobSetting[] settings, Job<?> jobInstance,
      Object... params) {
    JobRecord childJobRecord =
        PipelineManager.registerNewJobRecord(updateSpec, settings, thisJobRecord, currentRunGUID,
            jobInstance, params);
    thisJobRecord.appendChildKey(childJobRecord.getKey());
    return new FutureValueImpl<T>(childJobRecord.getOutputSlotInflated());
  }
View Full Code Here


   */
  public static String startNewPipeline(
      JobSetting[] settings, Job<?> jobInstance, Object... params) {
    UpdateSpec updateSpec = new UpdateSpec(null);
    String graphGUID = null; // The root job graph doesn't have a GUID
    JobRecord parentJobKey = null; // The root job graph doesn't have a parent
    // Create the root Job and its associated Barriers and Slots
    // Create HandleSlotFilledTasks for the input parameters.
    JobRecord jobRecord =
        registerNewJobRecord(updateSpec, settings, parentJobKey, graphGUID, jobInstance, params);
    updateSpec.setRootJobKey(jobRecord.getRootJobKey());
    // Save the Pipeline model objects and enqueue the tasks that start the
    // Pipeline executing.
    backEnd.save(updateSpec);
    return jobRecord.getKey().getName();
  }
View Full Code Here

  public static JobRecord registerNewJobRecord(UpdateSpec updateSpec, JobSetting[] settings,
      JobRecord generatorJob, String graphGUID, Job<?> jobInstance, Object[] params) {
    Key rootKey = (null == generatorJob ? null : generatorJob.getRootJobKey());
    Key generatorKey = (null == generatorJob ? null : generatorJob.getKey());

    JobRecord jobRecord = new JobRecord(rootKey, generatorKey, graphGUID, jobInstance, settings);

    updateSpec.setRootJobKey(jobRecord.getRootJobKey());

    // Add slots to the RunBarrier corresponding to the input parameters
    for (Object param : params) {
      Value<?> value;
      if (null != param && param instanceof Value<?>) {
        value = (Value<?>) param;
      } else {
        value = new ImmediateValue<Object>(param);
      }
      registerSlotsWithBarrier(updateSpec, value, jobRecord.getRootJobKey(), generatorKey,
          graphGUID, jobRecord.getRunBarrierInflated());
    }

    if (0 == jobRecord.getRunBarrierInflated().getWaitingOnKeys().size()) {
      // If the run barrier is not waiting on anything, add a phantom filled
      // slot in order to trigger a HandleSlotFilledTask in order to trigger
      // a RunJobTask.
      Slot slot = new Slot(jobRecord.getRootJobKey(), generatorKey, graphGUID);
      jobRecord.getRunBarrierInflated().addPhantomArgumentSlot(slot);
      registerSlotFilled(updateSpec, slot, null);
    }

    // Register the newly created objects with the UpdateSpec.
    // The slots in the run Barrier have already been registered
    // and the finalize Barrier doesn't have any slots yet.
    // Any HandleSlotFilledTasks have also been registered already.
    Group updateGroup = updateSpec.getNonTransactionalGroup();
    updateGroup.includeBarrier(jobRecord.getRunBarrierInflated());
    updateGroup.includeBarrier(jobRecord.getFinalizeBarrierInflated());
    updateGroup.includeSlot(jobRecord.getOutputSlotInflated());
    updateGroup.includeJob(jobRecord);
    updateGroup.includeJobInstanceRecord(jobRecord.getJobInstanceInflated());

    return jobRecord;
  }
View Full Code Here

   *         be found in the data store.
   */
  public static void stopJob(String jobHandle) throws NoSuchObjectException {
    checkNonEmpty(jobHandle, "jobHandle");
    Key key = KeyFactory.createKey(JobRecord.DATA_STORE_KIND, jobHandle);
    JobRecord jobRecord = backEnd.queryJob(key, JobRecord.InflationType.NONE);
    jobRecord.setState(JobRecord.State.STOPPED);
    UpdateSpec updateSpec = new UpdateSpec(jobRecord.getRootJobKey());
    updateSpec.getTransaction("stopJob").includeJob(jobRecord);
    backEnd.save(updateSpec);
  }
View Full Code Here

    if (null == generatorJobKey) {
      throw new RuntimeException(
          "Pipeline is fatally corrupted. Slot for promised value has no generatorJobKey: "
          + slot);
    }
    JobRecord generatorJob = backEnd.queryJob(generatorJobKey, JobRecord.InflationType.NONE);
    if (null == generatorJob) {
      throw new RuntimeException("Pipeline is fatally corrupted. "
          + "The generator job for a promised value slot was not found: " + generatorJobKey);
    }
    String childGraphGuid = generatorJob.getChildGraphGuid();
    if (null == childGraphGuid) {
      // The generator job has not been saved with a childGraphGuid yet. This
      // can happen if
      // the promise handle leaked out to an external thread before the job that
      // generated it
View Full Code Here

   * @see "http://goto/java-pipeline-model"
   *
   * @param jobKey
   */
  private static void runJob(Key jobKey) {
    JobRecord jobRecord = null;
    jobRecord = queryJobOrAbandonTask(jobKey, JobRecord.InflationType.FOR_RUN);
    Key rootJobKey = jobRecord.getRootJobKey();
    logger.info("Running pipeline job " + jobKey.getName()
        + "; UI at " + PipelineServlet.makeViewerUrl(rootJobKey, jobKey));
    JobRecord rootJobRecord = jobRecord;
    if (!rootJobKey.equals(jobKey)) {
      rootJobRecord = queryJobOrAbandonTask(rootJobKey, JobRecord.InflationType.NONE);
    }
    if (rootJobRecord.getState() == JobRecord.State.STOPPED) {
      logger.warning("The pipeline has been stopped: " + rootJobRecord);
      throw new AbandonTaskException();
    }
    JobRecord.State jobState = jobRecord.getState();
    Barrier runBarrier = jobRecord.getRunBarrierInflated();
View Full Code Here

   * @see "http://goto/java-pipeline-model"
   *
   * @param jobKey
   */
  private static void finalizeJob(Key jobKey) {
    JobRecord jobRecord = null;

    // Get the JobRecord, its finalize Barrier, all the slots in the
    // finalize Barrier, and the job's output Slot.
    jobRecord = queryJobOrAbandonTask(jobKey, JobRecord.InflationType.FOR_FINALIZE);
    Barrier finalizeBarrier = jobRecord.getFinalizeBarrierInflated();
    if (null == finalizeBarrier) {
      throw new RuntimeException("" + jobRecord + " has not been inflated");
    }
    Slot outputSlot = jobRecord.getOutputSlotInflated();
    if (null == outputSlot) {
      throw new RuntimeException("" + jobRecord + " has not been inflated.");
    }

    // release the finalize barrier now so that any concurrent
    // HandleSlotFilled tasks will stop trying
    finalizeBarrier.setReleased();
    UpdateSpec updateSpec = new UpdateSpec(jobRecord.getRootJobKey());
    updateSpec.getTransaction("releaseFinalizeBarrier").includeBarrier(finalizeBarrier);
    backEnd.save(updateSpec);
    updateSpec = new UpdateSpec(jobRecord.getRootJobKey());

    // Copy the finalize value to the output slot
    List<Object> finalizeArguments = finalizeBarrier.buildArgumentList();
    int numFinalizeArguments = finalizeArguments.size();
    if (1 != numFinalizeArguments) {
      throw new RuntimeException("Internal logic error: numFinalizeArguments="
          + numFinalizeArguments);
    }
    Object finalizeValue = finalizeArguments.get(0);
    logger.finest("Finalizing " + jobRecord + " with value=" + finalizeValue);
    outputSlot.fill(finalizeValue);

    // Change state of the job to FINALIZED and set the end time
    jobRecord.setState(JobRecord.State.FINALIZED);
    jobRecord.setEndTime(new Date());

    // Propagate the filler of the finalize slot to also be the filler of the
    // output slot. If there is no unique filler of the finalize slot then we
    // resort to assigning the current job as the filler job.
    Key fillerJobKey = getFinalizeSlotFiller(finalizeBarrier);
View Full Code Here

          entity = dataStore.get(jobKey);
        } catch (EntityNotFoundException e) {
          throw new RuntimeException(
              "Fatal Pipeline corruption error. No JobRecord found with key = " + jobKey);
        }
        JobRecord jobRecord = new JobRecord(entity);
        JobRecord.State state = jobRecord.getState();
        boolean stateIsExpected = false;
        for (JobRecord.State expectedState : expectedStates) {
          if (state == expectedState) {
            stateIsExpected = true;
            break;
View Full Code Here

  @Override
  public JobRecord queryJob(Key jobKey, JobRecord.InflationType inflationType)
      throws NoSuchObjectException {
    try {
      Entity entity = transactionallyQueryEntity(jobKey);
      JobRecord jobRecord = new JobRecord(entity);
      Barrier runBarrier = null;
      Barrier finalizeBarrier = null;
      Slot outputSlot = null;
      JobInstanceRecord jobInstanceRecord = null;
      switch (inflationType) {
        case FOR_RUN:
          runBarrier = queryBarrier(jobRecord.getRunBarrierKey(), true, true);
          finalizeBarrier = queryBarrier(jobRecord.getFinalizeBarrierKey(), false, true);
          jobInstanceRecord = queryJobInstanceRecord(jobRecord.getJobInstanceKey());
          outputSlot = querySlot(jobRecord.getOutputSlotKey(), false);
          break;
        case FOR_FINALIZE:
          finalizeBarrier = queryBarrier(jobRecord.getFinalizeBarrierKey(), true, true);
          outputSlot = querySlot(jobRecord.getOutputSlotKey(), false);
          break;
        case FOR_OUTPUT:
          outputSlot = querySlot(jobRecord.getOutputSlotKey(), false);
          break;
        default:
      }
      jobRecord.inflate(runBarrier, finalizeBarrier, outputSlot, jobInstanceRecord);
      logger.finest("Query returned: " + jobRecord);
      return jobRecord;
    } catch (EntityNotFoundException e) {
      throw new NoSuchObjectException(jobKey.toString(), e);
    }
View Full Code Here

      }
    }, Slot.DATA_STORE_KIND, rootJobKey);
    putAll(jobs, new Instantiator<JobRecord>() {
      @Override
      public JobRecord newObject(Entity entity) {
        JobRecord jobRecord = new JobRecord(entity);
        return jobRecord;
      }
    }, JobRecord.DATA_STORE_KIND, rootJobKey);
    putAll(jobInstanceRecords, new Instantiator<JobInstanceRecord>() {
      @Override
View Full Code Here

TOP

Related Classes of com.google.appengine.tools.pipeline.impl.model.JobRecord

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.