Examples of eu.stratosphere.nephele.jobgraph.JobGraph

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobGraph

eu.stratosphere.nephele.jobgraph.JobGraph
A job graph represents an entire job in Nephele. A job graph must consists at least of one job vertex and must be acyclic.

      failingWorkers = args[12];
      failingIteration = Integer.parseInt(args[13]);
      messageLoss = Double.parseDouble(args[14]);
    }


    JobGraph jobGraph = new JobGraph("CompensatableDanglingPageRank");
    
    // --------------- the inputs ---------------------


    // page rank input
    JobInputVertex pageWithRankInput = JobGraphUtils.createInput(new CustomImprovedDanglingPageRankInputFormat(),

View Full Code Here

   *        the instance manager that shall be used during the creation of the execution graph
   * @return a sample execution graph
   */
  private ExecutionGraph createExecutionGraph(final ChannelType channelType, final InstanceManager instanceManager) {


    final JobGraph jobGraph = new JobGraph("Job Graph");


    final JobInputVertex inputVertex = new JobInputVertex("Input 1", jobGraph);
    inputVertex.setInputClass(InputTask.class);
    inputVertex.setNumberOfSubtasks(1);


    final JobOutputVertex outputVertex = new JobOutputVertex("Output 1", jobGraph);
    outputVertex.setOutputClass(OutputTask.class);
    outputVertex.setNumberOfSubtasks(1);


    try {
      inputVertex.connectTo(outputVertex, channelType);
    } catch (JobGraphDefinitionException e) {
      fail(StringUtils.stringifyException(e));
    }


    try {
      LibraryCacheManager.register(jobGraph.getJobID(), new String[0]);
      return new ExecutionGraph(jobGraph, instanceManager);


    } catch (GraphConversionException e) {
      fail(StringUtils.stringifyException(e));
    } catch (IOException e) {

View Full Code Here

  }
    
  public void runAndCancelJob(Plan plan, int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
    try {
      // submit job
      final JobGraph jobGraph = getJobGraph(plan);


      final long startingTime = System.currentTimeMillis();
      long cancelTime = -1L;
      final JobClient client = this.executor.getJobClient(jobGraph);
      final JobSubmissionResult submissionResult = client.submitJob();

View Full Code Here

  private JobGraph createJobGraphV1(String pointsPath, String centersPath, String resultPath, int numSubTasks) throws JobGraphDefinitionException {


    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();


    JobGraph jobGraph = new JobGraph("Distance Builder");


    // -- vertices ---------------------------------------------------------------------------------------------
    JobInputVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
    JobInputVertex models = createModelsInput(jobGraph, centersPath, numSubTasks, serializer);
    JobTaskVertex mapper = createMapper(jobGraph, numSubTasks, serializer);

View Full Code Here

      JarFileCreator jfc = new JarFileCreator(jarFile);
      jfc.addClass(UnionTask.class);
      jfc.createJarFile();


      // Create job graph
      final JobGraph jg = new JobGraph("Job with large DoP (" + numberOfSubtasks + ")");


      // input vertex 1
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(numberOfSubtasks);
      i1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // input vertex 2
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(numberOfSubtasks);
      i2.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // union task
      final JobTaskVertex f1 = new JobTaskVertex("Forward 1", jg);
      f1.setTaskClass(DoubleTargetTask.class);
      f1.setNumberOfSubtasks(numberOfSubtasks);
      f1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));
      o1.setNumberOfSubtasks(numberOfSubtasks);
      o1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      i1.setVertexToShareInstancesWith(o1);
      i2.setVertexToShareInstancesWith(o1);
      f1.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(f1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      i2.connectTo(f1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      f1.connectTo(o1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      
      // disable logging for the taskmanager and the client, as they will have many

View Full Code Here

  }


  private JobGraph createJobGraph(int dataVolumeGb, boolean useForwarder, boolean isSlowSender, boolean isSlowReceiver,
                  int numSubtasks, int numSubtasksPerInstance) throws JobGraphDefinitionException {


    JobGraph jobGraph = new JobGraph("Speed Test");


    JobInputVertex producer = new JobGenericInputVertex("Speed Test Producer", jobGraph);
    producer.setInputClass(SpeedTestProducer.class);
    producer.setNumberOfSubtasks(numSubtasks);
    producer.setNumberOfSubtasksPerInstance(numSubtasksPerInstance);

View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // -- invariant vertices -----------------------------------------------------------------------------------
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
    JobTaskVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);

View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // input
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);

View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Intermediate Workset Update, Solution Set Tail)");


    // input
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);

View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Intermediate Solution Set Update, Workset Tail)");


    // input
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);

View Full Code Here

0 1 2 3

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobGraph

eu.stratosphere.client.LocalExecutor

eu.stratosphere.client.program.Client

eu.stratosphere.client.web.JobSubmissionServlet

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

eu.stratosphere.configuration.Configuration

eu.stratosphere.core.fs.FileStatus

eu.stratosphere.core.fs.FileSystem

eu.stratosphere.core.fs.FSDataInputStream

eu.stratosphere.core.fs.Path

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.