Examples of org.apache.flink.runtime.operators.util.TaskConfig

Package org.apache.flink.runtime.operators.util

Examples of org.apache.flink.runtime.operators.util.TaskConfig

org.apache.flink.runtime.operators.util.TaskConfig
Configuration class which stores all relevant parameters required to set up the Pact tasks.

  }


  private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks,
      TypeSerializerFactory<?> serializer) {
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Final Output", numSubTasks);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    {


      outputConfig.addInputToGroup(0);
      outputConfig.setInputSerializer(serializer, 0);


      outputConfig.setStubWrapper(new UserCodeClassWrapper<CsvOutputFormat>(CsvOutputFormat.class));
      outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, resultPath);


      Configuration outputUserConfig = outputConfig.getStubParameters();
      outputUserConfig.setString(CsvOutputFormat.RECORD_DELIMITER_PARAMETER, "\n");
      outputUserConfig.setString(CsvOutputFormat.FIELD_DELIMITER_PARAMETER, " ");
      outputUserConfig.setClass(CsvOutputFormat.FIELD_TYPE_PARAMETER_PREFIX + 0, LongValue.class);
      outputUserConfig.setInteger(CsvOutputFormat.RECORD_POSITION_PARAMETER_PREFIX + 0, 0);
      outputUserConfig.setClass(CsvOutputFormat.FIELD_TYPE_PARAMETER_PREFIX + 1, LongValue.class);

View Full Code Here

    return output;
  }


  private static AbstractJobVertex createSync(JobGraph jobGraph, int numSubTasks, int maxIterations) {
    AbstractJobVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);
    syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      new LongSumAggregator());
    syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME,
      new WorksetEmptyConvergenceCriterion());


    return sync;
  }

View Full Code Here

    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // --------------- the tail (solution set join) ---------------
    AbstractJobVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      tailConfig.setIsWorksetIteration();
      tailConfig.setIsWorksetUpdate();


      tailConfig.setIsSolutionSetUpdate();
      tailConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setDriverPairComparator(pairComparator);
      
      tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -- edges ------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);

View Full Code Here

    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setWaitForSolutionSetUpdate();


    // intermediate
    AbstractJobVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    OutputFormatVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    AbstractJobVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss join) ----------------------
    AbstractJobVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Join", jobGraph, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 1);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(
        new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    AbstractJobVertex ssTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail",
      jobGraph, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);
      ssTailConfig.setInputAsynchronouslyMaterialized(0, true);
      ssTailConfig.setRelativeInputMaterializationMemory(0, MEM_FRAC_PER_CONSUMER);


      // output
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // -------------------------- ws tail --------------------------------
    AbstractJobVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail",
      jobGraph, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);

View Full Code Here

  private static InputFormatVertex createPointsInput(JobGraph jobGraph, String pointsPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
    @SuppressWarnings("unchecked")
    CsvInputFormat pointsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
    InputFormatVertex pointsInput = JobGraphUtils.createInput(pointsInFormat, pointsPath, "[Points]", jobGraph, numSubTasks);
    {
      TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);
      
      TaskConfig chainedMapper = new TaskConfig(new Configuration());
      chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
      chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
      chainedMapper.setOutputSerializer(serializer);
      
      taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build points");
    }


    return pointsInput;

View Full Code Here

    @SuppressWarnings("unchecked")
    CsvInputFormat modelsInFormat = new CsvInputFormat('|', IntValue.class, DoubleValue.class, DoubleValue.class, DoubleValue.class);
    InputFormatVertex modelsInput = JobGraphUtils.createInput(modelsInFormat, centersPath, "[Models]", jobGraph, numSubTasks);


    {
      TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);


      TaskConfig chainedMapper = new TaskConfig(new Configuration());
      chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
      chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
      chainedMapper.setOutputSerializer(serializer);
      
      taskConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapper, "Build centers");
    }


    return modelsInput;

View Full Code Here

  private static OutputFormatVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
    
    OutputFormatVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks);


    {
      TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);


      PointOutFormat outFormat = new PointOutFormat();
      outFormat.setOutputFilePath(new Path(resultPath));
      
      taskConfig.setStubWrapper(new UserCodeObjectWrapper<PointOutFormat>(outFormat));
    }


    return output;
  }

View Full Code Here

  }
  
  private static AbstractJobVertex createIterationHead(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
    AbstractJobVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Iteration Head", jobGraph, numSubTasks);


    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setIterationId(ITERATION_ID);
    
    // initial input / partial solution
    headConfig.addInputToGroup(0);
    headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
    headConfig.setInputSerializer(serializer, 0);
    
    // back channel / iterations
    headConfig.setRelativeBackChannelMemory(MEMORY_FRACTION_PER_CONSUMER);
    
    // output into iteration. broadcasting the centers
    headConfig.setOutputSerializer(serializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(serializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync
    headConfig.setIterationHeadIndexOfSyncOutput(2);

View Full Code Here

    
    // now that the traversal is done, we have the chained tasks write their configs into their
    // parents' configurations
    for (int i = 0; i < this.chainedTasksInSequence.size(); i++) {
      TaskInChain tic = this.chainedTasksInSequence.get(i);
      TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration());
      t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
    }
    
    // create the jobgraph object
    JobGraph graph = new JobGraph(program.getJobName());
    graph.setNumberOfExecutionRetries(program.getOriginalPactPlan().getNumberOfExecutionRetries());

View Full Code Here

        
        // we adjust the joins / cogroups that go into the solution set here
        for (Channel c : node.getOutgoingChannels()) {
          DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
          AbstractJobVertex accessingVertex = this.vertices.get(target);
          TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
          int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
          
          // sanity checks
          if (inputNum == -1) {
            throw new CompilerException();
          }
          
          // adjust the driver
          if (conf.getDriver().equals(MatchDriver.class)) {
            conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
          }
          else if (conf.getDriver().equals(CoGroupDriver.class)) {
            conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
          }
          else {
            throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
          }
        }
        
        // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
        this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
        
        vertex = null;
      }
      else if (node instanceof WorksetPlanNode) {
        // create the iteration head here
        vertex = createWorksetIterationHead((WorksetPlanNode) node);
      }
      else {
        throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
      }
    }
    catch (Exception e) {
      throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
    }
    
    // check if a vertex was created, or if it was chained or skipped
    if (vertex != null) {
      // set degree of parallelism
      int pd = node.getDegreeOfParallelism();
      vertex.setParallelism(pd);
      
      vertex.setSlotSharingGroup(sharingGroup);
      
      // check whether this vertex is part of an iteration step function
      if (this.currentIteration != null) {
        // check that the task has the same DOP as the iteration as such
        PlanNode iterationNode = (PlanNode) this.currentIteration;
        if (iterationNode.getDegreeOfParallelism() < pd) {
          throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
        }


        // store the id of the iterations the step functions participate in
        IterationDescriptor descr = this.iterations.get(this.currentIteration);
        new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
      }
  
      // store in the map
      this.vertices.put(node, vertex);
    }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.flink.runtime.operators.util.TaskConfig

org.apache.flink.api.common.distributions.DataDistribution

org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator

org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator$IterationDescriptor

org.apache.flink.configuration.Configuration

org.apache.flink.core.memory.InputViewDataInputStreamWrapper

org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper

org.apache.flink.runtime.iterative.task.AbstractIterativePactTask

org.apache.flink.runtime.iterative.task.IterationHeadPactTask

org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask

org.apache.flink.runtime.jobgraph.InputFormatVertex

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.