Package org.apache.flink.runtime.operators.util

Examples of org.apache.flink.runtime.operators.util.TaskConfig

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
  public void registerFileOutputTask(AbstractInvokable outTask, Class<? extends FileOutputFormat> stubClass, String outPath) {
    registerFileOutputTask(outTask, InstantiationUtil.instantiate(stubClass, FileOutputFormat.class), outPath);
  }
 
  public void registerFileOutputTask(AbstractInvokable outTask, FileOutputFormat outputFormat, String outPath) {
    TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());
   
    outputFormat.setOutputFilePath(new Path(outPath));
    outputFormat.setWriteMode(WriteMode.OVERWRITE);

    dsConfig.setStubWrapper(new UserCodeObjectWrapper<FileOutputFormat>(outputFormat));

    outTask.setEnvironment(this.mockEnv);

    outTask.registerInputOutput();
  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
    }
   
    format.setFilePath(inPath);
    format.setDelimiter(delimiter);
   
    TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());
    dsConfig.setStubWrapper(new UserCodeObjectWrapper<DelimitedInputFormat>(format));
   
    this.inputSplitProvider.addInputSplits(inPath, 5);

    inTask.setEnvironment(this.mockEnv);
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
  // --------------------------------------------------------------------------------------------

  @Override
  public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    this.strategy = config.getDriverStrategy();
   
    if (strategy == DriverStrategy.ALL_GROUP_COMBINE) {
      if (!(this.taskContext.getStub() instanceof FlatCombineFunction)) {
        throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + FlatCombineFunction.class.getName());
      }
    }
    else if (strategy != DriverStrategy.ALL_GROUP_REDUCE) {
      throw new Exception("Unrecognized driver strategy for AllGroupReduce driver: " + config.getDriverStrategy().name());
    }
    this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
    this.input = this.taskContext.getInput(0);
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
  // --------------------------------------------------------------------------------------------

  @Override
  public void prepare() throws Exception {
    TaskConfig config = this.taskContext.getTaskConfig();
    if (config.getDriverStrategy() != DriverStrategy.SORTED_GROUP_REDUCE) {
      throw new Exception("Unrecognized driver strategy for GroupReduce driver: " + config.getDriverStrategy().name());
    }
    this.serializer = this.taskContext.<IT>getInputSerializer(0).getSerializer();
    this.comparator = this.taskContext.getDriverComparator(0);
    this.input = this.taskContext.getInput(0);
   
    this.mutableObjectMode = config.getMutableObjectMode();
   
    if (LOG.isDebugEnabled()) {
      LOG.debug("GroupReduceDriver uses " + (this.mutableObjectMode ? "MUTABLE" : "IMMUTABLE") + " object mode.");
    }
  }
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
   */
  private void initOutputFormat() {
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    this.config = new TaskConfig(taskConf);

    try {
      this.format = config.<OutputFormat<IT>>getStubWrapper(userCodeClassLoader).getUserCodeObject(OutputFormat.class, userCodeClassLoader);

      // check if the class is a subclass, if the check is required
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
  // --------------------------------------------------------------------------------------------

  @Override
  public void prepare() throws Exception {
    TaskConfig config = this.taskContext.getTaskConfig();
    if (config.getDriverStrategy() != DriverStrategy.SORTED_REDUCE) {
      throw new Exception("Unrecognized driver strategy for Reduce driver: " + config.getDriverStrategy().name());
    }
    this.serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
    this.comparator = this.taskContext.getDriverComparator(0);
    this.input = this.taskContext.getInput(0);
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 

  @Override
  public void prepare() throws Exception
  {
    final TaskConfig config = this.taskContext.getTaskConfig();
    if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
      throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
    }
   
    final MutableObjectIterator<IT1> in1 = this.taskContext.getInput(0);
    final MutableObjectIterator<IT2> in2 = this.taskContext.getInput(1);
   
    // get the key positions and types
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
    final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
   
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
          this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
      throw new Exception("Missing pair comparator factory for CoGroup driver");
    }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
      }
    } else {
      throw new RuntimeException("The task context of this driver is no iterative task context.");
    }
   
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
   
    TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
   
    this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
   
    solutionSideRecord = solutionSetSerializer.createInstance();
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
  // --------------------------------------------------------------------------------------------

  @Override
  public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    if (config.getDriverStrategy() != DriverStrategy.ALL_REDUCE) {
      throw new Exception("Unrecognized driver strategy for AllReduce driver: " + config.getDriverStrategy().name());
    }
   
    TypeSerializerFactory<T> serializerFactory = this.taskContext.getInputSerializer(0);
    this.serializer = serializerFactory.getSerializer();
    this.input = this.taskContext.getInput(0);
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    return inputNum == buildSideIndex;
  }

  @Override
  public void initialize() throws Exception {
    TaskConfig config = this.taskContext.getTaskConfig();
   
    TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
    TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
    MutableObjectIterator<IT1> input1 = this.taskContext.getInput(0);
    MutableObjectIterator<IT2> input2 = this.taskContext.getInput(1);

    TypePairComparatorFactory<IT1, IT2> pairComparatorFactory =
        this.taskContext.getTaskConfig().getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());

    double availableMemory = config.getRelativeMemoryDriver();

    if (buildSideIndex == 0 && probeSideIndex == 1) {
     
      matchIterator =
          new BuildFirstReOpenableHashMatchIterator<IT1, IT2, OT>(input1, input2,
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.operators.util.TaskConfig

  • org.apache.flink.api.common.distributions.DataDistribution
  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator
  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator$IterationDescriptor
  • org.apache.flink.configuration.Configuration
  • org.apache.flink.core.memory.InputViewDataInputStreamWrapper
  • org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper
  • org.apache.flink.runtime.iterative.task.AbstractIterativePactTask
  • org.apache.flink.runtime.iterative.task.IterationHeadPactTask
  • org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask
  • org.apache.flink.runtime.jobgraph.InputFormatVertex
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.