Package org.apache.flink.api.java.record.operators

Examples of org.apache.flink.api.java.record.operators.FileDataSink

75
76
77
78
79
80
81
82
83
84
85
86
87
88
    // init failing map task
    MapOperator testMapper = MapOperator.builder(FailingMapper.class).build();

    // init data sink
    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);

    // compose failing program
    output.setInput(testMapper);
    testMapper.setInput(input);

    // generate plan
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(DOP);
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    // init (working) map task
    MapOperator testMapper = MapOperator.builder(TestMapper.class).build();

    // init data sink
    FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath);

    // compose working program
    output.setInput(testMapper);
    testMapper.setInput(input);

    // generate plan
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    } else {
      testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
          config.getString("CrossTest#ShipStrategy", ""));
    }

    FileDataSink output = new FileDataSink(
        new ContractITCaseOutputFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testCross);
    testCross.setFirstInput(input_left);
    testCross.setSecondInput(input_right);

    return new Plan(output);
  }
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
   
    iteration.setNextWorkset(minAndUpdate);
    iteration.setSolutionSetDelta(minAndUpdate);

    // create DataSinkContract for writing the new cluster positions
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, iteration, "Result");
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(LongValue.class, 0)
      .field(LongValue.class, 1);
View Full Code Here
315
316
317
318
319
320
321
322
323
324
325
326
      .input2(filterVisits)
      .name("Antijoin DocsVisits")
      .build();

    // Create DataSinkContract for writing the result of the OLAP query
    FileDataSink result = new FileDataSink(new CsvOutputFormat(), output, antiJoinVisits, "Result");
    result.setDegreeOfParallelism(numSubTasks);
    CsvOutputFormat.configureRecordFormat(result)
      .recordDelimiter('\n')
      .fieldDelimiter('|')
      .lenient(true)
      .field(IntValue.class, 1)
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
      .keyField(IntValue.class, 1)
      .input(edgeCounter)
      .name("Join Counts")
      .build();

    FileDataSink triangles = new FileDataSink(new EdgeWithDegreesOutputFormat(), output, countJoiner, "Unique Edges With Degrees");

    Plan p = new Plan(triangles, "Normalize Edges and compute Vertex Degrees");
    p.setDefaultParallelism(numSubTasks);
    return p;
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
        .name("Compute sum (Reduce)")
        .build();
   
    iteration.setNextPartialSolution(sumReduce);

    @SuppressWarnings("unchecked")
    FileDataSink finalResult = new FileDataSink(new CsvOutputFormat("\n",  " ", StringValue.class), output, iteration, "Output");

    Plan plan = new Plan(finalResult, "Iteration with AllReducer (keyless Reducer)");
    plan.setDefaultParallelism(numSubTasks);
    return plan;
  }
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    testCoGrouper.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
        config.getString("CoGroupTest#LocalStrategy", ""));
    testCoGrouper.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
        config.getString("CoGroupTest#ShipStrategy", ""));

    FileDataSink output = new FileDataSink(new CoGroupOutFormat(), resultPath);
    output.setDegreeOfParallelism(1);

    output.setInput(testCoGrouper);
    testCoGrouper.setFirstInput(input_left);
    testCoGrouper.setSecondInput(input_right);

    return new Plan(output);
  }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
        .name("Compute termination criterion (Map)")
        .build();
   
    iteration.setTerminationCriterion(terminationMapper);

    FileDataSink finalResult = new FileDataSink(CsvOutputFormat.class, output, iteration, "Output");
    CsvOutputFormat.configureRecordFormat(finalResult)
      .recordDelimiter('\n')
      .fieldDelimiter(' ')
      .field(StringValue.class, 0);
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      ReduceOperator counts = ReduceOperator.builder(CountWords.class, StringValue.class, 0)
        .input(wordsFirstInput, wordsSecondInput)
        .name("Word Counts")
        .build();

      FileDataSink sink = new FileDataSink(CsvOutputFormat.class, outputPath, counts);
      CsvOutputFormat.configureRecordFormat(sink)
        .recordDelimiter('\n')
        .fieldDelimiter(' ')
        .field(StringValue.class, 0)
        .field(IntValue.class, 1);
View Full Code Here
TOP

Related Classes of org.apache.flink.api.java.record.operators.FileDataSink

  • org.apache.flink.addons.hbase.example.HBaseReadExample
  • org.apache.flink.api.common.typeinfo.NothingTypeInfo
  • org.apache.flink.api.java.typeutils.RecordTypeInfo
  • org.apache.flink.compiler.AdditionalOperatorsTest
  • org.apache.flink.compiler.BranchingPlansCompilerTest
  • org.apache.flink.compiler.DOPChangeTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.HardPlansCompilationTest
  • org.apache.flink.compiler.ReduceAllTest
  • org.apache.flink.compiler.UnionPropertyPropagationTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.