Package org.apache.tez.runtime.library.api

Examples of org.apache.tez.runtime.library.api.KeyValueWriter

112
113
114
115
116
117
118
119
120
121
122
      for (Entry<String, String> entry : incrementalConf) {
        jobConf.set(entry.getKey(), entry.getValue());
      }
    }

    KeyValueWriter kvWriter = null;
    if (!(out instanceof OnFileSortedOutput)) {
      kvWriter = ((MROutput)out).getWriter();
    } else {
      kvWriter = ((OnFileSortedOutput)out).getWriter();
    }
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
141
    List<Event> events = null;

    events = kvOutput.initialize(outputContext);
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);
View Full Code Here
228
229
230
231
232
233
234
235
236
237
238
        "-" + taskNumberFormat.format(outputContext.getTaskIndex());
  }

  @Override
  public KeyValueWriter getWriter() throws IOException {
    return new KeyValueWriter() {
      private final boolean useNewWriter = useNewApi;

      @SuppressWarnings("unchecked")
      @Override
      public void write(Object key, Object value) throws IOException {
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
      throw new IOException("Illegal input to reduce: " + in.getClass());
    }
    ShuffledMergedInputLegacy shuffleInput = (ShuffledMergedInputLegacy)in;
    KeyValuesReader kvReader = shuffleInput.getReader();

    KeyValueWriter kvWriter = null;
    if((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy) out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)) {
      kvWriter = ((OnFileSortedOutput) out).getWriter();
    } else {
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
      for (Entry<String, String> entry : incrementalConf) {
        jobConf.set(entry.getKey(), entry.getValue());
      }
    }

    KeyValueWriter kvWriter = null;
    if ((out instanceof MROutputLegacy)) {
      kvWriter = ((MROutputLegacy)out).getWriter();
    } else if ((out instanceof OnFileSortedOutput)){
      kvWriter = ((OnFileSortedOutput)out).getWriter();
    } else {
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
      Preconditions.checkArgument(outputs.size() == (inUnion ? 2 : 1));
      Preconditions.checkArgument(outputs.containsKey("checker"));
      MRInput input = (MRInput) inputs.values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      OnFileSortedOutput output = (OnFileSortedOutput) outputs.get("checker");
      KeyValueWriter kvWriter = output.getWriter();
      MROutput parts = null;
      KeyValueWriter partsWriter = null;
      if (inUnion) {
        parts = (MROutput) outputs.get("parts");
        partsWriter = parts.getWriter();
      }
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
          if (inUnion) {
            partsWriter.write(word, one);
          }
        }
      }
      if (inUnion) {
        if (parts.isCommitRequired()) {
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
      for (LogicalOutput output : outputs.values()) {
        output.start();
      }
      MROutput out = (MROutput) outputs.get("union");
      MROutput allParts = (MROutput) outputs.get("all-parts");
      KeyValueWriter kvWriter = out.getWriter();
      KeyValueWriter partsWriter = allParts.getWriter();
      Map<String, AtomicInteger> unionKv = Maps.newHashMap();
      LogicalInput union = inputs.get("union");
      KeyValuesReader kvReader = (KeyValuesReader) union.getReader();
      while (kvReader.next()) {
        String word = ((Text) kvReader.getCurrentKey()).toString();
        IntWritable intVal = (IntWritable) kvReader.getCurrentValues().iterator().next();
        for (int i=0; i<intVal.get(); ++i) {
          partsWriter.write(word, one);
        }
        AtomicInteger value = unionKv.get(word);
        if (value == null) {
          unionKv.put(word, new AtomicInteger(intVal.get()));
        } else {
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
      Preconditions.checkArgument(inputs.size() == 1);
      Preconditions.checkArgument(outputs.size() == 1);
      MRInput input = (MRInput) inputs.values().iterator().next();
      KeyValueReader kvReader = input.getReader();
      OnFileSortedOutput output = (OnFileSortedOutput) outputs.values().iterator().next();
      KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
      while (kvReader.next()) {
        StringTokenizer itr = new StringTokenizer(kvReader.getCurrentValue().toString());
        while (itr.hasMoreTokens()) {
          word.set(itr.nextToken());
          kvWriter.write(word, one);
        }
      }
    }
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
      for (LogicalOutput output : outputs.values()) {
        output.start();
      }

      MROutput out = (MROutput) outputs.values().iterator().next();
      KeyValueWriter kvWriter = out.getWriter();
      KeyValuesReader kvReader = (KeyValuesReader) inputs.values().iterator().next().getReader();
      while (kvReader.next()) {
        Text word = (Text) kvReader.getCurrentKey();
        int sum = 0;
        for (Object value : kvReader.getCurrentValues()) {
          sum += ((IntWritable) value).get();
        }
        kvWriter.write(word, new IntWritable(sum));
      }
      if (out.isCommitRequired()) {
        while (!context.canCommit()) {
          Thread.sleep(100);
        }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
        srcFile.set(fileName);
      }
    }

    KeyValueReader kvReader = mrInput.getReader();
    KeyValueWriter kvWriter = kvOutput.getWriter();

    while (kvReader.next()) {
      Object key = kvReader.getCurrentKey();
      Object val = kvReader.getCurrentValue();

      Text valText = (Text) val;
      String readVal = valText.toString();
      if (readVal.contains(filterWord)) {
        LongWritable lineNum = (LongWritable) key;
        TextLongPair outVal = new TextLongPair(srcFile, lineNum);
        kvWriter.write(valText, outVal);
      }
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.api.KeyValueWriter

  • org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez
  • org.apache.tez.examples.JoinDataGen$GenDataProcessor
  • org.apache.tez.examples.JoinExample$ForwardingProcessor
  • org.apache.tez.examples.JoinExample$JoinProcessor
  • org.apache.tez.examples.OrderedWordCount$NoOpSorter
  • org.apache.tez.examples.OrderedWordCount$SumProcessor
  • org.apache.tez.examples.WordCount$SumProcessor
  • org.apache.tez.examples.WordCount$TokenProcessor
  • org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample$InputProcessor
  • org.apache.tez.mapreduce.examples.processor.FilterByWordInputProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.