Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.Counters.findCounter()

141
142
143
144
145
146
147
148
149
150
151
    values.add(hihoValue1);
    values.add(hihoValue2);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value2);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT)
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
    values.add(hihoValue1);
    values.add(hihoValue2);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value2);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT)
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    if (counters == null) {
      return;
    }

    // TODO: remove deprecation suppress when we don't want to rely on org.apache.hadoop.mapred
    Counter count = counters.findCounter(Task.Counter.MAP_INPUT_RECORDS);

    for (int i = 0; i < safeLongToInt(count.getValue()); i++) {
      contribution.incrementReadCount();
    }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
    for (int i = 0; i < safeLongToInt(count.getValue()); i++) {
      contribution.incrementReadCount();
    }

    count = counters.findCounter(Task.Counter.MAP_SKIPPED_RECORDS);
    contribution.incrementReadSkipCount(safeLongToInt(count.getValue()));

    count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS);
    contribution.incrementWriteCount(safeLongToInt(count.getValue()));
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
    }

    count = counters.findCounter(Task.Counter.MAP_SKIPPED_RECORDS);
    contribution.incrementReadSkipCount(safeLongToInt(count.getValue()));

    count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS);
    contribution.incrementWriteCount(safeLongToInt(count.getValue()));

    count = counters.findCounter(Task.Counter.REDUCE_SKIPPED_RECORDS);

    for (int i = 0; i < safeLongToInt(count.getValue()); i++) {
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
    contribution.incrementReadSkipCount(safeLongToInt(count.getValue()));

    count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS);
    contribution.incrementWriteCount(safeLongToInt(count.getValue()));

    count = counters.findCounter(Task.Counter.REDUCE_SKIPPED_RECORDS);

    for (int i = 0; i < safeLongToInt(count.getValue()); i++) {
      contribution.incrementWriteSkipCount();
    }
  }
View Full Code Here
204
205
206
207
208
209
210
211
212
213
214
                  job.getConfiguration()).getClass()
              .getName());
      job.waitForCompletion(false);
      if (job.isComplete()) {
        Counters counters = job.getCounters();
        totalRecordsRead = counters.findCounter(
            DedupRecordCounter.TOTAL_RECORDS_READ).getValue();
        badRecords = counters.findCounter(
            DedupRecordCounter.BAD_RECORD).getValue();
        output = counters.findCounter(DedupRecordCounter.OUTPUT)
            .getValue();
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
      job.waitForCompletion(false);
      if (job.isComplete()) {
        Counters counters = job.getCounters();
        totalRecordsRead = counters.findCounter(
            DedupRecordCounter.TOTAL_RECORDS_READ).getValue();
        badRecords = counters.findCounter(
            DedupRecordCounter.BAD_RECORD).getValue();
        output = counters.findCounter(DedupRecordCounter.OUTPUT)
            .getValue();
        duplicateRecords = totalRecordsRead - output;
        logger.info("Total records read are: " + totalRecordsRead);
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
        Counters counters = job.getCounters();
        totalRecordsRead = counters.findCounter(
            DedupRecordCounter.TOTAL_RECORDS_READ).getValue();
        badRecords = counters.findCounter(
            DedupRecordCounter.BAD_RECORD).getValue();
        output = counters.findCounter(DedupRecordCounter.OUTPUT)
            .getValue();
        duplicateRecords = totalRecordsRead - output;
        logger.info("Total records read are: " + totalRecordsRead);
        logger.info("Bad Records are: " + badRecords);
        logger.info("Output records are: " + output);
View Full Code Here
285
286
287
288
289
290
291
292
293
294
295
          @Override
          public boolean dump(Job job) {
              Counters counters;
            try {
              counters = job.getCounters();
                    long wsize=counters.findCounter("higo", "dumpcount").getValue();
                    if(wsize>0)
                    {
                      try {
                  TableJoin.LOG.info("update "+tableName+ ",dump");
                  TableJoin.updatePercent(tableName, "Stage-2 map = 100%,  reduce = 100%", "DUMP")  ;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.