Examples of Counters

com.google.appengine.tools.mapreduce.Counters
Collection of all counters.
com.tek42.perforce.parse.Counters
Base API object for interacting with counters. @author Kamlesh Sangani
fr.neatmonster.nocheatplus.stats.Counters
Utility to count things, set up to get input from the primary server thread, as well as from other threads. Consequently summaries for both are only available from the primary thread. @author dev1mc
mage.counters.Counters
@author BetaSteward_at_googlemail.com
org.apache.hadoop.mapred.Counters
A set of named counters.
Counters represent global counters, defined either by the Map-Reduce framework or applications. Each Counter can be of any {@link Enum} type.
Counters are bunched into {@link Group}s, each comprising of counters from a particular Enum class.
org.apache.hadoop.mapreduce.Counters
org.apache.hadoop.mapreduce.v2.api.records.Counters
org.apache.hadoop.raid.Statistics.Counters
org.apache.hama.bsp.Counters
org.apache.sqoop.submission.counter.Counters

Examples of org.apache.hadoop.mapreduce.Counters

    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
    values.add(hihoValue1);
    values.add(hihoValue2);


    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value2);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT)

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

    return RepeatStatus.FINISHED;
  }


  @SuppressWarnings("deprecation")
  private void saveCounters(Job job, StepContribution contribution) {
    Counters counters = null;
    try {
      counters = job.getCounters();
    } catch (Exception ex) {
      if (RuntimeException.class.isAssignableFrom(ex.getClass())) {
        throw (RuntimeException)ex;
      } else {
        // ignore - we just can't get stats
      }
    }
    if (counters == null) {
      return;
    }


    // TODO: remove deprecation suppress when we don't want to rely on org.apache.hadoop.mapred
    Counter count = counters.findCounter(Task.Counter.MAP_INPUT_RECORDS);


    for (int i = 0; i < safeLongToInt(count.getValue()); i++) {
      contribution.incrementReadCount();
    }


    count = counters.findCounter(Task.Counter.MAP_SKIPPED_RECORDS);
    contribution.incrementReadSkipCount(safeLongToInt(count.getValue()));


    count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS);
    contribution.incrementWriteCount(safeLongToInt(count.getValue()));


    count = counters.findCounter(Task.Counter.REDUCE_SKIPPED_RECORDS);


    for (int i = 0; i < safeLongToInt(count.getValue()); i++) {
      contribution.incrementWriteSkipCount();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

              .newInstance(job.getOutputFormatClass(),
                  job.getConfiguration()).getClass()
              .getName());
      job.waitForCompletion(false);
      if (job.isComplete()) {
        Counters counters = job.getCounters();
        totalRecordsRead = counters.findCounter(
            DedupRecordCounter.TOTAL_RECORDS_READ).getValue();
        badRecords = counters.findCounter(
            DedupRecordCounter.BAD_RECORD).getValue();
        output = counters.findCounter(DedupRecordCounter.OUTPUT)
            .getValue();
        duplicateRecords = totalRecordsRead - output;
        logger.info("Total records read are: " + totalRecordsRead);
        logger.info("Bad Records are: " + badRecords);
        logger.info("Output records are: " + output);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

  private Mapper<K1, V1, K2, V2> myMapper;
  private Counters counters;


  public MapDriver(final Mapper<K1, V1, K2, V2> m) {
    myMapper = m;
    counters = new Counters();
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

    myMapper = m;
    counters = new Counters();
  }


  public MapDriver() {
    counters = new Counters();
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

  private Reducer<K1, V1, K2, V2> myReducer;
  private Counters counters;


  public ReduceDriver(final Reducer<K1, V1, K2, V2> r) {
    myReducer = r;
    counters = new Counters();
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

    myReducer = r;
    counters = new Counters();
  }


  public ReduceDriver() {
    counters = new Counters();
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

            }
          }


          @Override
          public boolean dump(Job job) {
              Counters counters;
            try {
              counters = job.getCounters();
                    long wsize=counters.findCounter("higo", "dumpcount").getValue();
                    if(wsize>0)
                    {
                      try {
                  TableJoin.LOG.info("update "+tableName+ ",dump");
                  TableJoin.updatePercent(tableName, "Stage-2 map = 100%,  reduce = 100%", "DUMP")  ;

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

          || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) {
        this.mayBeConstructFinalFullCounters();
        return fullCounters;
      }


      Counters counters = new Counters();
      counters.incrAllCounters(jobCounters);
      return incrTaskCounters(counters, tasks.values());


    } finally {
      readLock.unlock();
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.Counters

    }
  }


  @Private
  public void constructFinalFullcounters() {
    this.fullCounters = new Counters();
    this.finalMapCounters = new Counters();
    this.finalReduceCounters = new Counters();
    this.fullCounters.incrAllCounters(jobCounters);
    for (Task t : this.tasks.values()) {
      Counters counters = t.getCounters();
      switch (t.getType()) {
      case MAP:
        this.finalMapCounters.incrAllCounters(counters);
        break;
      case REDUCE:

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.