Examples of org.apache.tez.runtime.api.ObjectRegistry

org.apache.tez.runtime.api.ObjectRegistry
A simple shared object registry to cache objects in the memory of the container running the task. This registry can be used to avoid re-creating objects like common hash-tables, dictionaries etc. and provide performance improvements. The cached objects have different life-cycles. If a task adds an object to the cache with Vertex life-cycle then that object is in the cache while the Vertex (to which the task belongs) is running. DAG life-cycle is while the DAG (to which that task belongs) is running. Session life-cycle is while the session (to which that task belongs) is running.
This interface is not supposed to be implemented by users.


  }


  @Test
  public void testBasicCRUD() {
    ObjectRegistry objectRegistry = new ObjectRegistryImpl();
    testCRUD(objectRegistry);
  }

View Full Code Here

    testCRUD(objectRegistry);
  }


  @Test
  public void testClearCache() {
    ObjectRegistry objectRegistry = new ObjectRegistryImpl();
    testCRUD(objectRegistry);


    String one = "one";
    String two = "two";
    objectRegistry.cacheForVertex(one, one);
    objectRegistry.cacheForDAG(two, two);


    ((ObjectRegistryImpl)objectRegistry).clearCache(ObjectRegistryImpl.ObjectLifeCycle.VERTEX);
    Assert.assertNull(objectRegistry.get(one));
    Assert.assertNotNull(objectRegistry.get(two));


    objectRegistry.cacheForVertex(one, one);
    ((ObjectRegistryImpl)objectRegistry).clearCache(ObjectRegistryImpl.ObjectLifeCycle.DAG);
    Assert.assertNotNull(objectRegistry.get(one));
    Assert.assertNull(objectRegistry.get(two));
  }

View Full Code Here

      kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
      ByteBuffer userPayload = getContext().getUserPayload().getPayload();
      if (userPayload != null) {
        boolean doLocalityCheck = getContext().getUserPayload().getPayload().get(0) > 0 ? true : false;
        if (doLocalityCheck) {
          ObjectRegistry objectRegistry = getContext().getObjectRegistry();
          String entry = String.valueOf(getContext().getTaskIndex());
          objectRegistry.cacheForDAG(entry, entry);
        }
      }
    }

View Full Code Here

      System.out.println("Index: " + getContext().getTaskIndex() + 
          " sum: " + sum + " expectedSum: " + expectedSum + " broadcastSum: " + broadcastSum);
      Preconditions.checkState((sum == expectedSum), "Sum = " + sum);      
      
      if (doLocalityCheck) {
        ObjectRegistry objectRegistry = getContext().getObjectRegistry();
        String index = (String) objectRegistry.get(String.valueOf(getContext().getTaskIndex()));
        if (index == null || Integer.valueOf(index).intValue() != getContext().getTaskIndex()) {
          String msg = "Did not find expected local producer "
              + getContext().getTaskIndex() + " in the same JVM";
          System.out.println(msg);
          throw new TezUncheckedException(msg);

View Full Code Here

TOP

Related Classes of org.apache.tez.runtime.api.ObjectRegistry

org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample$InputProcessor

org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample$OneToOneProcessor

org.apache.tez.runtime.common.objectregistry.TestObjectRegistry

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.