Package com.ebay.erl.mobius.core.datajoin

Examples of com.ebay.erl.mobius.core.datajoin.DataJoinKey$Comparator


            // joinmap function might generate more than one output key
            // per <code>key</code> input.
            for( Tuple t:collector.getOutKey() )
            {
              Tuple mt = Tuple.merge(tuple, t);
              DataJoinKey nkey = this.getKey(mt, sorters, datasetID, mapper, job);
              samples.add(nkey);
            }
          }
          else
          {
            // When exceeding the maximum number of samples, replace
            // a random element with this one, then adjust the
            // frequency to reflect the possibility of existing
            // elements being pushed out
           
            mapper.joinmap(key, value, collector, Reporter.NULL);
            for( Tuple t:collector.getOutKey() )
            {
              int ind = r.nextInt(numSamples);
              if (ind != numSamples)
              {
                Tuple mt = Tuple.merge(tuple, t);
                DataJoinKey nkey = this.getKey(mt, sorters, datasetID, mapper, job);
                samples.set(ind, nkey);
              }
            }
           
            freq *= (numSamples - collector.getOutKey().size()) / (double) numSamples;
View Full Code Here


      String name    = aSorter.getColumn();
      Object value  = tuple.get(name);
      columnsUsedToSort.insert(name, value);
    }
   
    DataJoinKey nkey = new DataJoinKey(datasetID, columnsUsedToSort, mapper.extractSortValueKeyword(tuple), mapper.getSortValueComparator());
    nkey.setConf(conf);
    return nkey;
  }
View Full Code Here

            throw new IllegalArgumentException(p.toString()+" is a group function that generates " +
                "more than one rows ("+aggregatedResult.size()+") per key, so it is not combinable.");
        }
      }
     
      DataJoinKey outKey    = new DataJoinKey(datasetID, key);
      DataJoinValue outValue  = new DataJoinValue(datasetID, combinedValue);
      output.collect(outKey, outValue);
    }
  }
View Full Code Here

*/
public class DataJoinKeyTest {

  @Test
  public void testConstructor() {
    DataJoinKey djKey = new DataJoinKey(Byte.valueOf("1"), new Text("test value"));
   
    Assert.assertEquals("1", djKey.getDatasetID());
    Assert.assertEquals(new Text("test value"), djKey.getKey());
    //Assert.assertEquals(null, djKey.getSortKeyword());
    //Assert.assertEquals(Class.class, djKey.getSortComparator());
   
    djKey = new DataJoinKey(Byte.valueOf("1"), new Text("test value"), new Text("2"), DataJoinKey.Comparator.class);
   
    Assert.assertEquals("1", djKey.getDatasetID());
    Assert.assertEquals(new Text("test value"), djKey.getKey());
    //Assert.assertEquals(new Text("2"), djKey.getSortKeyword());
    //Assert.assertEquals(DataJoinKey.Comparator.class, djKey.getSortComparator());
   
    djKey = new DataJoinKey(Byte.valueOf("1"), new Text("test value"), new Text("2"), null);
   
    Assert.assertEquals(Byte.valueOf("1"), djKey.getDatasetID());
    Assert.assertEquals(new Text("test value"), djKey.getKey());
    //Assert.assertEquals(new Text("2"), djKey.getSortKeyword());
    //Assert.assertEquals(Class.class, djKey.getSortComparator());
  }
View Full Code Here

  }
 
 
  @Test
  public void testCompare() {
    DataJoinKey djKey1 = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey1_1 = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey2 = new DataJoinKey(Byte.valueOf("1"), new Text("2"));
   
    Assert.assertEquals(-1, djKey1.compareTo(djKey2));
    Assert.assertEquals(1, djKey2.compareTo(djKey1));
    Assert.assertEquals(0, djKey1.compareTo(djKey1_1));
   
    Assert.assertEquals(-1, djKey1.compare(djKey1, djKey2));
    Assert.assertEquals(1, djKey1.compare(djKey2, djKey1));
    Assert.assertEquals(0, djKey1.compare(djKey1, djKey1_1));
   
    djKey1    = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("1"), null);
    djKey1_1   = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("2"), null);
    djKey2     = new DataJoinKey(Byte.valueOf("1"), new Text("2"), new Text("1"), null);
   
    Assert.assertEquals(-1, djKey1.compareTo(djKey2));
    Assert.assertEquals(1, djKey2.compareTo(djKey1));
    Assert.assertEquals(0, djKey1.compareTo(djKey1_1));
    Assert.assertEquals(0, djKey1_1.compareTo(djKey1));
   
    Assert.assertEquals(-1, djKey1.compare(djKey1, djKey2));
    Assert.assertEquals(1, djKey1.compare(djKey2, djKey1));
    Assert.assertEquals(0, djKey1.compare(djKey1, djKey1_1));
    Assert.assertEquals(0, djKey1.compare(djKey1_1, djKey1));
   
    djKey1     = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("1"), NegativeComparator.class);
    djKey1_1   = new DataJoinKey(Byte.valueOf("1"), new Text("1"), new Text("2"), NegativeComparator.class);
    djKey2     = new DataJoinKey(Byte.valueOf("1"), new Text("2"), new Text("1"), NegativeComparator.class);
   
    Assert.assertEquals(-1, djKey1.compareTo(djKey2));
    Assert.assertEquals(1, djKey2.compareTo(djKey1));
    Assert.assertEquals(0, djKey1.compareTo(djKey1_1));
    Assert.assertEquals(0, djKey1_1.compareTo(djKey1));
   
    Assert.assertEquals(-1, djKey1.compare(djKey1, djKey2));
    Assert.assertEquals(1, djKey1.compare(djKey2, djKey1));
View Full Code Here

    Assert.assertEquals(0, djKey1.compare(djKey1_1, djKey1));
  }
 
  @Test
  public void testHash() {
    DataJoinKey djKey1     = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey1_1   = new DataJoinKey(Byte.valueOf("1"), new Text("1"));
    DataJoinKey djKey2     = new DataJoinKey(Byte.valueOf("1"), new Text("2"));
   
    Assert.assertEquals(djKey1.hashCode(), djKey1_1.hashCode());
    Assert.assertNotSame(djKey1.hashCode(), djKey2.hashCode());
  }
View Full Code Here

    t1.put("k1", "v1");
   
    Tuple t2 = new Tuple();
    t2.put("k2", "v2");
   
    DataJoinKey k1 = new DataJoinKey(Byte.valueOf("1"), t1);
    DataJoinKey k2 = new DataJoinKey(Byte.valueOf("1"), t2);
   
    ByteArrayOutputStream b1 = new ByteArrayOutputStream();
    ByteArrayOutputStream b2 = new ByteArrayOutputStream();
   
    DataOutputStream out1 = new DataOutputStream(b1);
    DataOutputStream out2 = new DataOutputStream(b2);
   
    k1.write(out1);
    k2.write(out2);
   
    out1.flush();
    out2.flush();
   
    byte[] ba1 = b1.toByteArray();
    byte[] ba2 = b2.toByteArray();
   
    DataInputStream in1 = new DataInputStream(new ByteArrayInputStream(ba1));
    DataInputStream in2 = new DataInputStream(new ByteArrayInputStream(ba2));
   
    DataJoinKey k_1 = new DataJoinKey();
    k_1.readFields(in1);
   
    DataJoinKey k_2 = new DataJoinKey();
    k_2.readFields(in2);
   
    Assert.assertEquals(-1, k_1.compareTo(k_2));
    Assert.assertEquals(-1, k_1.compare(ba1, 0, ba1.length, ba2, 0, ba2.length));
   
  }
View Full Code Here

  @Test
  public void testPartition() {
    DataJoinKeyPartitioner partitioner = new DataJoinKeyPartitioner();
   
    DataJoinKey djKey1 = new DataJoinKey(Byte.valueOf("1"), new Text("test value"));
    DataJoinKey djKey2 = new DataJoinKey(Byte.valueOf("1"), new Text("test value"));
    DataJoinKey djKey3 = new DataJoinKey(Byte.valueOf("1"), new Text("other test value"));
   
    System.out.println(partitioner.getPartition(djKey1, null, 999));
    System.out.println(partitioner.getPartition(djKey2, null, 999));
    System.out.println(partitioner.getPartition(djKey3, null, 999));
   
View Full Code Here

TOP

Related Classes of com.ebay.erl.mobius.core.datajoin.DataJoinKey$Comparator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.