Package eu.stratosphere.types

Examples of eu.stratosphere.types.Record$InternalDeSerializer


  @Test
  public void testNextKeyOnly() throws Exception
  {
    try {
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
      Assert.assertNull("KeyGroupedIterator must not have another value.", this.psi.getValues());
     
View Full Code Here


  {
    try {
      // Key 1, Value A
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
     
      // Key 2, Value B
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
     
      // Key 3, Values C, D
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
      catch (NoSuchElementException nseex) {}
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
      catch (NoSuchElementException nseex) {}
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      // Key 4, Values E, F, G
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("F"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("G"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      // Key 5, Values H, I, J, K, L
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("J"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("K"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("L"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
      catch (NoSuchElementException nseex) {}
      Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      try {
        this.psi.getValues().next();
        Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
      }
View Full Code Here

      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
     
      // Progression first though haNext() and next(), then through hasNext() - Key 3, Values C, D
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
     
      // Progression first via next() only, then hasNext() only Key 4, Values E, F, G
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
     
      // Key 5, Values H, I, J, K, L
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
      Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
     
      // end
View Full Code Here

  @Test
  public void testHasNextDoesNotOverweiteCurrentRecord() throws Exception
  {
    try {
      Iterator<Record> valsIter = null;
      Record rec = null;
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      valsIter = this.psi.getValues();
      Assert.assertNotNull("Returned Iterator must not be null", valsIter);
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));     
      Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      valsIter = this.psi.getValues();
      Assert.assertNotNull("Returned Iterator must not be null", valsIter);
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
     
      Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
      valsIter = this.psi.getValues();
      Assert.assertNotNull("Returned Iterator must not be null", valsIter);
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
      Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
      rec = valsIter.next();
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
      Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
      Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
      Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail("The test encountered an unexpected exception.");
    }
  }
View Full Code Here

   
    Sorter<Record> merger = new CombiningUnilateralSortMerger<Record>(comb,
        this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory, this.comparator,
        64 * 1024 * 1024, 64, 0.7f);

    final Record rec = new Record();
    rec.setField(1, new IntValue(1));
    final TestData.Key key = new TestData.Key();
   
    for (int i = 0; i < noKeyCnt; i++) {
      for (int j = 0; j < noKeys; j++) {
        key.setKey(j);
        rec.setField(0, key);
        reader.emit(rec);
      }
    }
    reader.close();
   
View Full Code Here

   
    Sorter<Record> merger = new CombiningUnilateralSortMerger<Record>(comb,
        this.memoryManager, this.ioManager, reader, this.parentTask, this.serializerFactory, this.comparator,
        3 * 1024 * 1024, 64, 0.005f);

    final Record rec = new Record();
    rec.setField(1, new IntValue(1));
    final TestData.Key key = new TestData.Key();
   
    for (int i = 0; i < noKeyCnt; i++) {
      for (int j = 0; j < noKeys; j++) {
        key.setKey(j);
        rec.setField(0, key);
        reader.emit(rec);
      }
    }
    reader.close();
   
View Full Code Here

        64 * 1024 * 1024, 2, 0.7f);

    // emit data
    LOG.debug("emitting data");
    TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
    Record rec = new Record();
    final TestData.Value value = new TestData.Value("1");
   
    for (int i = 0; i < NUM_PAIRS; i++) {
      Assert.assertTrue((rec = generator.next(rec)) != null);
      final TestData.Key key = rec.getField(0, TestData.Key.class);
      rec.setField(1, value);
      reader.emit(rec);
     
      countTable.put(new TestData.Key(key.getKey()), countTable.get(key) + 1);
    }
    reader.close();
    rec = null;

    // check order
    MutableObjectIterator<Record> iterator = merger.getIterator();
   
    LOG.debug("checking results");
   
    Record rec1 = new Record();
    Record rec2 = new Record();
   
    Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
    countTable.put(new TestData.Key(rec1.getField(0, TestData.Key.class).getKey()), countTable.get(rec1.getField(0, TestData.Key.class)) - (Integer.parseInt(rec1.getField(1, TestData.Value.class).toString())));

    while ((rec2 = iterator.next(rec2)) != null) {
      final Key k1 = rec1.getField(0, TestData.Key.class);
      final Key k2 = rec2.getField(0, TestData.Key.class);
     
      Assert.assertTrue(keyComparator.compare(k1, k2) <= 0);
      countTable.put(new TestData.Key(k2.getKey()), countTable.get(k2) - (Integer.parseInt(rec2.getField(1, TestData.Value.class).toString())));
     
      Record tmp = rec1;
      rec1 = rec2;
      k1.setKey(k2.getKey());
      rec2 = tmp;
    }

View Full Code Here

        if (hasNext()) {
          hasNext = false;
         
          Iterator<Record> values = groupIter.getValues();
         
          Record rec = null;
          int cnt = 0;
          while (values.hasNext()) {
            rec = values.next();
            cnt += rec.getField(1, IntValue.class).getValue();
          }
         
          return cnt;
        } else {
          throw new NoSuchElementException();
View Full Code Here

    public volatile boolean closed = false;
   
   
    @Override
    public void combine(Iterator<Record> values, Collector<Record> out) {
      Record rec = null;
      int cnt = 0;
      while (values.hasNext()) {
        rec = values.next();
        cnt += rec.getField(1, IntValue.class).getValue();
      }
     
      this.count.setValue(cnt);
      rec.setField(1, this.count);
      out.collect(rec);
    }
View Full Code Here

   
    public volatile boolean closed = false;
   
    @Override
    public void combine(Iterator<Record> values, Collector<Record> out) {
      Record rec = null;
      int cnt = 0;
      while (values.hasNext()) {
        rec = values.next();
        cnt += Integer.parseInt(rec.getField(1, TestData.Value.class).toString());
      }

      out.collect(new Record(rec.getField(0, Key.class), new TestData.Value(cnt + "")));
    }
View Full Code Here

TOP

Related Classes of eu.stratosphere.types.Record$InternalDeSerializer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.