Examples of DataInputBuffer

org.apache.cassandra.io.DataInputBuffer
An implementation of the DataInputStream interface. This instance is completely thread unsafe. Author : Avinash Lakshman ( alakshman@facebook.com) & Prashant Malik ( pmalik@facebook.com )
org.apache.hadoop.io.DataInputBuffer
A reusable {@link DataInput} implementation that reads from an in-memorybuffer.
This saves memory over creating a new DataInputStream and ByteArrayInputStream each time data is read.
Typical usage is something like the following:
```
 DataInputBuffer buffer = new DataInputBuffer(); while (... loop condition ...) { byte[] data = ... get data ...; int dataLength = ... get data length ...; buffer.reset(data, dataLength); ... read buffer using DataInput methods ... } 
```

Examples of org.apache.hadoop.io.DataInputBuffer

            long segmentStart = out.getPos();
            writer = new Writer<K, V>(job, out, keyClass, valClass, codec,
                                      spilledRecordsCounter);
            if (combinerRunner == null) {
              // spill directly
              DataInputBuffer key = new DataInputBuffer();
              while (spindex < endPosition &&
                  kvindices[kvoffsets[spindex % kvoffsets.length]
                            + PARTITION] == i) {
                final int kvoff = kvoffsets[spindex % kvoffsets.length];
                getVBytesForOffset(kvoff, value);
                key.reset(kvbuffer, kvindices[kvoff + KEYSTART],
                          (kvindices[kvoff + VALSTART] - 
                           kvindices[kvoff + KEYSTART]));
                writer.append(key, value);
                ++spindex;
              }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

      }


      public boolean next(DataInputBuffer key, DataInputBuffer value)
          throws IOException {
        if (kvIter.next()) {
          final DataInputBuffer kb = kvIter.getKey();
          final DataInputBuffer vb = kvIter.getValue();
          final int kp = kb.getPosition();
          final int klen = kb.getLength() - kp;
          key.reset(kb.getData(), kp, klen);
          final int vp = vb.getPosition();
          final int vlen = vb.getLength() - vp;
          value.reset(vb.getData(), vp, vlen);
          bytesRead += klen + vlen;
          return true;
        }
        return false;
      }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

          );
        }


        // TODO: Remove this after a 'fix' for HADOOP-3647
        if (mapOutputLength > 0) {
          DataInputBuffer dib = new DataInputBuffer();
          dib.reset(shuffleData, 0, shuffleData.length);
          LOG.info("Rec #1 from " + mapOutputLoc.getTaskAttemptId() + " -> (" + 
                   WritableUtils.readVInt(dib) + ", " + 
                   WritableUtils.readVInt(dib) + ") from " + 
                   mapOutputLoc.getHost());
        }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

    deflateOut.flush();
    deflateFilter.finish();
    LOG.info("Finished compressing data");
    
    // De-compress data
    DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
    deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0, 
                                 compressedDataBuffer.getLength());
    CompressionInputStream inflateFilter = 
      codec.createInputStream(deCompressedDataBuffer);
    DataInputStream inflateIn = 
      new DataInputStream(new BufferedInputStream(inflateFilter));


    // Check
    DataInputBuffer originalData = new DataInputBuffer();
    originalData.reset(data.getData(), 0, data.getLength());
    DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData));
    for(int i=0; i < count; ++i) {
      RandomDatum k1 = new RandomDatum();
      RandomDatum v1 = new RandomDatum();
      k1.readFields(originalIn);
      v1.readFields(originalIn);
      
      RandomDatum k2 = new RandomDatum();
      RandomDatum v2 = new RandomDatum();
      k2.readFields(inflateIn);
      v2.readFields(inflateIn);
    }


    // De-compress data byte-at-a-time
    originalData.reset(data.getData(), 0, data.getLength());
    deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0, 
                                 compressedDataBuffer.getLength());
    inflateFilter = 
      codec.createInputStream(deCompressedDataBuffer);

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

      r = new SequenceFile.Reader(fs, f.getPath(), getConf());
      key = ReflectionUtils.newInstance(r.getKeyClass().asSubclass(WritableComparable.class),
                                        getConf());
      val = ReflectionUtils.newInstance(r.getValueClass().asSubclass(Writable.class),
                                        getConf());
      inbuf = new DataInputBuffer();
      outbuf = new DataOutputBuffer();
    }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

   * @throws IOException
   */
  private static void decodeWritable(Writable obj, 
                                     String newValue) throws IOException {
    Base64 decoder = new Base64(0, null, true);
    DataInputBuffer buf = new DataInputBuffer();
    byte[] decoded = decoder.decode(newValue);
    buf.reset(decoded, decoded.length);
    obj.readFields(buf);
  }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

    public void stress_read() throws Exception {
        int count = 10000000;
        DecimalOption value = new DecimalOption(new BigDecimal("3.14"));
        byte[] bytes = toBytes(value);
        DecimalOption buf = new DecimalOption();
        DataInputBuffer in = new DataInputBuffer();
        for (int i = 0; i < count; i++) {
            in.reset(bytes, bytes.length);
            buf.readFields(in);
            if (i == 0) {
                assertThat(buf, is(value));
            }
        }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

    public void stress_read_large() throws Exception {
        int count = 10000000;
        DecimalOption value = new DecimalOption(new BigDecimal(Long.MAX_VALUE).multiply(BigDecimal.TEN));
        byte[] bytes = toBytes(value);
        DecimalOption buf = new DecimalOption();
        DataInputBuffer in = new DataInputBuffer();
        for (int i = 0; i < count; i++) {
            in.reset(bytes, bytes.length);
            buf.readFields(in);
            if (i == 0) {
                assertThat(buf, is(value));
            }
        }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer


        DataOutputBuffer output = new DataOutputBuffer();
        writable.write(output);


        Writable copy = (Writable) loader.newModel("Primitives").unwrap();
        DataInputBuffer input = new DataInputBuffer();
        input.reset(output.getData(), output.getLength());
        copy.readFields(input);


        assertThat(input.read(), is(-1));
        assertThat(writable, equalTo(copy));
    }

View Full Code Here

Examples of org.apache.hadoop.io.DataInputBuffer

        byte[] results = Arrays.copyOfRange(out.getData(), 0, out.getLength());
        return results;
    }


    static <T extends Writable> T des(T writable, byte[] serialized) throws IOException {
        DataInputBuffer buf = new DataInputBuffer();
        buf.reset(serialized, serialized.length);
        writable.readFields(buf);
        return writable;
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.