Examples of org.apache.hadoop.io.Writable

org.apache.hadoop.io.Writable
A simple, efficient, serialization protocol, based on {@link DataInput} and{@link DataOutput}.
Implementations typically implement a static read(DataInput) method which constructs a new instance, calls {@link #readFields(DataInput)}, and returns the instance.

    if (doPipe_ && outThread_ == null) {
      startOutputThreads(output, reporter);
    }
    try {
      while (values.hasNext()) {
        Writable val = (Writable) values.next();
        numRecRead_++;
        maybeLogRecord();
        if (doPipe_) {
          if (outerrThreadsThrowable != null) {
            mapRedFinished();

View Full Code Here

        // seek back to the beginning of mapfile
        r.reset();
        // get the first and last keys
        HStoreKey firstKey = new HStoreKey();
        HStoreKey lastKey = new HStoreKey();
        Writable value = new ImmutableBytesWritable();
        r.next(firstKey, value);
        r.finalKey(lastKey);
        // get the midkey
        HStoreKey mk = (HStoreKey)r.midKey();
        if (mk != null) {

View Full Code Here

          numEdits++;
          switch (opcode) {
          case OP_ADD: {
            UTF8 name = new UTF8();
            ArrayWritable aw = null;
            Writable writables[];
            // version 0 does not support per file replication
            if (logVersion >= 0)
              name.readFields(in);  // read name only
            else {  // other versions do
              // get name and replication

View Full Code Here

                  Reporter reporter)
    throws IOException {
    try {
      // allocate key & value instances that are re-used for all entries
      WritableComparable key = input.createKey();
      Writable value = input.createValue();
      
      while (input.next(key, value)) {
        // map pair to output
        mapper.map(key, value, output, reporter);
      }

View Full Code Here

        reporter.incrCounter(COMBINE_OUTPUT_RECORDS, 1);
      }
    }
    
    private void spill(RawKeyValueIterator resultIter) throws IOException {
      Writable key = null;
      Writable value = null;


      try {
        key = (WritableComparable)ReflectionUtils.newInstance(keyClass, job);
        value = (Writable)ReflectionUtils.newInstance(valClass, job);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }


      DataInputBuffer keyIn = new DataInputBuffer();
      DataInputBuffer valIn = new DataInputBuffer();
      DataOutputBuffer valOut = new DataOutputBuffer();
      while (resultIter.next()) {
        keyIn.reset(resultIter.getKey().getData(), 
                    resultIter.getKey().getLength());
        key.readFields(keyIn);
        valOut.reset();
        (resultIter.getValue()).writeUncompressedBytes(valOut);
        valIn.reset(valOut.getData(), valOut.getLength());
        value.readFields(valIn);


        writer.append(key, value);
      }
    }

View Full Code Here

      }


      @Override
      public void writeToBlock(DataOutput out) throws IOException {
        bfw.getMetaWriter().write(out);
        Writable dataWriter = bfw.getDataWriter();
        if (dataWriter != null)
          dataWriter.write(out);
      }
    });
  }

View Full Code Here

   */
  @Override
  public void addGeneralBloomFilter(BloomFilterWriter bfw) {
    appendMetaBlock(BLOOM_FILTER_META_KEY,
        bfw.getMetaWriter());
    Writable dataWriter = bfw.getDataWriter();
    if (dataWriter != null) {
      appendMetaBlock(BLOOM_FILTER_DATA_KEY, dataWriter);
    }
  }

View Full Code Here

          boolean isError = in.readBoolean();     // read if error
          if (isError) {
            call.setResult(null, WritableUtils.readString(in),
                           WritableUtils.readString(in));
          } else {
            Writable value = (Writable)ReflectionUtils.newInstance(valueClass, conf);
            try {
              readingCall = call;
              value.readFields(in);                 // read value
            } finally {
              readingCall = null;
            }
            call.setResult(value, null, null);
          }

View Full Code Here

    throws IOException {
    try {
      // allocate key & value instances these objects will not be reused
      // because execution of Mapper.map is not serialized.
      WritableComparable key = input.createKey();
      Writable value = input.createValue();


      while (input.next(key, value)) {


        // Run Mapper.map execution asynchronously in a separate thread.
        // If threads are not available from the thread-pool this method

View Full Code Here


    /** The current key. */
    public WritableComparable getKey() { return key; }


    private void getNext() throws IOException {
      Writable lastKey = key;                     // save previous key
      try {
        key = (WritableComparable)ReflectionUtils.newInstance(keyClass, this.conf);
        value = (Writable)ReflectionUtils.newInstance(valClass, this.conf);
      } catch (Exception e) {
        throw new RuntimeException(e);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.Writable

co.nubetech.hiho.dedup.TestDedupJob

co.nubetech.hiho.merge.TestMergeJob

com.asakusafw.testtools.TestDataHolder

com.esri.hadoop.hive.serde.JsonSerde

com.facebook.giraph.hive.impl.output.HiveApiRecordWriter

com.facebook.hive.orc.TestInputOutputFormat

com.facebook.hiveio.output.RecordWriterImpl

com.facebook.presto.hive.AbstractTestHiveFileFormats

com.sap.hadoop.ds.list.ByteBasedListTest

com.sap.hadoop.ds.list.PartitionedByteBasedListTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.