Examples of SerDe

com.netflix.suro.message.SerDe
SerDe(Serialization/Deserialization) is necessary for converting POJO to byte[] vice versa. serialize() receives POJO and returns byte[], deserialize does reverse. For Java String, StringSerDe is implemented. @author jbae @param < T>
org.apache.crunch.impl.spark.serde.SerDe
org.apache.hadoop.hive.serde2.SerDe
A union of HiveDeserializer and HiveSerializer interface. If a developer wants his hive table to be read-only, then he just want to return both readable and writable, then

Examples of org.apache.hadoop.hive.serde2.SerDe

      boolean noFilter) {
    TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter);
    if (desc == null) {
      return null;
    }
    SerDe sd = (SerDe) ReflectionUtils.newInstance(desc.getDeserializerClass(),
        null);
    try {
      sd.initialize(null, desc.getProperties());
    } catch (SerDeException e) {
      e.printStackTrace();
      return null;
    }
    return sd;

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe

      List<ObjectInspector> structFieldObjectInspectors,
      Byte alias,int containerSize, Map<Byte, TableDesc> spillTableDesc,
      JoinDesc conf,boolean noFilter, Reporter reporter) throws HiveException {


    TableDesc tblDesc = JoinUtil.getSpillTableDesc(alias,spillTableDesc,conf, noFilter);
    SerDe serde = JoinUtil.getSpillSerDe(alias, spillTableDesc, conf, noFilter);


    if (serde == null) {
      containerSize = -1;
    }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe


  public void generateMapMetaData() throws HiveException, SerDeException {
    // generate the meta data for key
    // index for key is -1
    TableDesc keyTableDesc = conf.getKeyTblDesc();
    SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
        null);
    keySerializer.initialize(null, keyTableDesc.getProperties());
    MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
        ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf));


    // index for values is just alias
    for (int tag = 0; tag < order.length; tag++) {
      int alias = (int) order[tag];


      if (alias == this.bigTableAlias) {
        continue;
      }




      TableDesc valueTableDesc;
      if (conf.getNoOuterJoin()) {
        valueTableDesc = conf.getValueTblDescs().get(tag);
      } else {
        valueTableDesc = conf.getValueFilteredTblDescs().get(tag);
      }
      SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(),
          null);
      valueSerDe.initialize(null, valueTableDesc.getProperties());


      MapJoinMetaData.put(Integer.valueOf(alias), new HashTableSinkObjectCtx(ObjectInspectorUtils
          .getStandardObjectInspector(valueSerDe.getObjectInspector(),
              ObjectInspectorCopyOption.WRITABLE), valueSerDe, valueTableDesc, hconf));
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe

          .getProperties());
      StructObjectInspector structTblKeyInpector = ObjectInspectorFactory
          .getStandardStructObjectInspector(keyColNames, skewTableKeyInspectors);


      try {
        SerDe serializer = (SerDe) ReflectionUtils.newInstance(tblDesc.get(
            alias).getDeserializerClass(), null);
        serializer.initialize(null, tblDesc.get(alias).getProperties());
        tblSerializers.put((byte) i, serializer);
      } catch (SerDeException e) {
        LOG.error("Skewjoin will be disabled due to " + e.getMessage(), e);
        joinOp.handleSkewJoin = false;
        break;

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe


  }


  private void setKeyMetaData() throws SerDeException {
    TableDesc keyTableDesc = conf.getKeyTblDesc();
    SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
        null);
    keySerializer.initialize(null, keyTableDesc.getProperties());


    MapJoinMetaData.clear();
    MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
        ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf));
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe


  }


  private void setValueMetaData(int tag) throws SerDeException {
    TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(tag);
    SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(),
        null);


    valueSerDe.initialize(null, valueTableDesc.getProperties());


    List<ObjectInspector> newFields = rowContainerStandardObjectInspectors.get((Byte) alias);
    int length = newFields.size();
    List<String> newNames = new ArrayList<String>(length);
    for (int i = 0; i < length; i++) {

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe

  }


  RowContainer getRowContainer(Configuration hconf, byte pos, Byte alias,
      int containerSize) throws HiveException {
    TableDesc tblDesc = getSpillTableDesc(alias);
    SerDe serde = getSpillSerDe(alias);


    if (serde == null) {
      containerSize = 1;
    }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe

  private SerDe getSpillSerDe(byte alias) {
    TableDesc desc = getSpillTableDesc(alias);
    if (desc == null) {
      return null;
    }
    SerDe sd = (SerDe) ReflectionUtils.newInstance(desc.getDeserializerClass(),
        null);
    try {
      sd.initialize(null, desc.getProperties());
    } catch (SerDeException e) {
      e.printStackTrace();
      return null;
    }
    return sd;

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe

    {
        JobConf jobConf = new JobConf();
        RCFileOutputFormat outputFormat = new RCFileOutputFormat();
        @SuppressWarnings("rawtypes")
        RCFileInputFormat inputFormat = new RCFileInputFormat();
        @SuppressWarnings("deprecation")
        SerDe serde = new ColumnarSerDe();
        File file = File.createTempFile("presto_test", "rc-text");
        try {
            FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null);
            @SuppressWarnings("unchecked")

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.SerDe

    {
        JobConf jobConf = new JobConf();
        RCFileOutputFormat outputFormat = new RCFileOutputFormat();
        @SuppressWarnings("rawtypes")
        RCFileInputFormat inputFormat = new RCFileInputFormat();
        @SuppressWarnings("deprecation")
        SerDe serde = new LazyBinaryColumnarSerDe();
        File file = File.createTempFile("presto_test", "rc-binary");
        try {
            FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null);
            @SuppressWarnings("unchecked")

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.