Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator

Package org.apache.hadoop.hive.ql.udf.ptf

Examples of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator

org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator
Based on Hive {@link GenericUDAFEvaluator}. Break up the responsibility of the old AsbtractTableFunction class into a Resolver and Evaluator.
The Evaluator also holds onto the {@link TableFunctionDef}. This provides information about the arguments to the function, the shape of the Input partition and the Partitioning details. The Evaluator is responsible for providing the 2 execute methods:
1. execute: which is invoked after the input is partitioned; the contract is, it is given an input Partition and must return an output Partition. The shape of the output Partition is obtained from the getOutputOI call.
2. transformRawInput: In the case where this function indicates that it will transform the raw input before it is fed through the partitioning mechanics, this function is called. Again the contract is t is given an input Partition and must return an Partition. The shape of the output Partition is obtained from getRawInputOI() call.

    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);


    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();

View Full Code Here

        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      ArrayList<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    ArrayList<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,

View Full Code Here

    ShapeDetails inpShape = def.getInput().getOutputShape();


    /*
     * 1. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    WindowingTableFunctionResolver tResolver =
        (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);

View Full Code Here

    }


    /*
     * 2. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    // TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName());
    TableFunctionResolver tResolver = constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);


    /*
     * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape
     */
    if (tEval.isTransformsRawInput()) {
      tResolver.initializeRawInputOI();
      initialize(def.getRawInputShape(), tEval.getRawInputOI());
    } else {
      def.setRawInputShape(inpShape);
    }


    inpShape = def.getRawInputShape();


    /*
     * 4. give Evaluator chance to setup for Output execution; setup Output shape.
     */
    tResolver.initializeOutputOI();
    initialize(def.getOutputShape(), tEval.getOutputOI());
  }

View Full Code Here

    wdwTFnDef.setResolverClassName(tFn.getClass().getName());
    wdwTFnDef.setAlias("ptf_" + 1);
    wdwTFnDef.setExpressionTreeString(null);
    wdwTFnDef.setTransformsRawInput(false);
    tFn.initialize(hCfg, ptfDesc, wdwTFnDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    wdwTFnDef.setTFunction(tEval);
    wdwTFnDef.setCarryForwardNames(tFn.carryForwardNames());
    wdwTFnDef.setRawInputShape(inpShape);


    PartitioningSpec partiSpec = wdwSpec.getQueryPartitioningSpec();

View Full Code Here

        def.addArg(argDef);
      }
    }


    tFn.initialize(hCfg, ptfDesc, def);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    def.setTFunction(tEval);
    def.setCarryForwardNames(tFn.carryForwardNames());
    tFn.setupRawInputOI();


    if (tFn.transformsRawInput()) {
      StructObjectInspector rawInOutOI = tEval.getRawInputOI();
      List<String> rawInOutColNames = tFn.getRawInputColumnNames();
      RowResolver rawInRR = buildRowResolverForPTF(def.getName(),
          spec.getAlias(),
          rawInOutOI,
          rawInOutColNames,
          inpDef.getOutputShape().getRr());
      ShapeDetails rawInpShape = setupTableFnShape(def.getName(),
          inpDef.getOutputShape(),
          rawInOutOI,
          rawInOutColNames,
          rawInRR);
      def.setRawInputShape(rawInpShape);
    }
    else {
      def.setRawInputShape(inpDef.getOutputShape());
    }


    translatePartitioning(def, spec);
    tFn.setupOutputOI();


    StructObjectInspector outputOI = tEval.getOutputOI();
    List<String> outColNames = tFn.getOutputColumnNames();
    RowResolver outRR = buildRowResolverForPTF(def.getName(),
        spec.getAlias(),
        outputOI,
        outColNames,

View Full Code Here

   * @throws HiveException
   */
  public PTFPartition createFirstPartitionForChain(ObjectInspector oi,
      HiveConf hiveConf, boolean isMapSide) throws HiveException {
    PartitionedTableFunctionDef tabDef = conf.getStartOfChain();
    TableFunctionEvaluator tEval = tabDef.getTFunction();


    PTFPartition part = null;
    SerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() :
      tabDef.getRawInputShape().getSerde();
    StructObjectInspector outputOI = isMapSide ? tabDef.getInput().getOutputShape().getOI() :

View Full Code Here

    ShapeDetails inpShape = def.getInput().getOutputShape();


    /*
     * 1. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    WindowingTableFunctionResolver tResolver =
        (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);

View Full Code Here

    }


    /*
     * 2. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    // TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName());
    TableFunctionResolver tResolver = constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);


    /*
     * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape
     */
    if (tEval.isTransformsRawInput()) {
      tResolver.initializeRawInputOI();
      initialize(def.getRawInputShape(), tEval.getRawInputOI());
    } else {
      def.setRawInputShape(inpShape);
    }


    inpShape = def.getRawInputShape();


    /*
     * 4. give Evaluator chance to setup for Output execution; setup Output shape.
     */
    tResolver.initializeOutputOI();
    initialize(def.getOutputShape(), tEval.getOutputOI());
  }

View Full Code Here

   * If the final PTF in a PTFChain can stream its output, then set the OI of its OutputShape
   * to the OI returned by the TableFunctionEvaluator.
   */
  public static void alterOutputOIForStreaming(PTFDesc ptfDesc) {
    PartitionedTableFunctionDef tDef = ptfDesc.getFuncDef();
    TableFunctionEvaluator tEval = tDef.getTFunction();


    if ( tEval.canIterateOutput() ) {
      tDef.getOutputShape().setOI(tEval.getOutputOI());
    }
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator

org.apache.hadoop.hive.ql.exec.PTFOperator

org.apache.hadoop.hive.ql.exec.PTFPartition

org.apache.hadoop.hive.ql.parse.PTFTranslator

org.apache.hadoop.hive.ql.plan.PTFDeserializer

org.apache.hadoop.hive.ql.metadata.HiveException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.