Examples of GenMapRedCtx


Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

      // find the branch on which this processor was invoked
      int pos = getPositionParent(mapJoin, stack);

      Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx
          .getMapCurrCtx();
      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(
          pos));
      Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
      MapredWork currPlan = (MapredWork) currTask.getWork();
      mapredCtx.getCurrAliasId();
      Operator<? extends OperatorDesc> reducer = mapJoin;
      HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
        ctx.getOpTaskMap();
      Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);

      ctx.setCurrTask(currTask);

      // If the plan for this reducer does not exist, initialize the plan
      if (opMapTask == null) {
        assert currPlan.getReducer() == null;
        GenMapRedUtils.initMapJoinPlan(mapJoin, ctx, true, false, false, pos);
      } else {
        // The current plan can be thrown away after being merged with the
        // original plan
        GenMapRedUtils.joinPlan(mapJoin, currTask, opMapTask, ctx, pos, false,
            true, false);
        currTask = opMapTask;
        ctx.setCurrTask(currTask);
      }

      mapCurrCtx.put(mapJoin, new GenMapRedCtx(ctx.getCurrTask(), null, null));
      return null;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

      // find the branch on which this processor was invoked
      int pos = getPositionParent(mapJoin, stack);

      Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx
          .getMapCurrCtx();
      GenMapRedCtx mapredCtx = mapCurrCtx.get(mapJoin.getParentOperators().get(
          pos));
      Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
      MapredWork currPlan = (MapredWork) currTask.getWork();
      Operator<? extends OperatorDesc> reducer = mapJoin;
      HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
        ctx.getOpTaskMap();
      Task<? extends Serializable> opMapTask = opTaskMap.get(reducer);

      // union result cannot be a map table
      boolean local = (pos == (mapJoin.getConf()).getPosBigTable()) ? false
          : true;
      if (local) {
        throw new SemanticException(ErrorMsg.INVALID_MAPJOIN_TABLE.getMsg());
      }

      // If the plan for this reducer does not exist, initialize the plan
      if (opMapTask == null) {
        assert currPlan.getReducer() == null;
        ctx.setCurrMapJoinOp(mapJoin);
        GenMapRedUtils.initMapJoinPlan(mapJoin, ctx, true, true, false, pos);
        ctx.setCurrUnionOp(null);
      } else {
        // The current plan can be thrown away after being merged with the
        // original plan
        Task<? extends Serializable> uTask = ctx.getUnionTask(
            ctx.getCurrUnionOp()).getUTask();
        if (uTask.getId().equals(opMapTask.getId())) {
          GenMapRedUtils.joinPlan(mapJoin, null, opMapTask, ctx, pos, false,
              false, true);
        } else {
          GenMapRedUtils.joinPlan(mapJoin, uTask, opMapTask, ctx, pos, false,
              false, true);
        }
        currTask = opMapTask;
        ctx.setCurrTask(currTask);
      }

      mapCurrCtx.put(mapJoin, new GenMapRedCtx(ctx.getCurrTask(), ctx
          .getCurrTopOp(), ctx.getCurrAliasId()));
      return null;
    }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

    for (String alias : parseCtx.getTopOps().keySet()) {
      Operator<? extends OperatorDesc> currOp = parseCtx.getTopOps().get(alias);
      if (currOp == op) {
        String currAliasId = alias;
        ctx.setCurrAliasId(currAliasId);
        mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId));

        QBParseInfo parseInfo = parseCtx.getQB().getParseInfo();
        if (parseInfo.isAnalyzeCommand()) {

          //   ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

    // union consisted on a bunch of map-reduce jobs, and it has been split at
    // the union
    Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
    Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx = ctx
        .getMapCurrCtx();
    GenMapRedCtx mapredCtx = mapCurrCtx.get(ctx.getCurrUnionOp());

    Task<? extends Serializable> unionTask = null;
    if(mapredCtx != null) {
      unionTask = mapredCtx.getCurrTask();
    } else {
      unionTask = ctx.getCurrTask();
    }


    MapredWork plan = (MapredWork) unionTask.getWork();
    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap = ctx
        .getOpTaskMap();
    Task<? extends Serializable> reducerTask = opTaskMap.get(reducer);

    ctx.setCurrTask(unionTask);

    // If the plan for this reducer does not exist, initialize the plan
    if (reducerTask == null) {
      // When the reducer is encountered for the first time
      if (plan.getReducer() == null) {
        GenMapRedUtils.initUnionPlan(op, ctx, unionTask);
        // When union is followed by a multi-table insert
      } else {
        GenMapRedUtils.splitPlan(op, ctx);
      }
    } else if (plan.getReducer() == reducer) {
      // The union is already initialized. However, the union is walked from
      // another input
      // initUnionPlan is idempotent
      GenMapRedUtils.initUnionPlan(op, ctx, unionTask);
    } else {
      GenMapRedUtils.joinUnionPlan(ctx, unionTask, reducerTask, false);
      ctx.setCurrTask(reducerTask);
    }

    mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
        ctx.getCurrAliasId()));

    // the union operator has been processed
    ctx.setCurrUnionOp(null);
    return null;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

          UnionProcFactory.getPositionParent(union, stack));
    }
    else {
      ctx.getMapCurrCtx().put(
          (Operator<? extends OperatorDesc>) union,
          new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(),
              ctx.getCurrAliasId()));
    }

    // if the union is the first time seen, set current task to GenMRUnionCtx
    uCtxTask = ctx.getUnionTask(union);
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

    }

    ctx.setCurrTask(uTask);

    mapCurrCtx.put((Operator<? extends OperatorDesc>) nd,
        new GenMapRedCtx(ctx.getCurrTask(), null, null));

    return null;
  }
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

  public static void initPlan(ReduceSinkOperator op, GenMRProcContext opProcCtx)
      throws SemanticException {
    Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);
    Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx =
      opProcCtx.getMapCurrCtx();
    GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(0));
    Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
    MapredWork plan = (MapredWork) currTask.getWork();
    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>> opTaskMap =
      opProcCtx.getOpTaskMap();
    Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

      throws SemanticException {
    Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx =
      opProcCtx.getMapCurrCtx();
    assert (((pos == -1) && (readInputMapJoin)) || (pos != -1));
    int parentPos = (pos == -1) ? 0 : pos;
    GenMapRedCtx mapredCtx = mapCurrCtx.get(op.getParentOperators().get(
        parentPos));
    Task<? extends Serializable> currTask = mapredCtx.getCurrTask();
    MapredWork plan = (MapredWork) currTask.getWork();
    HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>>  opTaskMap =
      opProcCtx.getOpTaskMap();
    Operator<? extends OperatorDesc> currTopOp = opProcCtx.getCurrTopOp();
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

    ts_op.setChildOperators(childOpList);
    op.getParentOperators().set(posn, ts_op);

    Map<Operator<? extends OperatorDesc>, GenMapRedCtx> mapCurrCtx =
      opProcCtx.getMapCurrCtx();
    mapCurrCtx.put(ts_op, new GenMapRedCtx(childTask, null, null));

    String streamDesc = taskTmpDir;
    MapredWork cplan = (MapredWork) childTask.getWork();

    if (setReducer) {
      Operator<? extends OperatorDesc> reducer = op.getChildOperators().get(0);

      if (reducer.getClass() == JoinOperator.class) {
        String origStreamDesc;
        streamDesc = "$INTNAME";
        origStreamDesc = streamDesc;
        int pos = 0;
        while (cplan.getAliasToWork().get(streamDesc) != null) {
          streamDesc = origStreamDesc.concat(String.valueOf(++pos));
        }
      }

      // TODO: Allocate work to remove the temporary files and make that
      // dependent on the redTask
      if (reducer.getClass() == JoinOperator.class) {
        cplan.setNeedsTagging(true);
      }
    }

    // Add the path to alias mapping
    setTaskPlan(taskTmpDir, streamDesc, ts_op, cplan, local, tt_desc);

    // This can be cleaned up as a function table in future
    if (op instanceof AbstractMapJoinOperator<?>) {
      AbstractMapJoinOperator<? extends MapJoinDesc> mjOp = (AbstractMapJoinOperator<? extends MapJoinDesc>) op;
      opProcCtx.setCurrMapJoinOp(mjOp);
      GenMRMapJoinCtx mjCtx = opProcCtx.getMapJoinCtx(mjOp);
      if (mjCtx == null) {
        mjCtx = new GenMRMapJoinCtx(taskTmpDir, tt_desc, ts_op, null);
      } else {
        mjCtx.setTaskTmpDir(taskTmpDir);
        mjCtx.setTTDesc(tt_desc);
        mjCtx.setRootMapJoinOp(ts_op);
      }
      opProcCtx.setMapJoinCtx(mjOp, mjCtx);
      opProcCtx.getMapCurrCtx().put(parent,
          new GenMapRedCtx(childTask, null, null));
      setupBucketMapJoinInfo(cplan, mjOp, false);
    }

    currTopOp = null;
    String currAliasId = null;
View Full Code Here

Examples of org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx

    ctx.setCurrAliasId(null);
    ctx.setCurrTopOp(null);
    ctx.setCurrMapJoinOp(null);

    ctx.getMapCurrCtx().put(union,
        new GenMapRedCtx(ctx.getCurrTask(), null, null));
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.