Package org.apache.tajo.engine.planner.global

Examples of org.apache.tajo.engine.planner.global.MasterPlan


     * @param subQuery
     * @return
     */
    public static int calculateShuffleOutputNum(SubQuery subQuery, DataChannel channel) {
      TajoConf conf = subQuery.context.getConf();
      MasterPlan masterPlan = subQuery.getMasterPlan();
      ExecutionBlock parent = masterPlan.getParent(subQuery.getBlock());

      GroupbyNode grpNode = null;
      if (parent != null) {
        grpNode = PlannerUtil.findMostBottomNode(parent.getPlan(), NodeType.GROUP_BY);
      }

      // Is this subquery the first step of join?
      if (parent != null && parent.getScanNodes().length == 2) {
        List<ExecutionBlock> childs = masterPlan.getChilds(parent);

        // for outer
        ExecutionBlock outer = childs.get(0);
        long outerVolume = getInputVolume(subQuery.masterPlan, subQuery.context, outer);

        // for inner
        ExecutionBlock inner = childs.get(1);
        long innerVolume = getInputVolume(subQuery.masterPlan, subQuery.context, inner);
        LOG.info(subQuery.getId() + ", Outer volume: " + Math.ceil((double) outerVolume / 1048576) + "MB, "
            + "Inner volume: " + Math.ceil((double) innerVolume / 1048576) + "MB");

        long bigger = Math.max(outerVolume, innerVolume);

        int mb = (int) Math.ceil((double) bigger / 1048576);
        LOG.info(subQuery.getId() + ", Bigger Table's volume is approximately " + mb + " MB");

        int taskNum = (int) Math.ceil((double) mb /
            conf.getIntVar(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME));

        int totalMem = getClusterTotalMemory(subQuery);
        LOG.info(subQuery.getId() + ", Total memory of cluster is " + totalMem + " MB");
        int slots = Math.max(totalMem / conf.getIntVar(ConfVars.TASK_DEFAULT_MEMORY), 1);

        // determine the number of task
        taskNum = Math.min(taskNum, slots);
        LOG.info(subQuery.getId() + ", The determined number of join partitions is " + taskNum);

        // The shuffle output numbers of join may be inconsistent by execution block order.
        // Thus, we need to compare the number with DataChannel output numbers.
        // If the number is right, the number and DataChannel output numbers will be consistent.
        int outerShuffleOutptNum = 0, innerShuffleOutputNum = 0;
        for (DataChannel eachChannel : masterPlan.getOutgoingChannels(outer.getId())) {
          outerShuffleOutptNum = Math.max(outerShuffleOutptNum, eachChannel.getShuffleOutputNum());
        }

        for (DataChannel eachChannel : masterPlan.getOutgoingChannels(inner.getId())) {
          innerShuffleOutputNum = Math.max(innerShuffleOutputNum, eachChannel.getShuffleOutputNum());
        }

        if (outerShuffleOutptNum != innerShuffleOutputNum
            && taskNum != outerShuffleOutptNum
View Full Code Here


        return taskNum;
      }
    }

    private static void schedule(SubQuery subQuery) throws IOException {
      MasterPlan masterPlan = subQuery.getMasterPlan();
      ExecutionBlock execBlock = subQuery.getBlock();
      if (subQuery.getMasterPlan().isLeaf(execBlock.getId()) && execBlock.getScanNodes().length == 1) { // Case 1: Just Scan
        scheduleFragmentsForLeafQuery(subQuery);
      } else if (execBlock.getScanNodes().length > 1) { // Case 2: Join
        Repartitioner.scheduleFragmentsForJoinQuery(subQuery.schedulerContext, subQuery);
View Full Code Here

  private static int HTTP_REQUEST_MAXIMUM_LENGTH = 1900;
  private final static String UNKNOWN_HOST = "unknown";

  public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerContext, SubQuery subQuery)
      throws IOException {
    MasterPlan masterPlan = subQuery.getMasterPlan();
    ExecutionBlock execBlock = subQuery.getBlock();
    QueryMasterTask.QueryMasterTaskContext masterContext = subQuery.getContext();
    AbstractStorageManager storageManager = subQuery.getStorageManager();

    ScanNode[] scans = execBlock.getScanNodes();

    Path tablePath;
    FileFragment[] fragments = new FileFragment[scans.length];
    long[] stats = new long[scans.length];

    // initialize variables from the child operators
    for (int i = 0; i < scans.length; i++) {
      TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName());
      if (tableDesc == null) { // if it is a real table stored on storage
        // TODO - to be fixed (wrong directory)
        ExecutionBlock [] childBlocks = new ExecutionBlock[2];
        childBlocks[0] = masterPlan.getChild(execBlock.getId(), 0);
        childBlocks[1] = masterPlan.getChild(execBlock.getId(), 1);

        tablePath = storageManager.getTablePath(scans[i].getTableName());
        stats[i] = masterContext.getSubQuery(childBlocks[i].getId()).getResultStats().getNumBytes();
        fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
      } else {
View Full Code Here

    ExecutionBlock execBlock = subQuery.getBlock();
    Column[] keys;
    // if the next query is join,
    // set the partition number for the current logicalUnit
    // TODO: the union handling is required when a join has unions as its child
    MasterPlan masterPlan = subQuery.getMasterPlan();
    keys = channel.getShuffleKeys();
    if (!masterPlan.isRoot(subQuery.getBlock()) ) {
      ExecutionBlock parentBlock = masterPlan.getParent(subQuery.getBlock());
      if (parentBlock.getPlan().getType() == NodeType.JOIN) {
        channel.setShuffleOutputNum(desiredNum);
      }
    }
View Full Code Here

            ScanNode scanNode = (ScanNode) eachScanNode;
            tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
          }
        }
      }
      MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
      queryMasterContext.getGlobalPlanner().build(masterPlan);

      query = new Query(queryTaskContext, queryId, querySubmitTime,
          "", queryTaskContext.getEventHandler(), masterPlan);
View Full Code Here

  public void testConstructFromString() {
    QueryId qid1 = LocalTajoTestingUtility.newQueryId();
    QueryId qid2 = TajoIdUtils.parseQueryId(qid1.toString());
    assertEquals(qid1, qid2);

    MasterPlan plan1 = new MasterPlan(qid1, null, null);
    ExecutionBlockId sub1 = plan1.newExecutionBlockId();
    ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString());
    assertEquals(sub1, sub2);
   
    QueryUnitId u1 = QueryIdFactory.newQueryUnitId(sub1);
    QueryUnitId u2 = new QueryUnitId(u1.getProto());
View Full Code Here

  public void testConstructFromPB() {
    QueryId qid1 = LocalTajoTestingUtility.newQueryId();
    QueryId qid2 = new QueryId(qid1.getProto());
    assertEquals(qid1, qid2);

    MasterPlan plan = new MasterPlan(qid1, null, null);
    ExecutionBlockId sub1 = plan.newExecutionBlockId();
    ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString());
    assertEquals(sub1, sub2);

    QueryUnitId u1 = QueryIdFactory.newQueryUnitId(sub1);
    QueryUnitId u2 = new QueryUnitId(u1.getProto());
View Full Code Here

  private static int taskAttemptId;

  public static QueryUnitAttemptId newQueryUnitAttemptId() {
    return QueryIdFactory.newQueryUnitAttemptId(
        QueryIdFactory.newQueryUnitId(new MasterPlan(newQueryId(), null, null).newExecutionBlockId()), taskAttemptId++);
  }
View Full Code Here

            "join partsupp on s_suppkey = ps_suppkey " +
            "join part on p_partkey = ps_partkey and p_type like '%BRASS' and p_size = 15");
    LogicalPlan logicalPlan = logicalPlanner.createPlan(LocalTajoTestingUtility.createDummySession(), context);
    optimizer.optimize(logicalPlan);
    QueryContext queryContext = new QueryContext();
    MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), queryContext, logicalPlan);
    planner.build(plan);

    ExecutionBlockCursor cursor = new ExecutionBlockCursor(plan);

    int count = 0;
View Full Code Here

    server.stop();
  }
 
  @Test
  public final void testInterDataRetriver() throws Exception {
    MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
    ExecutionBlockId schid = plan.newExecutionBlockId();
    QueryUnitId qid1 = QueryIdFactory.newQueryUnitId(schid);
    QueryUnitId qid2 = QueryIdFactory.newQueryUnitId(schid);
   
    File qid1Dir = new File(TEST_DATA + "/" + qid1.toString() + "/out");
    qid1Dir.mkdirs();
View Full Code Here

TOP

Related Classes of org.apache.tajo.engine.planner.global.MasterPlan

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.