Package org.apache.tajo

Examples of org.apache.tajo.QueryUnitAttemptId

466
467
468
469
470
471
472
473
474
475
476
      }
    }
  }

  private void assignTask(QueryUnitAttemptScheduleContext attemptContext, QueryUnitAttempt taskAttempt) {
    QueryUnitAttemptId attemptId = taskAttempt.getId();
    ContainerProxy containerProxy = context.getMasterContext().getResourceAllocator().
        getContainer(attemptContext.getContainerId());
    QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
        attemptId,
        new ArrayList<FragmentProto>(taskAttempt.getQueryUnit().getAllFragments()),
View Full Code Here
23
24
25
26
27
28
29
30
31
public class TaskFatalErrorEvent extends TaskAttemptEvent {
  private final String message;

  public TaskFatalErrorEvent(TaskFatalErrorReport report) {
    super(new QueryUnitAttemptId(report.getId()),
        TaskAttemptEventType.TA_FATAL_ERROR);
    this.message = report.getErrorMessage();
  }
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
                  }
                } else {
                  taskRunnerManager.getWorkerContext().getWorkerSystemMetrics().counter("query", "task").inc();
                  LOG.info("Accumulated Received Task: " + (++receivedNum));

                  QueryUnitAttemptId taskAttemptId = new QueryUnitAttemptId(taskRequest.getId());
                  if (tasks.containsKey(taskAttemptId)) {
                    LOG.error("Duplicate Task Attempt: " + taskAttemptId);
                    fatalError(qmClientService, taskAttemptId, "Duplicate Task Attempt: " + taskAttemptId);
                    continue;
                  }
View Full Code Here
364
365
366
367
368
369
370
371
372
373
374
     *  2. unknown block or Non-splittable task in host
     *  3. remote tasks. unassignedTaskForEachVolume is only contained local task. so it will be null
     */
    public synchronized QueryUnitAttemptId getLocalTask(ContainerId containerId) {
      int volumeId;
      QueryUnitAttemptId queryUnitAttemptId = null;

      if (!lastAssignedVolumeId.containsKey(containerId)) {
        volumeId = getLowestVolumeId();
        increaseConcurrency(containerId, volumeId);
      } else {
View Full Code Here
401
402
403
404
405
406
407
408
409
410
411
      }
      return queryUnitAttemptId;
    }

    public synchronized QueryUnitAttemptId getQueryUnitAttemptIdByRack(String rack) {
      QueryUnitAttemptId queryUnitAttemptId = null;

      if (unassignedTaskForEachVolume.size() > 0 && this.rack.equals(rack)) {
        int retry = unassignedTaskForEachVolume.size();
        do {
          //clean and get a remaining task
View Full Code Here
423
424
425
426
427
428
429
430
431
432
433
      }
      return queryUnitAttemptId;
    }

    private synchronized QueryUnitAttemptId getAndRemove(int volumeId){
      QueryUnitAttemptId queryUnitAttemptId = null;
      if(!unassignedTaskForEachVolume.containsKey(volumeId)) return queryUnitAttemptId;

      LinkedHashSet<QueryUnitAttempt> list = unassignedTaskForEachVolume.get(volumeId);
      if(list != null && list.size() > 0){
        QueryUnitAttempt queryUnitAttempt;
View Full Code Here
635
636
637
638
639
640
641
642
643
644
645
    private QueryUnitAttemptId allocateLocalTask(String host, ContainerId containerId){
      HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);

      if (hostVolumeMapping != null) { //tajo host is located in hadoop datanode
        while (hostVolumeMapping.getRemainingLocalTaskSize() > 0) {
          QueryUnitAttemptId attemptId = hostVolumeMapping.getLocalTask(containerId);
          //find remaining local task
          if (leafTasks.contains(attemptId)) {
            leafTasks.remove(attemptId);
            //LOG.info(attemptId + " Assigned based on host match " + hostName);
            hostLocalAssigned++;
View Full Code Here
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
    private QueryUnitAttemptId allocateRackTask(String host) {

      List<HostVolumeMapping> remainingTasks = new ArrayList<HostVolumeMapping>(leafTaskHostMapping.values());
      String rack = RackResolver.resolve(host).getNetworkLocation();
      QueryUnitAttemptId attemptId = null;

      if (remainingTasks.size() > 0) {
        //find largest remaining task of other host in rack
        Collections.sort(remainingTasks, new Comparator<HostVolumeMapping>() {
          @Override
          public int compare(HostVolumeMapping v1, HostVolumeMapping v2) {
            // descending remaining tasks
            return Integer.valueOf(v2.remainTasksNum.get()).compareTo(Integer.valueOf(v1.remainTasksNum.get()));
          }
        });

        for (HostVolumeMapping tasks : remainingTasks) {
          while (tasks.getRemainingLocalTaskSize() > 0){
            QueryUnitAttemptId tId = tasks.getQueryUnitAttemptIdByRack(rack);

            if (tId == null) break;

            if (leafTasks.contains(tId)) {
              leafTasks.remove(tId);
              attemptId = tId;
              break;
            }
          }
          if(attemptId != null) break;
        }
      }

      //find task in rack
      if (attemptId == null) {
        HashSet<QueryUnitAttemptId> list = leafTasksRackMapping.get(rack);
        if (list != null) {
          synchronized (list) {
            Iterator<QueryUnitAttemptId> iterator = list.iterator();
            while (iterator.hasNext()) {
              QueryUnitAttemptId tId = iterator.next();
              iterator.remove();
              if (leafTasks.contains(tId)) {
                leafTasks.remove(tId);
                attemptId = tId;
                break;
View Full Code Here
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
            "containerId=" + containerId);

        //////////////////////////////////////////////////////////////////////
        // disk or host-local allocation
        //////////////////////////////////////////////////////////////////////
        QueryUnitAttemptId attemptId = allocateLocalTask(host, containerId);

        if (attemptId == null) { // if a local task cannot be found
          HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);

          if(hostVolumeMapping != null) {
            if(!hostVolumeMapping.isRemote(containerId)){
              // assign to remote volume
              hostVolumeMapping.decreaseConcurrency(containerId);
              hostVolumeMapping.increaseConcurrency(containerId, HostVolumeMapping.REMOTE);
            }
            // this part is remote concurrency management of a tail tasks
            int tailLimit = Math.max(remainingScheduledObjectNum() / (leafTaskHostMapping.size() * 2), 1);

            if(hostVolumeMapping.getRemoteConcurrency() > tailLimit){
              //release container
              hostVolumeMapping.decreaseConcurrency(containerId);
              taskRequest.getCallback().run(stopTaskRunnerReq);
              subQuery.releaseContainer(containerId);
              continue;
            }
          }

          //////////////////////////////////////////////////////////////////////
          // rack-local allocation
          //////////////////////////////////////////////////////////////////////
          attemptId = allocateRackTask(host);

          //////////////////////////////////////////////////////////////////////
          // random node allocation
          //////////////////////////////////////////////////////////////////////
          if (attemptId == null && leafTaskNum() > 0) {
            synchronized (leafTasks){
              attemptId = leafTasks.iterator().next();
              leafTasks.remove(attemptId);
              rackLocalAssigned++;
              totalAssigned++;
              LOG.info(String.format("Assigned Local/Remote/Total: (%d/%d/%d), Locality: %.2f%%,",
                  hostLocalAssigned, rackLocalAssigned, totalAssigned,
                  ((double) hostLocalAssigned / (double) totalAssigned) * 100));
            }
          }
        }

        if (attemptId != null) {
          QueryUnit task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
          QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
              attemptId,
              new ArrayList<FragmentProto>(task.getAllFragments()),
              "",
              false,
View Full Code Here
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
      TaskRequestEvent taskRequest;
      while (!taskRequests.isEmpty()) {
        taskRequest = taskRequests.pollFirst();
        LOG.debug("assignToNonLeafTasks: " + taskRequest.getExecutionBlockId());

        QueryUnitAttemptId attemptId;
        // random allocation
        if (nonLeafTasks.size() > 0) {
          synchronized (nonLeafTasks){
            attemptId = nonLeafTasks.iterator().next();
            nonLeafTasks.remove(attemptId);
          }
          LOG.debug("Assigned based on * match");

          QueryUnit task;
          task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
          QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
              attemptId,
              Lists.newArrayList(task.getAllFragments()),
              "",
              false,
View Full Code Here
TOP

Related Classes of org.apache.tajo.QueryUnitAttemptId

  • org.apache.tajo.engine.planner.physical.TestPhysicalPlanner
  • org.apache.tajo.engine.query.QueryUnitRequestImpl
  • org.apache.tajo.master.DefaultTaskScheduler$HostVolumeMapping
  • org.apache.tajo.master.DefaultTaskScheduler$ScheduledRequests
  • org.apache.tajo.master.event.TaskCompletionEvent
  • org.apache.tajo.master.event.TaskFatalErrorEvent
  • org.apache.tajo.master.LazyTaskScheduler
  • org.apache.tajo.master.TaskSchedulerImpl$ScheduledRequests
  • org.apache.tajo.pullserver.retriever.AdvancedDataRetriever
  • org.apache.tajo.util.TajoIdUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.