Package org.apache.pig.tools.pigstats.tez

Examples of org.apache.pig.tools.pigstats.tez.TezScriptState


        // set various parallelism into the job conf for later analysis, PIG-2779
        payloadConf.setInt(PigImplConstants.REDUCER_DEFAULT_PARALLELISM, pc.defaultParallel);
        payloadConf.setInt(PigImplConstants.REDUCER_REQUESTED_PARALLELISM, tezOp.getRequestedParallelism());
        payloadConf.setInt(PigImplConstants.REDUCER_ESTIMATED_PARALLELISM, tezOp.getEstimatedParallelism());

        TezScriptState ss = TezScriptState.get();
        ss.addVertexSettingsToConf(dag.getName(), tezOp, payloadConf);

        // Take our assembled configuration and create a vertex
        UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf);
        procDesc.setUserPayload(userPayload);
View Full Code Here


        this.launcher = new TezLauncher();
    }

    @Override
    public ScriptState instantiateScriptState() {
        TezScriptState ss = new TezScriptState(UUID.randomUUID().toString());
        ss.setPigContext(pigContext);
        return ss;
    }
View Full Code Here

    private static SessionInfo createSession(Configuration conf,
            Map<String, LocalResource> requestedAMResources, Credentials creds,
            TezJobConfig tezJobConf) throws TezException, IOException,
            InterruptedException {
        TezConfiguration amConf = MRToTezHelper.getDAGAMConfFromMRConf(conf);
        TezScriptState ss = TezScriptState.get();
        ss.addDAGSettingsToConf(amConf);
        adjustAMConfig(amConf, tezJobConf);
        String jobName = conf.get(PigContext.JOB_NAME, "pig");
        TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds);
        tezClient.start();
        TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus();
View Full Code Here

        return comp.getTezPlan();
    }

    private DAG getTezDAG(TezOperPlan tezPlan, PigContext pc) {
        TezPlanContainerNode tezPlanNode = new TezPlanContainerNode(OperatorKey.genOpKey("DAGName"), tezPlan);
        TezScriptState scriptState = new TezScriptState("test");
        ScriptState.start(scriptState);
        scriptState.setDAGScriptInfo(tezPlanNode);
        DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString());
        return tezDag;
    }
View Full Code Here

        return comp.getTezPlan();
    }

    private DAG getTezDAG(TezOperPlan tezPlan, PigContext pc) {
        TezPlanContainerNode tezPlanNode = new TezPlanContainerNode(OperatorKey.genOpKey("DAGName"), tezPlan);
        TezScriptState scriptState = new TezScriptState("test");
        ScriptState.start(scriptState);
        scriptState.setDAGScriptInfo(tezPlanNode);
        DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString());
        return tezDag;
    }
View Full Code Here

        TezCompiler comp = new TezCompiler(pp, pc);
        TezOperPlan tezPlan = comp.compile();
        TezLauncher.processLoadAndParallelism(tezPlan, pc);

        TezPlanContainerNode tezPlanNode = new TezPlanContainerNode(OperatorKey.genOpKey("DAGName"), tezPlan);
        TezScriptState scriptState = new TezScriptState("test");
        ScriptState.start(scriptState);
        scriptState.setDAGScriptInfo(tezPlanNode);

        TezJobCompiler jobComp = new TezJobCompiler(pc, new Configuration());
        DAG dag = jobComp.buildDAG(tezPlanNode, new HashMap<String, LocalResource>());
        return new Pair<TezOperPlan, DAG>(tezPlan, dag);
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.tools.pigstats.tez.TezScriptState

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.