Examples of DagEngine


Examples of org.apache.oozie.DagEngine

            throws XServletException {
        JSONObject json = new JSONObject();

        try {
            String user = conf.get(OozieClient.USER_NAME);
            DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, getAuthToken(request));
            String id = dagEngine.submitHttpJob(conf, jobType);
            json.put(JsonTags.JOB_ID, id);
        }
        catch (DagEngineException ex) {
            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
        }
View Full Code Here

Examples of org.apache.oozie.DagEngine

    public void testKillNodeErrorMessage() throws Exception {
        Reader reader = IOUtils.getResourceAsReader("wf-test-kill-node-message.xml", -1);
        Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
        IOUtils.copyCharStream(reader, writer);

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("error", "end.error");
        conf.set("external-status", "FAILED/KILLED");
        conf.set("signal-value", "fail");

        final String jobId = engine.submitJob(conf, true);

        final JPAService jpaService = Services.get().get(JPAService.class);
        final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);

        waitFor(50000, new Predicate() {
View Full Code Here

Examples of org.apache.oozie.DagEngine

    private void _testNonTransient(String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
        Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
        Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
        IOUtils.copyCharStream(reader, writer);

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("signal-value", "OK");
        conf.set("external-status", "ok");
        conf.set("error", errorType);

        final String jobId = engine.submitJob(conf, true);

        waitFor(5000, new Predicate() {
            public boolean evaluate() throws Exception {
                return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
            }
        });

        final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
        store.beginTrx();
        List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true);
        int n = actions.size();
        WorkflowActionBean action = actions.get(n - 1);
        assertEquals("TEST_ERROR", action.getErrorCode());
        assertEquals(expErrorMsg, action.getErrorMessage());
        assertEquals(expStatus1, action.getStatus());
        assertTrue(action.getPending() == false);

        assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);

        String actionConf = action.getConf();
        String fixedActionConf = actionConf.replaceAll(errorType, "none");
        action.setConf(fixedActionConf);
        store.updateAction(action);
        store.commitTrx();
        store.closeTrx();

        engine.resume(jobId);

        waitFor(5000, new Predicate() {
            public boolean evaluate() throws Exception {
                return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED);
            }
        });

        assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());

        final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
        store2.beginTrx();
        actions = store2.getActionsForWorkflow(jobId, false);
        action = actions.get(0);
View Full Code Here

Examples of org.apache.oozie.DagEngine

    private void _testNonTransientWithCoordActionUpdate(String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
        Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
        Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
        IOUtils.copyCharStream(reader, writer);

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("signal-value", "OK");
        conf.set("external-status", "ok");
        conf.set("error", errorType);

        JPAService jpaService = Services.get().get(JPAService.class);
        CoordinatorJobBean coordJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, false, false);
        CoordinatorActionBean coordAction = addRecordToCoordActionTable(coordJob.getId(), 1,
                CoordinatorAction.Status.RUNNING, "coord-action-get.xml", "wfId", "RUNNING", 0);

        final String jobId = engine.submitJob(conf, true);

        coordAction.setExternalId(jobId);
        CoordActionUpdateJPAExecutor coordActionUpdateExecutor = new CoordActionUpdateJPAExecutor(coordAction);
        jpaService.execute(coordActionUpdateExecutor);

        waitFor(5000, new Predicate() {
            public boolean evaluate() throws Exception {
                return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
            }
        });

        assertNotNull(jpaService);
        WorkflowJobGetJPAExecutor wfGetCmd = new WorkflowJobGetJPAExecutor(jobId);
View Full Code Here

Examples of org.apache.oozie.DagEngine

        IOUtils.copyCharStream(reader, writer);

        final int maxRetries = 2;
        final int retryInterval = 10;

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("signal-value", "OK");
        conf.set("external-status", "ok");
        conf.set("error", errorType);
        conf.setInt(OozieClient.ACTION_MAX_RETRIES, maxRetries);
        conf.setInt(OozieClient.ACTION_RETRY_INTERVAL, retryInterval);

        final String jobId = engine.submitJob(conf, true);

        int retryCount = 1;
        WorkflowActionBean.Status expectedStatus = expStatus1;
        int expectedRetryCount = 2;

        Thread.sleep(20000);
        String aId = null;
        final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
        store.beginTrx();
        while (retryCount <= maxRetries) {
            List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
            WorkflowActionBean action = actions.get(0);
            aId = action.getId();
            assertEquals(expectedStatus, action.getStatus());
            assertEquals(expectedRetryCount, action.getRetries());
            assertEquals("TEST_ERROR", action.getErrorCode());
            assertEquals(expErrorMsg, action.getErrorMessage());
            if (action.getRetries() == maxRetries) {
                expectedRetryCount = 0;
                expectedStatus = expStatus2;
                break;
            }
            else {
                expectedRetryCount++;
            }
            Thread.sleep(retryInterval * 1000);
            retryCount++;
        }
        store.commitTrx();
        store.closeTrx();
        Thread.sleep(5000);

        final String actionId = aId;

        waitFor(5000, new Predicate() {
            public boolean evaluate() throws Exception {
                return (engine.getWorkflowAction(actionId).getStatus() == expStatus2);
            }
        });

        final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
        store2.beginTrx();
        WorkflowActionBean action = engine.getWorkflowAction(actionId);
        assertEquals("TEST_ERROR", action.getErrorCode());
        assertEquals(expErrorMsg, action.getErrorMessage());
        assertEquals(expStatus2, action.getStatus());
        assertTrue(action.getPending() == false);
        assertEquals(WorkflowJob.Status.SUSPENDED, engine.getJob(jobId).getStatus());
        store2.commitTrx();
        store2.closeTrx();
    }
View Full Code Here

Examples of org.apache.oozie.DagEngine

    private void _testError(String errorType, String externalStatus, String signalValue) throws Exception {
        Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
        Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
        IOUtils.copyCharStream(reader, writer);

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("error", errorType);
        conf.set("external-status", externalStatus);
        conf.set("signal-value", signalValue);

        final String jobId = engine.submitJob(conf, true);

        final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
        store.beginTrx();
        waitFor(5000, new Predicate() {
            public boolean evaluate() throws Exception {
                WorkflowJobBean bean = store.getWorkflow(jobId, false);
                return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
            }
        });
        assertEquals(WorkflowJob.Status.KILLED, engine.getJob(jobId).getStatus());
        store.commitTrx();
        store.closeTrx();
    }
View Full Code Here

Examples of org.apache.oozie.DagEngine

    private void _testErrorWithUserRetry(String errorType, String externalStatus, String signalValue) throws Exception {
        Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid-user-retry.xml", -1);
        Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
        IOUtils.copyCharStream(reader, writer);

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("error", errorType);
        conf.set("external-status", externalStatus);
        conf.set("signal-value", signalValue);

        final String jobId = engine.submitJob(conf, true);

        final JPAService jpaService = Services.get().get(JPAService.class);
        final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);
   
        final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor = new WorkflowActionsGetForJobJPAExecutor(jobId);
View Full Code Here

Examples of org.apache.oozie.DagEngine

    private void _testDataNotSet(String avoidParam, String expActionErrorCode) throws Exception {
        Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
        Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
        IOUtils.copyCharStream(reader, writer);

        final DagEngine engine = new DagEngine("u", "a");
        Configuration conf = new XConfiguration();
        conf.set(OozieClient.APP_PATH, "file://" + getTestCaseDir() + File.separator + "workflow.xml");
        conf.set(OozieClient.USER_NAME, getTestUser());

        conf.set(OozieClient.LOG_TOKEN, "t");
        conf.set("external-status", "ok");
        conf.set("signal-value", "based_on_action_status");
        conf.set(avoidParam, "true");

        final String jobId = engine.submitJob(conf, true);

        final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
        store.beginTrx();
        Thread.sleep(2000);

        waitFor(5000, new Predicate() {
            public boolean evaluate() throws Exception {
                WorkflowJobBean bean = store.getWorkflow(jobId, false);
                return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.FAILED);
            }
        });
        store.commitTrx();
        store.closeTrx();

        final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
        store2.beginTrx();
        assertEquals(WorkflowInstance.Status.FAILED, store2.getWorkflow(jobId, false).getWorkflowInstance().getStatus());
        assertEquals(WorkflowJob.Status.FAILED, engine.getJob(jobId).getStatus());

        List<WorkflowActionBean> actions = store2.getActionsForWorkflow(jobId, false);
        WorkflowActionBean action = actions.get(0);
        assertEquals(expActionErrorCode, action.getErrorCode());
        store2.commitTrx();
View Full Code Here

Examples of org.apache.oozie.DagEngine

            // create merged runConf to pass to WF Engine
            Configuration runConf = mergeConfig(coordAction);
            coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
            // log.debug("%%% merged runconf=" +
            // XmlUtils.prettyPrint(runConf).toString());
            DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
            try {
                boolean startJob = true;
                Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
                SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(), Status.STARTED,
                        SlaAppType.COORDINATOR_ACTION, log);
                if(slaEvent != null) {
                    insertList.add(slaEvent);
                }

                // Normalize workflow appPath here;
                JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
                String wfId = dagEngine.submitJob(conf, startJob);
                coordAction.setStatus(CoordinatorAction.Status.RUNNING);
                coordAction.setExternalId(wfId);
                coordAction.incrementAndGetPending();

                //store.updateCoordinatorAction(coordAction);
View Full Code Here

Examples of org.apache.oozie.DagEngine

                obj = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
                assertEquals(MockDagEngineService.JOB_ID + wfCount + MockDagEngineService.JOB_ID_END,
                             obj.get(JsonTags.JOB_ID));
                assertTrue(MockDagEngineService.started.get(wfCount));
                Services services = Services.get();
                DagEngine de = services.get(DagEngineService.class).getDagEngine(getTestUser(), "undef");
                StringReader sr = new StringReader(de.getJob(MockDagEngineService.JOB_ID + wfCount).getConf());
                Configuration conf1 = new XConfiguration(sr);
                return null;
            }
        });
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.