Examples of XLog


Examples of org.apache.oozie.util.XLog

        WorkflowJobBean wfBean = store.getWorkflow(jobId, false);
        setLogInfo(wfBean);
        List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
        WorkflowInstance oldWfInstance = wfBean.getWorkflowInstance();
        WorkflowInstance newWfInstance;
        XLog log = XLog.getLog(getClass());
        parseSkippedNodeConf();
        checkPreConditions(wfBean, actions);

        WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
        try {
            XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
            WorkflowApp app = wps.parseDef(conf, authToken);
            XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken, true);
            WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();

            Path configDefault = new Path(new Path(conf.get(OozieClient.APP_PATH)).getParent(), SubmitCommand.CONFIG_DEFAULT);
            FileSystem fs = Services.get().get(HadoopAccessorService.class).
                    createFileSystem(wfBean.getUser(), wfBean.getGroup(), configDefault.toUri(), protoActionConf);

            if (fs.exists(configDefault)) {
                Configuration defaultConf = new XConfiguration(fs.open(configDefault));
                PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
                XConfiguration.injectDefaults(defaultConf, conf);
            }

            PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);

            try {
                newWfInstance = workflowLib.createInstance(app, conf, jobId);
            }
            catch (WorkflowException e) {
                throw new StoreException(e);
            }
            wfBean.setAppName(app.getName());
            wfBean.setProtoActionConf(protoActionConf.toXmlString());
        }
        catch (WorkflowException ex) {
            throw new CommandException(ex);
        }
        catch (IOException ex) {
            throw new CommandException(ErrorCode.E0803, ex);
        }
        catch (HadoopAccessorException e) {
            throw new CommandException(e);
        }

        for (int i = 0; i < actions.size(); i++) {
            if (!nodesToSkip.contains(actions.get(i).getName())) {
                store.deleteAction(actions.get(i).getId());
                log.info("Deleting Action[{0}] for re-run", actions.get(i).getId());
            }
            else {
                copyActionData(newWfInstance, oldWfInstance);
            }
        }
View Full Code Here

Examples of org.apache.oozie.util.XLog

            this.bundleOlderThan = bundleOlderThan;
        }

        public void run() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());
            msg = new StringBuilder();
            jpaService = Services.get().get(JPAService.class);
            runWFRecovery();
            runCoordActionRecovery();
            runCoordActionRecoveryForReady();
            runBundleRecovery();
            log.debug("QUEUING [{0}] for potential recovery", msg.toString());
            boolean ret = false;
            if (null != callables) {
                ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
                if (ret == false) {
                    log.warn("Unable to queue the callables commands for RecoveryService. "
                            + "Most possibly command queue is full. Queue size is :"
                            + Services.get().get(CallableQueueService.class).queueSize());
                }
                callables = null;
            }
            if (null != delayedCallables) {
                ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, this.delay);
                if (ret == false) {
                    log.warn("Unable to queue the delayedCallables commands for RecoveryService. "
                            + "Most possibly Callable queue is full. Queue size is :"
                            + Services.get().get(CallableQueueService.class).queueSize());
                }
                delayedCallables = null;
                this.delay = 0;
View Full Code Here

Examples of org.apache.oozie.util.XLog

            }
        }

        private void runBundleRecovery(){
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());

            try {
                List<BundleActionBean> bactions = jpaService.execute(new BundleActionsGetWaitingOlderJPAExecutor(bundleOlderThan));
                msg.append(", BUNDLE_ACTIONS : " + bactions.size());
                for (BundleActionBean baction : bactions) {
                    Services.get().get(InstrumentationService.class).get().incr(INSTRUMENTATION_GROUP,
                            INSTR_RECOVERED_BUNDLE_ACTIONS_COUNTER, 1);
                    if(baction.getStatus() == Job.Status.PREP){
                        BundleJobBean bundleJob = null;
                        try {
                            if (jpaService != null) {
                                bundleJob = jpaService.execute(new BundleJobGetJPAExecutor(baction.getBundleId()));
                            }
                            if(bundleJob != null){
                                Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
                                List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
                                for (Element coordElem : coordElems) {
                                    Attribute name = coordElem.getAttribute("name");
                                    if (name.getValue().equals(baction.getCoordName())) {
                                        Configuration coordConf = mergeConfig(coordElem,bundleJob);
                                        coordConf.set(OozieClient.BUNDLE_ID, baction.getBundleId());
                                        queueCallable(new CoordSubmitXCommand(coordConf, bundleJob.getAuthToken(), bundleJob.getId(), name.getValue()));
                                    }
                                }
                            }
                        }
                        catch (JDOMException jex) {
                            throw new CommandException(ErrorCode.E1301, jex);
                        }
                        catch (JPAExecutorException je) {
                            throw new CommandException(je);
                        }
                    }
                    else if(baction.getStatus() == Job.Status.KILLED){
                        queueCallable(new CoordKillXCommand(baction.getCoordId()));
                    }
                    else if(baction.getStatus() == Job.Status.SUSPENDED){
                        queueCallable(new CoordSuspendXCommand(baction.getCoordId()));
                    }
                    else if(baction.getStatus() == Job.Status.RUNNING){
                        queueCallable(new CoordResumeXCommand(baction.getCoordId()));
                    }
                }
            }
            catch (Exception ex) {
                log.error("Exception, {0}", ex.getMessage(), ex);
            }
        }
View Full Code Here

Examples of org.apache.oozie.util.XLog

        /**
         * Recover coordinator actions that are staying in WAITING or SUBMITTED too long
         */
        private void runCoordActionRecovery() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());

            try {
                List<CoordinatorActionBean> cactions = jpaService.execute(new CoordActionsGetForRecoveryJPAExecutor(coordOlderThan));
                msg.append(", COORD_ACTIONS : " + cactions.size());
                for (CoordinatorActionBean caction : cactions) {
                    Services.get().get(InstrumentationService.class).get().incr(INSTRUMENTATION_GROUP,
                                                                                INSTR_RECOVERED_COORD_ACTIONS_COUNTER, 1);
                    if (caction.getStatus() == CoordinatorActionBean.Status.WAITING) {
                        if (useXCommand) {
                            queueCallable(new CoordActionInputCheckXCommand(caction.getId()));
                        } else {
                            queueCallable(new CoordActionInputCheckCommand(caction.getId()));
                        }

                        log.info("Recover a WAITTING coord action and resubmit CoordActionInputCheckXCommand :" + caction.getId());
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.SUBMITTED) {
                        CoordinatorJobBean coordJob = jpaService.execute(new CoordJobGetJPAExecutor(caction.getJobId()));

                        if (useXCommand) {
                            queueCallable(new CoordActionStartXCommand(caction.getId(), coordJob.getUser(), coordJob
                                    .getAuthToken()));
                        } else {
                            queueCallable(new CoordActionStartCommand(caction.getId(), coordJob.getUser(), coordJob
                                    .getAuthToken()));
                        }

                        log.info("Recover a SUBMITTED coord action and resubmit CoordActionStartCommand :" + caction.getId());
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.SUSPENDED) {
                        if (caction.getExternalId() != null) {
                            queueCallable(new SuspendXCommand(caction.getExternalId()));
                            log.debug("Recover a SUSPENDED coord action and resubmit SuspendXCommand :" + caction.getId());
                        }
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.KILLED) {
                        if (caction.getExternalId() != null) {
                            queueCallable(new KillXCommand(caction.getExternalId()));
                            log.debug("Recover a KILLED coord action and resubmit KillXCommand :" + caction.getId());
                        }
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.RUNNING) {
                        if (caction.getExternalId() != null) {
                            queueCallable(new ResumeXCommand(caction.getExternalId()));
                            log.debug("Recover a RUNNING coord action and resubmit ResumeXCommand :" + caction.getId());
                        }
                    }
                }
            }
            catch (Exception ex) {
                log.error("Exception, {0}", ex.getMessage(), ex);
            }
        }
View Full Code Here

Examples of org.apache.oozie.util.XLog

        /**
         * Recover coordinator actions that are staying in READY too long
         */
        private void runCoordActionRecoveryForReady() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());

            try {
                List<String> jobids = jpaService.execute(new CoordActionsGetReadyGroupbyJobIDJPAExecutor(coordOlderThan));
                msg.append(", COORD_READY_JOBS : " + jobids.size());
                for (String jobid : jobids) {
                    if (useXCommand) {
                        queueCallable(new CoordActionReadyXCommand(jobid));
                    } else {
                        queueCallable(new CoordActionReadyCommand(jobid));
                    }

                    log.info("Recover READY coord actions for jobid :" + jobid);
                }
            }
            catch (Exception ex) {
                log.error("Exception, {0}", ex.getMessage(), ex);
            }
        }
View Full Code Here

Examples of org.apache.oozie.util.XLog

        /**
         * Recover wf actions
         */
        private void runWFRecovery() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());
            // queue command for action recovery
            try {
                List<WorkflowActionBean> actions = null;
                try {
                    actions = jpaService.execute(new WorkflowActionsGetPendingJPAExecutor(olderThan));
                }
                catch (JPAExecutorException ex) {
                    log.warn("Exception while reading pending actions from storage", ex);
                }
                //log.debug("QUEUING[{0}] pending wf actions for potential recovery", actions.size());
                msg.append(" WF_ACTIONS " + actions.size());

                for (WorkflowActionBean action : actions) {
                    Services.get().get(InstrumentationService.class).get().incr(INSTRUMENTATION_GROUP,
                            INSTR_RECOVERED_ACTIONS_COUNTER, 1);
                    if (action.getStatus() == WorkflowActionBean.Status.PREP
                            || action.getStatus() == WorkflowActionBean.Status.START_MANUAL) {

                        if (useXCommand) {
                            queueCallable(new ActionStartXCommand(action.getId(), action.getType()));
                        } else {
                            queueCallable(new ActionStartCommand(action.getId(), action.getType()));
                        }

                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.START_RETRY) {
                        Date nextRunTime = action.getPendingAge();
                        if (useXCommand) {
                            queueCallable(new ActionStartXCommand(action.getId(), action.getType()), nextRunTime.getTime()
                                    - System.currentTimeMillis());
                        } else {
                            queueCallable(new ActionStartCommand(action.getId(), action.getType()), nextRunTime.getTime()
                                    - System.currentTimeMillis());
                        }

                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.DONE
                            || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
                        if (useXCommand) {
                            queueCallable(new ActionEndXCommand(action.getId(), action.getType()));
                        } else {
                            queueCallable(new ActionEndCommand(action.getId(), action.getType()));
                        }

                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.END_RETRY) {
                        Date nextRunTime = action.getPendingAge();
                        if (useXCommand) {
                            queueCallable(new ActionEndXCommand(action.getId(), action.getType()), nextRunTime.getTime()
                                    - System.currentTimeMillis());
                        } else {
                            queueCallable(new ActionEndCommand(action.getId(), action.getType()), nextRunTime.getTime()
                                    - System.currentTimeMillis());
                        }

                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.OK
                            || action.getStatus() == WorkflowActionBean.Status.ERROR) {
                        if (useXCommand) {
                            queueCallable(new SignalXCommand(action.getJobId(), action.getId()));
                        } else {
                            queueCallable(new SignalCommand(action.getJobId(), action.getId()));
                        }

                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.USER_RETRY) {
                      queueCallable(new ActionStartXCommand(action.getId(), action.getType()));
                    }
                }
            }
            catch (Exception ex) {
                log.error("Exception, {0}", ex.getMessage(), ex);
            }
        }
View Full Code Here

Examples of org.apache.oozie.util.XLog

            this.actionCheckDelay = actionCheckDelay;
        }

        public void run() {
            XLog.Info.get().clear();
            XLog LOG = XLog.getLog(getClass());
            msg = new StringBuilder();
            try {
                runWFActionCheck();
                runCoordActionCheck();
            }
            catch (CommandException ce) {
                LOG.error("Unable to run action checks, ", ce);
            }

            LOG.debug("QUEUING [{0}] for potential checking", msg.toString());
            if (null != callables) {
                boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
                if (ret == false) {
                    LOG.warn("Unable to queue the callables commands for CheckerService. "
                            + "Most possibly command queue is full. Queue size is :"
                            + Services.get().get(CallableQueueService.class).queueSize());
                }
                callables = null;
            }
View Full Code Here

Examples of org.apache.oozie.util.XLog

        /**
         * Recover coordinator jobs that should be materialized
         */
        private void runCoordJobMatLookup() {
            XLog.Info.get().clear();
            XLog LOG = XLog.getLog(getClass());
            JPAService jpaService = Services.get().get(JPAService.class);
            try {

                // get current date
                Date currDate = new Date(new Date().getTime() + CONF_LOOKUP_INTERVAL_DEFAULT * 1000);
                // get list of all jobs that have actions that should be materialized.
                CoordJobsToBeMaterializedJPAExecutor cmatcmd = new CoordJobsToBeMaterializedJPAExecutor(currDate, 50);
                List<CoordinatorJobBean> materializeJobs = jpaService.execute(cmatcmd);
                LOG.debug("CoordMaterializeTriggerService - Curr Date= " + currDate + ", Num jobs to materialize = "
                        + materializeJobs.size());
                for (CoordinatorJobBean coordJob : materializeJobs) {
                    Services.get().get(InstrumentationService.class).get().incr(INSTRUMENTATION_GROUP,
                            INSTR_MAT_JOBS_COUNTER, 1);
                    int numWaitingActions = jpaService
                            .execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
                    LOG.debug("Job :" + coordJob.getId() + "  numWaitingActions : " + numWaitingActions + " MatThrottle : "
                            + coordJob.getMatThrottling());
                    if (numWaitingActions >= coordJob.getMatThrottling()) {
                        LOG.debug("Materialization skipped for JobID [" + coordJob.getId() + " already waiting "
                                + numWaitingActions + " actions. MatThrottle is : " + coordJob.getMatThrottling());
                        continue;
                    }
                    queueCallable(new CoordMaterializeTransitionXCommand(coordJob.getId(), materializationWindow));
                    //update lastModifiedTime so next time others might have higher chance to get pick up
                    coordJob.setLastModifiedTime(new Date());
                    jpaService.execute(new CoordJobUpdateJPAExecutor(coordJob));

                }

            }
            catch (JPAExecutorException jex) {
                LOG.warn("JPAExecutorException while attempting to materialize coordinator jobs", jex);
            }
        }
View Full Code Here

Examples of org.apache.oozie.util.XLog

     * without committing, thus doing a rollback.
     */
    @SuppressWarnings({"ThrowFromFinallyBlock", "unchecked"})
    public final T call() throws CommandException {
        XLog.Info.get().setParameters(logInfo);
        XLog log = XLog.getLog(getClass());
        log.trace(logMask, "Start");
        Instrumentation.Cron cron = new Instrumentation.Cron();
        cron.start();
        callables = new ArrayList<XCallable<Void>>();
        delayedCallables = new ArrayList<XCallable<Void>>();
        exceptionCallables = new ArrayList<XCallable<Void>>();
        delay = 0;
        S store = null;
        boolean exception = false;

        try {
            if (withStore) {
                store = (S) Services.get().get(StoreService.class).getStore(getStoreClass());
                store.beginTrx();
            }
            T result = execute(store);
            /*
             *
             * if (store != null && log != null) { log.info(XLog.STD,
             * "connection log from store Flush Mode {0} ",
             * store.getFlushMode()); }
             */
            if (withStore) {
                if (store == null) {
                    throw new IllegalStateException("WorkflowStore should not be null");
                }
                if (FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection")) {
                    throw new RuntimeException("Skipping Commit for Failover Testing");
                }
                store.commitTrx();
            }

            // TODO figure out the reject due to concurrency problems and remove
            // the delayed queuing for callables.
            boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables, 10);
            if (ret == false) {
                logQueueCallableFalse(callables);
            }

            ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, delay);
            if (ret == false) {
                logQueueCallableFalse(delayedCallables);
            }

            return result;
        }
        catch (XException ex) {
            log.error(logMask | XLog.OPS, "XException, {0}", ex);
            if (store != null) {
                log.info(XLog.STD, "XException - connection logs from store {0}, {1}", store.getConnection(), store
                        .isClosed());
            }
            exception = true;
            if (store != null && store.isActive()) {
                try {
                    store.rollbackTrx();
                }
                catch (RuntimeException rex) {
                    log.warn(logMask | XLog.OPS, "openjpa error, {0}, {1}", name, rex.getMessage(), rex);
                }
            }

            // TODO figure out the reject due to concurrency problems and remove
            // the delayed queuing for callables.
            boolean ret = Services.get().get(CallableQueueService.class).queueSerial(exceptionCallables, 10);
            if (ret == false) {
                logQueueCallableFalse(exceptionCallables);
            }
            if (ex instanceof CommandException) {
                throw (CommandException) ex;
            }
            else {
                throw new CommandException(ex);
            }
        }
        catch (Exception ex) {
            log.error(logMask | XLog.OPS, "Exception, {0}", ex);
            exception = true;
            if (store != null && store.isActive()) {
                try {
                    store.rollbackTrx();
                }
                catch (RuntimeException rex) {
                    log.warn(logMask | XLog.OPS, "openjpa error, {0}, {1}", name, rex.getMessage(), rex);
                }
            }
            throw new CommandException(ErrorCode.E0607, ex);
        }
        catch (Error er) {
            log.error(logMask | XLog.OPS, "Error, {0}", er);
            exception = true;
            if (store != null && store.isActive()) {
                try {
                    store.rollbackTrx();
                }
                catch (RuntimeException rex) {
                    log.warn(logMask | XLog.OPS, "openjpa error, {0}, {1}", name, rex.getMessage(), rex);
                }
            }
            throw er;
        }
        finally {
            FaultInjection.deactivate("org.apache.oozie.command.SkipCommitFaultInjection");
            cron.stop();
            instrumentation.addCron(INSTRUMENTATION_GROUP, name, cron);
            incrCommandCounter(1);
            log.trace(logMask, "End");
            if (locks != null) {
                for (LockToken lock : locks) {
                    lock.release();
                }
                locks.clear();
            }
            if (store != null) {
                if (!store.isActive()) {
                    try {
                        store.closeTrx();
                    }
                    catch (RuntimeException rex) {
                        if (exception) {
                            log.warn(logMask | XLog.OPS, "openjpa error, {0}, {1}", name, rex.getMessage(), rex);
                        }
                        else {
                            throw rex;
                        }
                    }
                }
                else {
                    log.warn(logMask | XLog.OPS, "transaction is not committed or rolled back before closing entitymanager.");
                }
            }
        }
    }
View Full Code Here

Examples of org.apache.oozie.util.XLog

    public Class<? extends Service> getInterface() {
        return ActionService.class;
    }

    public void register(Class<? extends ActionExecutor> klass) throws ServiceException {
        XLog log = XLog.getLog(getClass());
        ActionExecutor executor = (ActionExecutor) ReflectionUtils.newInstance(klass, services.getConf());
        if (executors.containsKey(executor.getType())) {
            throw new ServiceException(ErrorCode.E0150, XLog.format(
                    "Action executor for action type [{1}] already registered", executor.getType()));
        }
        ActionExecutor.enableInit();
        executor.initActionType();
        ActionExecutor.disableInit();
        executors.put(executor.getType(), klass);
        log.trace("Registered Action executor for action type [{0}] class [{1}]", executor.getType(), klass);
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.