Package org.apache.oozie.util

Examples of org.apache.oozie.util.XLog


        public List<XCallable<?>> getCallables() {
            return this.callables;
        }

        public Void call() throws Exception {
            XLog log = XLog.getLog(getClass());

            for (XCallable<?> callable : callables) {
                log.trace("executing callable [{0}]", callable.getName());
                try {
                    callable.call();
                    incrCounter(INSTR_EXECUTED_COUNTER, 1);
                    log.trace("executed callable [{0}]", callable.getName());
                }
                catch (Exception ex) {
                    incrCounter(INSTR_FAILED_COUNTER, 1);
                    log.warn("exception callable [{0}], {1}", callable.getName(), ex.getMessage(), ex);
                }
            }

            // ticking -1 not to count the call to the composite callable
            incrCounter(INSTR_EXECUTED_COUNTER, -1);
View Full Code Here


            this.bundleOlderThan = bundleOlderThan;
        }

        public void run() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());
            msg = new StringBuilder();
            jpaService = Services.get().get(JPAService.class);
            runWFRecovery();
            runCoordActionRecovery();
            runCoordActionRecoveryForReady();
            runBundleRecovery();
            log.debug("QUEUING [{0}] for potential recovery", msg.toString());
            boolean ret = false;
            if (null != callables) {
                ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
                if (ret == false) {
                    log.warn("Unable to queue the callables commands for RecoveryService. "
                            + "Most possibly command queue is full. Queue size is :"
                            + Services.get().get(CallableQueueService.class).queueSize());
                }
                callables = null;
            }
            if (null != delayedCallables) {
                ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, this.delay);
                if (ret == false) {
                    log.warn("Unable to queue the delayedCallables commands for RecoveryService. "
                            + "Most possibly Callable queue is full. Queue size is :"
                            + Services.get().get(CallableQueueService.class).queueSize());
                }
                delayedCallables = null;
                this.delay = 0;
View Full Code Here

            }
        }

        private void runBundleRecovery(){
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());
            List<BundleActionBean> bactions = null;
            try {
                bactions = jpaService.execute(new BundleActionsGetWaitingOlderJPAExecutor(bundleOlderThan));
            }
            catch (JPAExecutorException ex) {
                log.warn("Error reading bundle actions from database", ex);
                return;
            }
            msg.append(", BUNDLE_ACTIONS : " + bactions.size());
            for (BundleActionBean baction : bactions) {
                try {
                    Services.get().get(InstrumentationService.class).get()
                            .incr(INSTRUMENTATION_GROUP, INSTR_RECOVERED_BUNDLE_ACTIONS_COUNTER, 1);
                    if (baction.getCoordId() == null) {
                        log.error("CoordId is null for Bundle action " + baction.getBundleActionId());
                        continue;
                    }
                    if (baction.getStatus() == Job.Status.PREP) {
                        BundleJobBean bundleJob = null;

                        if (jpaService != null) {
                            bundleJob = jpaService.execute(new BundleJobGetJPAExecutor(baction.getBundleId()));
                        }
                        if (bundleJob != null) {
                            Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
                            List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
                            for (Element coordElem : coordElems) {
                                Attribute name = coordElem.getAttribute("name");
                                if (name.getValue().equals(baction.getCoordName())) {
                                    Configuration coordConf = mergeConfig(coordElem, bundleJob);
                                    coordConf.set(OozieClient.BUNDLE_ID, baction.getBundleId());
                                    queueCallable(new CoordSubmitXCommand(coordConf, bundleJob.getAuthToken(),
                                            bundleJob.getId(), name.getValue()));
                                }
                            }
                        }

                    }
                    else if (baction.getStatus() == Job.Status.KILLED) {
                        queueCallable(new CoordKillXCommand(baction.getCoordId()));
                    }
                    else if (baction.getStatus() == Job.Status.SUSPENDED
                            || baction.getStatus() == Job.Status.SUSPENDEDWITHERROR) {
                        queueCallable(new CoordSuspendXCommand(baction.getCoordId()));
                    }
                    else if (baction.getStatus() == Job.Status.RUNNING
                            || baction.getStatus() == Job.Status.RUNNINGWITHERROR) {
                        queueCallable(new CoordResumeXCommand(baction.getCoordId()));
                    }
                }
                catch (Exception ex) {
                    log.error("Exception, {0}", ex.getMessage(), ex);
                }
            }


        }
View Full Code Here

        /**
         * Recover coordinator actions that are staying in WAITING or SUBMITTED too long
         */
        private void runCoordActionRecovery() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());
            List<CoordinatorActionBean> cactions = null;
            try {
                cactions = jpaService.execute(new CoordActionsGetForRecoveryJPAExecutor(coordOlderThan));
            }
            catch (JPAExecutorException ex) {
                log.warn("Error reading coord actions from database", ex);
                return;
            }
            msg.append(", COORD_ACTIONS : " + cactions.size());
            for (CoordinatorActionBean caction : cactions) {
                try {
                    Services.get().get(InstrumentationService.class).get()
                            .incr(INSTRUMENTATION_GROUP, INSTR_RECOVERED_COORD_ACTIONS_COUNTER, 1);
                    if (caction.getStatus() == CoordinatorActionBean.Status.WAITING) {
                        queueCallable(new CoordActionInputCheckXCommand(caction.getId(), caction.getJobId()));

                        log.info("Recover a WAITTING coord action and resubmit CoordActionInputCheckXCommand :"
                                + caction.getId());
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.SUBMITTED) {
                        CoordinatorJobBean coordJob = jpaService
                                .execute(new CoordJobGetJPAExecutor(caction.getJobId()));
                        queueCallable(new CoordActionStartXCommand(caction.getId(), coordJob.getUser(),
                                coordJob.getAuthToken(), caction.getJobId()));

                        log.info("Recover a SUBMITTED coord action and resubmit CoordActionStartCommand :"
                                + caction.getId());
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.SUSPENDED) {
                        if (caction.getExternalId() != null) {
                            queueCallable(new SuspendXCommand(caction.getExternalId()));
                            log.debug("Recover a SUSPENDED coord action and resubmit SuspendXCommand :"
                                    + caction.getId());
                        }
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.KILLED) {
                        if (caction.getExternalId() != null) {
                            queueCallable(new KillXCommand(caction.getExternalId()));
                            log.debug("Recover a KILLED coord action and resubmit KillXCommand :" + caction.getId());
                        }
                    }
                    else if (caction.getStatus() == CoordinatorActionBean.Status.RUNNING) {
                        if (caction.getExternalId() != null) {
                            queueCallable(new ResumeXCommand(caction.getExternalId()));
                            log.debug("Recover a RUNNING coord action and resubmit ResumeXCommand :" + caction.getId());
                        }
                    }
                }
                catch (Exception ex) {
                    log.error("Exception, {0}", ex.getMessage(), ex);
                }
            }


        }
View Full Code Here

        /**
         * Recover coordinator actions that are staying in READY too long
         */
        private void runCoordActionRecoveryForReady() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());

            try {
                List<String> jobids = jpaService.execute(new CoordActionsGetReadyGroupbyJobIDJPAExecutor(coordOlderThan));
                msg.append(", COORD_READY_JOBS : " + jobids.size());
                for (String jobid : jobids) {
                        queueCallable(new CoordActionReadyXCommand(jobid));

                    log.info("Recover READY coord actions for jobid :" + jobid);
                }
            }
            catch (Exception ex) {
                log.error("Exception, {0}", ex.getMessage(), ex);
            }
        }
View Full Code Here

        /**
         * Recover wf actions
         */
        private void runWFRecovery() {
            XLog.Info.get().clear();
            XLog log = XLog.getLog(getClass());
            // queue command for action recovery
            List<WorkflowActionBean> actions = null;
            try {
                actions = jpaService.execute(new WorkflowActionsGetPendingJPAExecutor(olderThan));
            }
            catch (JPAExecutorException ex) {
                log.warn("Exception while reading pending actions from storage", ex);
                return;
            }
            // log.debug("QUEUING[{0}] pending wf actions for potential recovery",
            // actions.size());
            msg.append(" WF_ACTIONS " + actions.size());

            for (WorkflowActionBean action : actions) {
                try {
                    Services.get().get(InstrumentationService.class).get()
                            .incr(INSTRUMENTATION_GROUP, INSTR_RECOVERED_ACTIONS_COUNTER, 1);
                    if (action.getStatus() == WorkflowActionBean.Status.PREP
                            || action.getStatus() == WorkflowActionBean.Status.START_MANUAL) {
                        queueCallable(new ActionStartXCommand(action.getId(), action.getType()));
                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.START_RETRY) {
                        Date nextRunTime = action.getPendingAge();
                        queueCallable(new ActionStartXCommand(action.getId(), action.getType()), nextRunTime.getTime()
                                - System.currentTimeMillis());
                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.DONE
                            || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
                        queueCallable(new ActionEndXCommand(action.getId(), action.getType()));
                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.END_RETRY) {
                        Date nextRunTime = action.getPendingAge();
                        queueCallable(new ActionEndXCommand(action.getId(), action.getType()), nextRunTime.getTime()
                                - System.currentTimeMillis());

                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.OK
                            || action.getStatus() == WorkflowActionBean.Status.ERROR) {
                        queueCallable(new SignalXCommand(action.getJobId(), action.getId()));
                    }
                    else if (action.getStatus() == WorkflowActionBean.Status.USER_RETRY) {
                        queueCallable(new ActionStartXCommand(action.getId(), action.getType()));
                    }
                }
                catch (Exception ex) {
                    log.error("Exception, {0}", ex.getMessage(), ex);
                }
            }

        }
View Full Code Here

            this.actionCheckDelay = actionCheckDelay;
        }

        public void run() {
            XLog.Info.get().clear();
            XLog LOG = XLog.getLog(getClass());
            msg = new StringBuilder();
            try {
                runWFActionCheck();
                runCoordActionCheck();
            }
            catch (CommandException ce) {
                LOG.error("Unable to run action checks, ", ce);
            }

            LOG.debug("QUEUING [{0}] for potential checking", msg.toString());
            if (null != callables) {
                boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
                if (ret == false) {
                    LOG.warn("Unable to queue the callables commands for CheckerService. "
                            + "Most possibly command queue is full. Queue size is :"
                            + Services.get().get(CallableQueueService.class).queueSize());
                }
                callables = null;
            }
View Full Code Here

        /**
         * Recover coordinator jobs that should be materialized
         */
        private void runCoordJobMatLookup() {
            XLog.Info.get().clear();
            XLog LOG = XLog.getLog(getClass());
            JPAService jpaService = Services.get().get(JPAService.class);
            try {

                // get current date
                Date currDate = new Date(new Date().getTime() + lookupInterval * 1000);
                // get list of all jobs that have actions that should be materialized.
                int materializationLimit = Services.get().getConf()
                        .getInt(CONF_MATERIALIZATION_SYSTEM_LIMIT, CONF_MATERIALIZATION_SYSTEM_LIMIT_DEFAULT);
                CoordJobsToBeMaterializedJPAExecutor cmatcmd = new CoordJobsToBeMaterializedJPAExecutor(currDate,
                        materializationLimit);
                List<CoordinatorJobBean> materializeJobs = jpaService.execute(cmatcmd);
                LOG.info("CoordMaterializeTriggerService - Curr Date= " + currDate + ", Num jobs to materialize = "
                        + materializeJobs.size());
                for (CoordinatorJobBean coordJob : materializeJobs) {
                    Services.get().get(InstrumentationService.class).get()
                            .incr(INSTRUMENTATION_GROUP, INSTR_MAT_JOBS_COUNTER, 1);
                    int numWaitingActions = jpaService
                            .execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
                    LOG.info("Job :" + coordJob.getId() + "  numWaitingActions : " + numWaitingActions
                            + " MatThrottle : " + coordJob.getMatThrottling());
                    // update lastModifiedTime so next time others might have higher chance to get pick up
                    coordJob.setLastModifiedTime(new Date());
                    jpaService.execute(new CoordJobUpdateJPAExecutor(coordJob));
                    if (numWaitingActions >= coordJob.getMatThrottling()) {
                        LOG.info("info for JobID [" + coordJob.getId() + " already waiting "
                                + numWaitingActions + " actions. MatThrottle is : " + coordJob.getMatThrottling());
                        continue;
                    }
                    queueCallable(new CoordMaterializeTransitionXCommand(coordJob.getId(), materializationWindow));

                }

            }
            catch (JPAExecutorException jex) {
                LOG.warn("JPAExecutorException while attempting to materialize coordinator jobs", jex);
            }
        }
View Full Code Here

            else {
                interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL));
                PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
            }

            log = new XLog(LogFactory.getLog(getClass()));

            log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
                    BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
                            .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
                            BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL));
View Full Code Here

     * @return Configuration merged configuration
     * @throws CommandException thrown if failed to merge configuration
     */
    private static Configuration mergeConfig(Element coordElem,BundleJobBean bundleJob) throws CommandException {
        XLog.Info.get().clear();
        XLog log = XLog.getLog("RecoveryService");

        String jobConf = bundleJob.getConf();
        // Step 1: runConf = jobConf
        Configuration runConf = null;
        try {
            runConf = new XConfiguration(new StringReader(jobConf));
        }
        catch (IOException e1) {
            log.warn("Configuration parse error in:" + jobConf);
            throw new CommandException(ErrorCode.E1306, e1.getMessage(), e1);
        }
        // Step 2: Merge local properties into runConf
        // extract 'property' tags under 'configuration' block in the coordElem
        // convert Element to XConfiguration
        Element localConfigElement = coordElem.getChild("configuration", coordElem.getNamespace());

        if (localConfigElement != null) {
            String strConfig = XmlUtils.prettyPrint(localConfigElement).toString();
            Configuration localConf;
            try {
                localConf = new XConfiguration(new StringReader(strConfig));
            }
            catch (IOException e1) {
                log.warn("Configuration parse error in:" + strConfig);
                throw new CommandException(ErrorCode.E1307, e1.getMessage(), e1);
            }

            // copy configuration properties in the coordElem to the runConf
            XConfiguration.copy(localConf, runConf);
View Full Code Here

TOP

Related Classes of org.apache.oozie.util.XLog

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.