Examples of COORDINATORAPP

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP


        private COORDINATORAPP getRetentionCoordinator(Cluster cluster, Path bundlePath, Feed feed,
                                                       org.apache.falcon.entity.v0.feed.Cluster feedCluster)
            throws FalconException {


            COORDINATORAPP retentionApp = new COORDINATORAPP();
            String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
            retentionApp.setName(coordName);
            retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
            retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
            retentionApp.setTimezone(feed.getTimezone().getID());
            TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
            if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
                retentionApp.setFrequency("${coord:hours(6)}");
            } else {
                retentionApp.setFrequency("${coord:days(1)}");
            }


            Path wfPath = getCoordPath(bundlePath, coordName);
            retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
            return retentionApp;
        }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

                        + trgCluster.getName()
                        + " do not have overlapping dates");
                return null;
            }


            COORDINATORAPP replicationCoord;
            try {
                replicationCoord = getCoordinatorTemplate(REPLICATION_COORD_TEMPLATE);
            } catch (FalconException e) {
                throw new FalconException("Cannot unmarshall replication coordinator template", e);
            }


            String coordName = EntityUtil.getWorkflowName(
                    Tag.REPLICATION, Arrays.asList(srcCluster.getName()), feed).toString();
            String start = sourceStartDate.after(targetStartDate)
                    ? SchemaHelper.formatDateUTC(sourceStartDate) : SchemaHelper.formatDateUTC(targetStartDate);
            String end = sourceEndDate.before(targetEndDate)
                    ? SchemaHelper.formatDateUTC(sourceEndDate) : SchemaHelper.formatDateUTC(targetEndDate);


            initializeCoordAttributes(replicationCoord, coordName, feed, start, end, replicationDelayInMillis);
            setCoordControls(feed, replicationCoord);


            final Storage sourceStorage = FeedHelper.createReadOnlyStorage(srcCluster, feed);
            initializeInputDataSet(feed, srcCluster, replicationCoord, sourceStorage);


            final Storage targetStorage = FeedHelper.createStorage(trgCluster, feed);
            initializeOutputDataSet(feed, trgCluster, replicationCoord, targetStorage);


            Path wfPath = getCoordPath(bundlePath, coordName);
            ACTION replicationWorkflowAction = getReplicationWorkflowAction(
                    srcCluster, trgCluster, wfPath, coordName, sourceStorage, targetStorage);
            replicationCoord.setAction(replicationWorkflowAction);


            return replicationCoord;
        }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");


        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        testDefCoordMap(process, coord);
        assertEquals(coord.getControls().getThrottle(), throttle);
        assertEquals(coord.getControls().getTimeout(), timeout);


        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        return getParentWorkflow(new Path(wfPath));
    }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, processEntity).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");


        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }


        String[] expected = {
            EntityInstanceMessage.ARG.feedNames.getPropName(),

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");


        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }


        // verify table props
        Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process);
        for (Map.Entry<String, String> entry : props.entrySet()) {
            if (expected.containsKey(entry.getKey())) {
                Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
            }
        }


        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        WORKFLOWAPP parentWorkflow = getParentWorkflow(new Path(wfPath));
        testParentWorkflow(process, parentWorkflow);


        List<Object> decisionOrForkOrJoin = parentWorkflow.getDecisionOrForkOrJoin();

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");


        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }


        // verify table props
        Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process);
        for (Map.Entry<String, String> entry : props.entrySet()) {
            if (expected.containsKey(entry.getKey())) {
                Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
            }
        }


        // verify the late data params
        Assert.assertEquals(props.get("falconInputFeeds"), process.getInputs().getInputs().get(0).getFeed());
        Assert.assertEquals(props.get("falconInPaths"), "${coord:dataIn('input')}");
        Assert.assertEquals(props.get("falconInputFeedStorageTypes"), Storage.TYPE.TABLE.name());


        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), process.getOutputs().getOutputs().get(0).getFeed());
        Assert.assertEquals(props.get("feedInstancePaths"), "${coord:dataOut('output')}");


        String wfPath = coord.getAction().getWorkflow().getAppPath().replace("${nameNode}", "");
        WORKFLOWAPP parentWorkflow = getParentWorkflow(new Path(wfPath));


        Assert.assertTrue(Storage.TYPE.TABLE == ProcessHelper.getStorageType(cluster, process));
        assertHCatCredentials(parentWorkflow, wfPath);
    }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

    public COORDINATORAPP createDefaultCoordinator(Cluster cluster, Path bundlePath) throws FalconException {
        if (entity == null) {
            return null;
        }


        COORDINATORAPP coord = new COORDINATORAPP();
        String coordName = EntityUtil.getWorkflowName(Tag.DEFAULT, entity).toString();
        Path coordPath = getCoordPath(bundlePath, coordName);


        // coord attributes
        initializeCoordAttributes(cluster, entity, coord, coordName);


        CONTROLS controls = initializeControls(entity); // controls
        coord.setControls(controls);


        // Configuration
        Map<String, String> props = createCoordDefaultConfiguration(cluster, coordPath, coordName);


        initializeInputPaths(cluster, entity, coord, props); // inputs
        initializeOutputPaths(cluster, entity, coord, props);  // outputs


        Workflow processWorkflow = entity.getWorkflow();
        propagateUserWorkflowProperties(processWorkflow, props, entity.getName());


        // create parent wf
        createWorkflow(cluster, entity, processWorkflow, coordName, coordPath);


        WORKFLOW wf = new WORKFLOW();
        wf.setAppPath(getStoragePath(coordPath.toString()));
        wf.setConfiguration(getCoordConfig(props));


        // set coord action to parent wf
        org.apache.falcon.oozie.coordinator.ACTION action = new org.apache.falcon.oozie.coordinator.ACTION();
        action.setWorkflow(wf);
        coord.setAction(action);


        return coord;
    }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

    private final ReplicationOozieWorkflowMapper replicationMapper = new ReplicationOozieWorkflowMapper();


    @Override
    public List<COORDINATORAPP> getCoordinators(Cluster cluster, Path bundlePath) throws FalconException {
        List<COORDINATORAPP> coords = new ArrayList<COORDINATORAPP>();
        COORDINATORAPP retentionCoord = getRetentionCoordinator(cluster, bundlePath);
        if (retentionCoord != null) {
            coords.add(retentionCoord);
        }
        List<COORDINATORAPP> replicationCoords = getReplicationCoordinators(cluster, bundlePath);
        coords.addAll(replicationCoords);

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP


                    // Different workflow for each source since hive credentials vary for each cluster
                    replicationMapper.createReplicationWorkflow(
                            targetCluster, srcCluster, sourceSpecificWfPath, coordName);


                    COORDINATORAPP coord = replicationMapper.createAndGetCoord(
                            entity, srcCluster, targetCluster, sourceSpecificWfPath);


                    if (coord != null) {
                        replicationCoords.add(coord);
                    }

View Full Code Here

Examples of org.apache.falcon.oozie.coordinator.COORDINATORAPP

    @Test
    public void testReplicationCoordsForFSStorage() throws Exception {
        OozieFeedWorkflowBuilder builder = new OozieFeedWorkflowBuilder(feed);
        List<COORDINATORAPP> coords = builder.getCoordinators(trgCluster, new Path("/projects/falcon/"));
        //Assert retention coord
        COORDINATORAPP coord = coords.get(0);
        assertLibExtensions(coord, "retention");


        //Assert replication coord
        coord = coords.get(1);
        Assert.assertEquals("2010-01-01T00:40Z", coord.getStart());
        Assert.assertEquals(getWorkflowAppPath(), coord.getAction().getWorkflow().getAppPath());
        Assert.assertEquals("FALCON_FEED_REPLICATION_" + feed.getName() + "_"
                + srcCluster.getName(), coord.getName());
        Assert.assertEquals("${coord:minutes(20)}", coord.getFrequency());
        SYNCDATASET inputDataset = (SYNCDATASET) coord.getDatasets()
                .getDatasetOrAsyncDataset().get(0);
        SYNCDATASET outputDataset = (SYNCDATASET) coord.getDatasets()
                .getDatasetOrAsyncDataset().get(1);


        Assert.assertEquals("${coord:minutes(20)}", inputDataset.getFrequency());
        Assert.assertEquals("input-dataset", inputDataset.getName());
        Assert.assertEquals(
                ClusterHelper.getReadOnlyStorageUrl(srcCluster)
                        + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
                inputDataset.getUriTemplate());


        Assert.assertEquals("${coord:minutes(20)}",
                outputDataset.getFrequency());
        Assert.assertEquals("output-dataset", outputDataset.getName());
        Assert.assertEquals(ClusterHelper.getStorageUrl(trgCluster)
                + "/examples/input-data/rawLogs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}",
                        outputDataset.getUriTemplate());
        String inEventName =coord.getInputEvents().getDataIn().get(0).getName();
        String inEventDataset =coord.getInputEvents().getDataIn().get(0).getDataset();
        String inEventInstance = coord.getInputEvents().getDataIn().get(0).getInstance().get(0);
        Assert.assertEquals("input", inEventName);
        Assert.assertEquals("input-dataset", inEventDataset);
        Assert.assertEquals("${now(0,-40)}", inEventInstance);


        String outEventInstance = coord.getOutputEvents().getDataOut().get(0).getInstance();
        Assert.assertEquals("${now(0,-40)}", outEventInstance);


        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }


        // verify the replication param that feed replicator depends on
        String pathsWithPartitions = getPathsWithPartitions(srcCluster, trgCluster, feed);

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.