Package org.apache.ivory.entity.v0.process

Examples of org.apache.ivory.entity.v0.process.Process


        resource = this.getClass().getResource("/config/feed/hive-table-feed-out.xml");
        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.FEED, outFeed);

        resource = this.getClass().getResource("/config/process/pig-process-table.xml");
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(resource);
        ConfigurationStore.get().publish(EntityType.PROCESS, process);

        Cluster cluster = ConfigurationStore.get().get(EntityType.CLUSTER, "corp");
        OozieProcessMapper mapper = new OozieProcessMapper(process);
        Path bundlePath = new Path("/", EntityUtil.getStagingPath(process));
        mapper.map(cluster, bundlePath);
        assertTrue(fs.exists(bundlePath));

        BUNDLEAPP bundle = getBundle(bundlePath);
        assertEquals(EntityUtil.getWorkflowName(process).toString(), bundle.getName());
        assertEquals(1, bundle.getCoordinator().size());
        assertEquals(EntityUtil.getWorkflowName(Tag.DEFAULT, process).toString(),
                bundle.getCoordinator().get(0).getName());
        String coordPath = bundle.getCoordinator().get(0).getAppPath().replace("${nameNode}", "");

        COORDINATORAPP coord = getCoordinator(new Path(coordPath));
        HashMap<String, String> props = new HashMap<String, String>();
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            props.put(prop.getName(), prop.getValue());
        }

        // verify table props
        Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process, cluster);
        for (Map.Entry<String, String> entry : props.entrySet()) {
            if (expected.containsKey(entry.getKey())) {
                Assert.assertEquals(entry.getValue(), expected.get(entry.getKey()));
            }
        }

        // verify the late data params
        Assert.assertEquals(props.get("falconInputFeeds"), process.getInputs().getInputs().get(0).getName());
        Assert.assertEquals(props.get("falconInPaths"), "${coord:dataIn('input')}");
        Assert.assertEquals(props.get("falconInputFeedStorageTypes"), Storage.TYPE.TABLE.name());

        // verify the post processing params
        Assert.assertEquals(props.get("feedNames"), process.getOutputs().getOutputs().get(0).getName());
        Assert.assertEquals(props.get("feedInstancePaths"), "${coord:dataOut('output')}");
    }
View Full Code Here


    @Override
    public void cleanup() throws FalconException {
        Collection<String> processes = STORE.getEntities(EntityType.PROCESS);
        for (String processName : processes) {
            Process process;
            process = STORE.get(EntityType.PROCESS, processName);
            long retention = getRetention(process, process.getFrequency()
                    .getTimeUnit());
            for (org.apache.falcon.entity.v0.process.Cluster cluster : process
                    .getClusters().getClusters()) {
                Cluster currentCluster = STORE.get(EntityType.CLUSTER,
                        cluster.getName());
                if (currentCluster.getColo().equals(getCurrentColo())) {
                    LOG.info("Cleaning up logs for process:" + processName
View Full Code Here

    @Test
    public void testUpdateCheckUser() throws Exception {
        TestContext context = newContext();
        Map<String, String> overlay = context.getUniqueOverlay();
        String tmpFileName = context.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay);
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName));
        Validity processValidity = process.getClusters().getClusters().get(0).getValidity();
        processValidity.setEnd(new Date(new Date().getTime() + 2 * 24 * 60 * 60 * 1000));
        File tmpFile = context.getTempFile();
        EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
        context.scheduleProcess(tmpFile.getAbsolutePath(), overlay);
        context.waitForBundleStart(Status.RUNNING);
View Full Code Here

            feed.setName(name);
            store.publish(type, feed);
            break;

        case PROCESS:
            Process process = (Process) unmarshaller.unmarshal(this.getClass().getResource(PROCESS_XML));
            process.setName(name);
            FileSystem fs = dfsCluster.getFileSystem();
            fs.mkdirs(new Path(process.getWorkflow().getPath()));
            if (!fs.exists(new Path(process.getWorkflow() + "/lib"))) {
                fs.mkdirs(new Path(process.getWorkflow() + "/lib"));
            }
            store.publish(type, process);
            break;
        default:
        }
View Full Code Here

    public void testOptionalInput() throws Exception {
        TestContext context = newContext();
        Map<String, String> overlay = context.getUniqueOverlay();
        String tmpFileName = context.
                overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay);
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName));

        Input in1 = process.getInputs().getInputs().get(0);
        Input in2 = new Input();
        in2.setFeed(in1.getFeed());
        in2.setName("input2");
        in2.setOptional(true);
        in2.setPartition(in1.getPartition());
        in2.setStart("now(-1,0)");
        in2.setEnd("now(0,0)");
        process.getInputs().getInputs().add(in2);

        File tmpFile = context.getTempFile();
        EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
        context.scheduleProcess(tmpFile.getAbsolutePath(), overlay);
        context.waitForWorkflowStart(context.processName);
View Full Code Here

    public void testProcessDeleteAndSchedule() throws Exception {
        //Submit process with invalid property so that coord submit fails and bundle goes to failed state
        TestContext context = newContext();
        Map<String, String> overlay = context.getUniqueOverlay();
        String tmpFileName = context.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay);
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(new File(tmpFileName));
        Property prop = new Property();
        prop.setName("newProp");
        prop.setValue("${formatTim()}");
        process.getProperties().getProperties().add(prop);
        File tmpFile = context.getTempFile();
        EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
        context.scheduleProcess(tmpFile.getAbsolutePath(), overlay);
        context.waitForBundleStart(Status.FAILED);

        //Delete and re-submit the process with correct workflow
        ClientResponse clientRepsonse = context.service.path("api/entities/delete/process/"
                + context.processName).header(
                "Remote-User", TestContext.REMOTE_USER)
                .accept(MediaType.TEXT_XML).delete(ClientResponse.class);
        context.assertSuccessful(clientRepsonse);
        process.getWorkflow().setPath("/falcon/test/workflow");
        tmpFile = context.getTempFile();
        EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
        clientRepsonse = context.service.path("api/entities/submitAndSchedule/process").
                header("Remote-User", TestContext.REMOTE_USER)
                .accept(MediaType.TEXT_XML).type(MediaType.TEXT_XML)
View Full Code Here

        ClientResponse response = context.service.path("api/entities/definition/process/"
                + context.processName).header(
                "Remote-User", TestContext.REMOTE_USER)
                .accept(MediaType.TEXT_XML).get(ClientResponse.class);
        Process process = (Process) EntityType.PROCESS.getUnmarshaller()
                .unmarshal(new StringReader(response.getEntity(String.class)));

        String feed3 = "f3" + System.currentTimeMillis();
        Map<String, String> overlay = new HashMap<String, String>();
        overlay.put("inputFeedName", feed3);
        overlay.put("cluster", context.clusterName);
        response = context.submitToFalcon(TestContext.FEED_TEMPLATE1, overlay, EntityType.FEED);
        context.assertSuccessful(response);

        Input input = new Input();
        input.setFeed(feed3);
        input.setName("inputData2");
        input.setStart("today(20,0)");
        input.setEnd("today(20,20)");
        process.getInputs().getInputs().add(input);

        Validity processValidity = process.getClusters().getClusters().get(0).getValidity();
        processValidity.setEnd(new Date(new Date().getTime() + 2 * 24 * 60 * 60 * 1000));
        File tmpFile = context.getTempFile();
        EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
        response = context.service.path("api/entities/update/process/"
                + context.processName).header("Remote-User",
View Full Code Here

        ClientResponse response = context.service.path("api/entities/definition/process/"
                + context.processName).header(
                "Remote-User", TestContext.REMOTE_USER)
                .accept(MediaType.TEXT_XML).get(ClientResponse.class);
        Process process = (Process) EntityType.PROCESS.getUnmarshaller()
                .unmarshal(new StringReader(response.getEntity(String.class)));

        Validity processValidity = process.getClusters().getClusters().get(0).getValidity();
        processValidity.setEnd(new Date(new Date().getTime() + 60 * 60 * 1000));
        File tmpFile = context.getTempFile();
        EntityType.PROCESS.getMarshaller().marshal(process, tmpFile);
        response = context.service.path("api/entities/update/process/" + context.processName).header("Remote-User",
                TestContext.REMOTE_USER).accept(MediaType.TEXT_XML)
View Full Code Here

        storeEntity(EntityType.FEED, "impressionFeed");
        storeEntity(EntityType.FEED, "clicksFeed");
        storeEntity(EntityType.FEED, "imp-click-join1");
        storeEntity(EntityType.FEED, "imp-click-join2");
        storeEntity(EntityType.PROCESS, "sample");
        Process process = ConfigurationStore.get().get(EntityType.PROCESS, "sample");
        Process otherProcess = (Process) process.copy();
        otherProcess.setName("sample2");
        otherProcess.setFrequency(new Frequency("days(1)"));
        ConfigurationStore.get().remove(EntityType.PROCESS,
                otherProcess.getName());
        ConfigurationStore.get().publish(EntityType.PROCESS, otherProcess);

        fs.mkdirs(instanceLogPath);
        fs.mkdirs(instanceLogPath1);
        fs.mkdirs(instanceLogPath2);
View Full Code Here

public class EntityUtilTest extends AbstractTestBase {
    private static TimeZone tz = TimeZone.getTimeZone("UTC");

    @Test
    public void testProcessView() throws Exception {
        Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(
                getClass().getResourceAsStream(PROCESS_XML));
        Cluster cluster = new Cluster();
        cluster.setName("newCluster");
        cluster.setValidity(process.getClusters().getClusters().get(0).getValidity());
        process.getClusters().getClusters().add(cluster);
        Assert.assertEquals(process.getClusters().getClusters().size(), 2);
        String currentCluster = process.getClusters().getClusters().get(0).getName();
        Process newProcess = EntityUtil.getClusterView(process, currentCluster);
        Assert.assertFalse(EntityUtil.equals(process, newProcess));
        Assert.assertEquals(newProcess.getClusters().getClusters().size(), 1);
        Assert.assertEquals(newProcess.getClusters().getClusters().get(0).getName(), currentCluster);
    }
View Full Code Here

TOP

Related Classes of org.apache.ivory.entity.v0.process.Process

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.