Package org.apache.falcon.entity.v0.feed

Examples of org.apache.falcon.entity.v0.feed.Feed

291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
        parser.validate(process);
    }

    @Test
    public void testProcessForTableStorage() throws Exception {
        Feed inFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource("/config/feed/hive-table-feed.xml"));
        getStore().publish(EntityType.FEED, inFeed);

        Feed outFeed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource("/config/feed/hive-table-feed-out.xml"));
        getStore().publish(EntityType.FEED, outFeed);

        Process process = parser.parse(
                ProcessEntityParserTest.class.getResourceAsStream("/config/process/process-table.xml"));
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
                .unmarshal(this.getClass().getResource(CLUSTER_XML));
        ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(hdfsUrl);

        STORE.publish(EntityType.CLUSTER, cluster);

        Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource(FEED1_XML));
        Feed feed2 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource(FEED2_XML));
        Feed feed3 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
                this.getClass().getResource(FEED3_XML));

        STORE.publish(EntityType.FEED, feed1);
        STORE.publish(EntityType.FEED, feed2);
        STORE.publish(EntityType.FEED, feed3);
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
                coord.getOutputEvents().getDataOut().get(0).getDataset());

        assertEquals(6, coord.getDatasets().getDatasetOrAsyncDataset().size());

        ConfigurationStore store = ConfigurationStore.get();
        Feed feed = store.get(EntityType.FEED, process.getInputs().getInputs().get(0).getFeed());
        SYNCDATASET ds = (SYNCDATASET) coord.getDatasets().getDatasetOrAsyncDataset().get(0);
        assertEquals(SchemaHelper.formatDateUTC(feed.getClusters().getClusters().get(0).getValidity().getStart()),
                ds.getInitialInstance());
        assertEquals(feed.getTimezone().getID(), ds.getTimezone());
        assertEquals("${coord:" + feed.getFrequency().toString() + "}", ds.getFrequency());
        assertEquals("", ds.getDoneFlag());
        assertEquals(ds.getUriTemplate(), "${nameNode}" + FeedHelper.getLocation(feed, LocationType.DATA,
                feed.getClusters().getClusters().get(0).getName()).getPath());
        for (Property prop : coord.getAction().getWorkflow().getConfiguration().getProperty()) {
            if (prop.getName().equals("mapred.job.priority")) {
                assertEquals(prop.getValue(), "LOW");
                break;
            }
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
        validateFeedGroups(feed);

        // Seems like a good enough entity object for a new one
        // But is this an update ?

        Feed oldFeed = ConfigurationStore.get().get(EntityType.FEED, feed.getName());
        if (oldFeed == null) {
            return; // Not an update case
        }

        // Is actually an update. Need to iterate over all the processes
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        return propertiesMap;
    }

    private void addOptionalInputProperties(Properties properties, Input in, String clusterName)
        throws FalconException {
        Feed feed = EntityUtil.getEntity(EntityType.FEED, in.getFeed());
        org.apache.falcon.entity.v0.feed.Cluster cluster = FeedHelper.getCluster(feed, clusterName);
        String inName = in.getName();
        properties.put(inName + ".frequency", String.valueOf(feed.getFrequency().getFrequency()));
        properties.put(inName + ".freq_timeunit", mapToCoordTimeUnit(feed.getFrequency().getTimeUnit()).name());
        properties.put(inName + ".timezone", feed.getTimezone().getID());
        properties.put(inName + ".end_of_duration", Timeunit.NONE.name());
        properties.put(inName + ".initial-instance", SchemaHelper.formatDateUTC(cluster.getValidity().getStart()));
        properties.put(inName + ".done-flag", "notused");

        String locPath = FeedHelper.getLocation(feed, LocationType.DATA, clusterName).getPath().replace('$', '%');
View Full Code Here
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
        return joinedStr;
    }

    private SYNCDATASET createDataSet(String feedName, Cluster cluster, String datasetName,
                                      LocationType locationType) throws FalconException {
        Feed feed = EntityUtil.getEntity(EntityType.FEED, feedName);

        SYNCDATASET syncdataset = new SYNCDATASET();
        syncdataset.setName(datasetName);
        String locPath = FeedHelper.getLocation(feed, locationType,
                cluster.getName()).getPath();
        syncdataset.setUriTemplate(new Path(locPath).toUri().getScheme() != null ? locPath : "${nameNode}"
                + locPath);
        syncdataset.setFrequency("${coord:" + feed.getFrequency().toString() + "}");

        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
        syncdataset.setInitialInstance(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
        syncdataset.setTimezone(feed.getTimezone().getID());
        if (feed.getAvailabilityFlag() == null) {
            syncdataset.setDoneFlag("");
        } else {
            syncdataset.setDoneFlag(feed.getAvailabilityFlag());
        }
        return syncdataset;
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
        ClientResponse response = context.service.path("api/entities/definition/feed/"
                + context.outputFeedName).header(
                "Remote-User", TestContext.REMOTE_USER)
                .accept(MediaType.TEXT_XML).get(ClientResponse.class);
        Feed feed = (Feed) EntityType.FEED.getUnmarshaller()
                .unmarshal(new StringReader(response.getEntity(String.class)));

        //change output feed path and update feed as another user
        feed.getLocations().getLocations().get(0).setPath("/falcon/test/output2/${YEAR}/${MONTH}/${DAY}");
        tmpFile = context.getTempFile();
        EntityType.FEED.getMarshaller().marshal(feed, tmpFile);
        response = context.service.path("api/entities/update/feed/"
                + context.outputFeedName).header("Remote-User",
                "testuser").accept(MediaType.TEXT_XML)
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
437
                .header("Remote-User", TestContext.REMOTE_USER)
                .accept(MediaType.TEXT_XML).get(ClientResponse.class);

        String feedXML = response.getEntity(String.class);
        try {
            Feed result = (Feed) context.unmarshaller.
                    unmarshal(new StringReader(feedXML));
            Assert.assertEquals(result.getName(), overlay.get("inputFeedName"));
        } catch (JAXBException e) {
            Assert.fail("Reponse " + feedXML + " is not valid", e);
        }
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    @Test
    public void testFeedUpdateWithOneDependentProcess() {
        try {
            ConfigurationStore.get().remove(EntityType.FEED, "clicks");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample");
            Feed feed = parser.parseAndValidate(this.getClass()
                    .getResourceAsStream(FEED_XML));
            ConfigurationStore.get().publish(EntityType.FEED, feed);
            storeEntity(EntityType.PROCESS, "sample");

            //Try parsing the same feed xml
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
        try {
            ConfigurationStore.get().remove(EntityType.FEED, "clicks");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample2");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample3");
            Feed feed = parser.parseAndValidate(this.getClass()
                    .getResourceAsStream(FEED_XML));
            ConfigurationStore.get().publish(EntityType.FEED, feed);
            storeEntity(EntityType.PROCESS, "sample");
            storeEntity(EntityType.PROCESS, "sample2");
            storeEntity(EntityType.PROCESS, "sample3");
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.v0.feed.Feed

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.cleanup.ProcessCleanupHandler
  • org.apache.falcon.cluster.util.EmbeddedCluster
  • org.apache.falcon.converter.AbstractTestBase
  • org.apache.falcon.converter.OozieFeedMapper
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.