Package org.apache.falcon.entity.v0.feed

Examples of org.apache.falcon.entity.v0.feed.CatalogTable

175
176
177
178
179
180
181
182
183
184
185
        final List<Location> locations = getLocations(cluster, feed);
        if (locations != null) {
            return Storage.TYPE.FILESYSTEM;
        }

        final CatalogTable table = getTable(cluster, feed);
        if (table != null) {
            return Storage.TYPE.TABLE;
        }

        throw new FalconException("Both catalog and locations are not defined.");
View Full Code Here
237
238
239
240
241
242
243
244
245
246
                setStart(FeedHelper.getCluster(oldFeed,
                        process.getClusters().getClusters().get(0).getName()).getValidity().getStart());

        //Change location to table should trigger process update
        newFeed.setLocations(null);
        CatalogTable table = new CatalogTable();
        table.setUri("catalog:default:clicks-blah#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}");
        newFeed.setTable(table);
        Assert.assertFalse(UpdateHelper.shouldUpdate(oldFeed, newFeed, process, cluster));
    }
View Full Code Here
258
259
260
261
262
263
264
265
266
267
268
269
        newTableFeed.setGroups("newgroups");
        Assert.assertFalse(UpdateHelper.isEntityUpdated(oldTableFeed, newTableFeed, cluster));
        newTableFeed.setFrequency(Frequency.fromString("days(1)"));
        Assert.assertTrue(UpdateHelper.isEntityUpdated(oldTableFeed, newTableFeed, cluster));

        final CatalogTable table = new CatalogTable();
        table.setUri("catalog:default:clicks-blah#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}");
        newTableFeed.setTable(table);
        Assert.assertTrue(UpdateHelper.isEntityUpdated(oldTableFeed, newTableFeed, cluster));

        inputStream = getClass().getResourceAsStream("/config/process/process-table.xml");
        Process oldProcess = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(inputStream);
View Full Code Here
393
394
395
396
397
398
399
400
401
402
            Location location = new Location();
            location.setType(LocationType.DATA);
            location.setPath(uriTemplate);
            feed.getLocations().getLocations().add(location);
        } else {
            CatalogTable table = new CatalogTable();
            table.setUri(uriTemplate);
            feed.setTable(table);
        }
    }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        overlay.put("tableUri", TABLE_URI);

        String filePath = context.overlayParametersOverTemplate("/hive-table-feed.xml", overlay);
        InputStream stream = new FileInputStream(filePath);
        FeedEntityParser parser = (FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED);
        Feed feed = parser.parse(stream);
        Assert.assertNotNull(feed);

        final LateArrival lateArrival = new LateArrival();
        lateArrival.setCutOff(new Frequency("4", Frequency.TimeUnit.hours));
        feed.setLateArrival(lateArrival);

        StringWriter stringWriter = new StringWriter();
        Marshaller marshaller = EntityType.FEED.getMarshaller();
        marshaller.marshal(feed, stringWriter);
        System.out.println(stringWriter.toString());
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
            return null;
        }

        Cluster cluster = CONFIG_STORE.get(EntityType.CLUSTER, feedCluster.getName());
        Path bundlePath = new Path(ClusterHelper.getLocation(cluster, "staging"), EntityUtil.getStagingPath(feed));
        Feed feedClone = (Feed) feed.copy();
        EntityUtil.setStartDate(feedClone, clusterName, startDate);

        AbstractOozieEntityMapper<Feed> mapper = new OozieFeedMapper(feedClone);
        if (!mapper.map(cluster, bundlePath)) {
            return null;
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    @Override
    public void onAdd(Entity entity, boolean ignoreFailure) throws FalconException {

        if (entity.getEntityType().equals(EntityType.FEED)) {
            Feed feed = (Feed) entity;
            if (feed.getGroups() == null || feed.getGroups().equals("")) {
                return;
            }
            Set<FeedGroup> groupSet = getGroups(feed);
            addGroups(feed.getName(), groupSet);
        }

    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    }

    @Override
    public void onRemove(Entity entity) throws FalconException {
        if (entity.getEntityType().equals(EntityType.FEED)) {
            Feed feed = (Feed) entity;
            if (StringUtils.isEmpty(feed.getGroups())) {
                return;
            }
            String[] groups = feed.getGroups().split(",");
            for (String group : groups) {
                groupsMapping.get(group).getFeeds().remove(entity.getName());
                if (groupsMapping.get(group).getFeeds().size() == 0) {
                    groupsMapping.remove(group);
                }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
*/
public class TestLateRerunHandler {

    @Test
    public void testFeedCutOff() throws FalconException {
        Feed feed = new Feed();
        LateArrival lateArrival = new LateArrival();
        lateArrival.setCutOff(Frequency.fromString("days(1)"));
        feed.setLateArrival(lateArrival);
        String nm = "2013-10-01T12:00Z";
        Date cutOff = LateRerunHandler.getCutOffTime(feed, nm);
        Assert.assertEquals(EntityUtil.parseDateUTC("2013-10-02T12:00Z"), cutOff);

        lateArrival.setCutOff(Frequency.fromString("days(10000000)"));
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    @Test
    public void testFeedUpdateWithOneDependentProcess() {
        try {
            ConfigurationStore.get().remove(EntityType.FEED, "clicks");
            ConfigurationStore.get().remove(EntityType.PROCESS, "sample");
            Feed feed = parser.parseAndValidate(this.getClass()
                    .getResourceAsStream(FEED_XML));
            ConfigurationStore.get().publish(EntityType.FEED, feed);
            storeEntity(EntityType.PROCESS, "sample");

            //Try parsing the same feed xml
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.v0.feed.CatalogTable

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.cleanup.LogCleanupServiceTest
  • org.apache.falcon.cleanup.ProcessCleanupHandler
  • org.apache.falcon.cluster.util.EmbeddedCluster
  • org.apache.falcon.converter.AbstractTestBase
  • org.apache.falcon.converter.OozieFeedMapper
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.