Package org.apache.falcon.entity

Examples of org.apache.falcon.entity.Storage

788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
            return;
        }

        for (Output output : process.getOutputs().getOutputs()) {
            Feed feed = EntityUtil.getEntity(EntityType.FEED, output.getFeed());
            Storage storage = FeedHelper.createStorage(cluster, feed);

            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                final String outputName = output.getName();  // no prefix for backwards compatibility
                paramList.add(outputName + "=${" + outputName + "}");
            } else if (storage.getType() == Storage.TYPE.TABLE) {
                Map<String, String> props = new HashMap<String, String>();
                propagateCatalogTableProperties(output, (CatalogStorage) storage, props); // prefix is auto added
                for (String key : props.keySet()) {
                    paramList.add(key + "=${wf:conf('" + key + "')}");
                }
View Full Code Here
368
369
370
371
372
373
374
375
376
377
378
379
380
                    EntityUtil.getEntity(EntityType.CLUSTER, cluster.getName());
            if (!EntityUtil.responsibleFor(clusterEntity.getColo())) {
                continue;
            }

            final Storage storage = FeedHelper.createStorage(cluster, feed);
            // this is only true for table, filesystem always returns true
            if (storage.getType() == Storage.TYPE.FILESYSTEM) {
                continue;
            }

            CatalogStorage catalogStorage = (CatalogStorage) storage;
            String metaStorePrincipal = ClusterHelper.getPropertyValue(clusterEntity,
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
        LOG.info("Applying retention on " + feedPattern + " type: " + retentionType
                + ", Limit: " + retentionLimit + ", timezone: " + timeZone
                + ", frequency: " + frequency + ", storage" + feedStorageType);

        Storage storage = FeedHelper.createStorage(feedStorageType, feedPattern);
        evict(storage, retentionLimit, timeZone);

        logInstancePaths(new Path(logFile));

        int len = buffer.length();
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
219
        return feed.getName() + "/" + instanceStorage.toPartitionAsPath();
    }

    private String getFileSystemFeedInstanceName(String feedInstancePath, Feed feed,
                                                 Cluster cluster) throws FalconException {
        Storage rawStorage = FeedHelper.createStorage(cluster, feed);
        String feedPathTemplate = rawStorage.getUriTemplate(LocationType.DATA);
        String instance = feedInstancePath;

        String[] elements = FeedDataPath.PATTERN.split(feedPathTemplate);
        for (String element : elements) {
            instance = instance.replaceFirst(element, "");
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
     * @param retention retention limit
     * @throws FalconException
     */
    private void deleteStagedData(Cluster cluster, Feed feed, long retention)
        throws FalconException {
        Storage storage = FeedHelper.createStorage(cluster, feed);
        if (storage.getType() == Storage.TYPE.FILESYSTEM) {  // FS does NOT use staging dirs
            return;
        }

        final CatalogStorage tableStorage = (CatalogStorage) storage;
        String stagingDir = FeedHelper.getStagingDir(cluster, feed, tableStorage, Tag.REPLICATION);
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
207
            if (!StringUtils.equals(oldFeed.getAvailabilityFlag(), newFeed.getAvailabilityFlag())) {
                LOG.debug(oldFeed.toShortString() + ": Availability flag has changed. Updating...");
                return true;
            }

            Storage oldFeedStorage = FeedHelper.createStorage(cluster, oldFeed);
            Storage newFeedStorage = FeedHelper.createStorage(cluster, newFeed);

            if (!oldFeedStorage.isIdentical(newFeedStorage)) {
                LOG.debug(oldFeed.toShortString() + ": Storage has changed. Updating...");
                return true;
            }
View Full Code Here
TOP

Related Classes of org.apache.falcon.entity.Storage

  • org.apache.falcon.cleanup.FeedCleanupHandler
  • org.apache.falcon.converter.OozieFeedMapper$ReplicationOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapper$RetentionOozieWorkflowMapper
  • org.apache.falcon.converter.OozieFeedMapperTest
  • org.apache.falcon.converter.OozieFeedWorkflowBuilderTest
  • org.apache.falcon.converter.OozieProcessMapper
  • org.apache.falcon.entity.parser.FeedEntityParser
  • org.apache.falcon.metadata.InstanceRelationshipGraphBuilder
  • org.apache.falcon.retention.FeedEvictor
  • org.apache.falcon.update.UpdateHelper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.