Package com.ikanow.infinit.e.data_model.store.config.source

Examples of com.ikanow.infinit.e.data_model.store.config.source.SourcePipelinePojo

199
200
201
202
203
204
205
206
207
208
209
    // (note in practice this is only an issue for multiple threads going to the same domain)
    InfiniteFile f;
    synchronized (FileHarvester.class) {
      try {
        if (null != source.getProcessingPipeline()) { // new style...
          SourcePipelinePojo firstElement = source.getProcessingPipeline().iterator().next();
          source.setFileConfig(firstElement.file);
          source.setUrl(firstElement.file.getUrl());
        }//TESTED
        if (source.getUrl().startsWith("inf://")) { // Infinit.e share/custom object
          NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(source.getCommunityIds().iterator().next().toString(), source.getOwnerId().toString(), null);
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    if (context.isStandalone()) { // test mode
     
      // Get the configuration
     
      String logStashConfig = null;
      SourcePipelinePojo logstashElement = null;
      for (SourcePipelinePojo pxPipe: source.getProcessingPipeline()) { /// (must be non null if here)
        if (null != pxPipe.logstash) {
          logStashConfig = pxPipe.logstash.config;
          logstashElement = pxPipe;
        }
        break;
      }
      if ((null == logStashConfig) || logStashConfig.isEmpty()) {
        context.getHarvestStatus().update(source,new Date(),HarvestEnum.error, "Logstash internal logic error, no configuration", true, false);
        return;       
      }

      // Admin check (passed down)
     
      boolean isAdmin = AuthUtils.isAdmin(source.getOwnerId());

      // Perform the request
     
      ObjectId requestId = new ObjectId();
      BasicDBObject logQ = new BasicDBObject("_id", requestId);
      boolean removeJobWhenDone = true;
     
      try {

        // (See records.service for the programmatic definition of this message)
        logstashElement.logstash.config = logStashConfig;
        BasicDBObject logStashDbo = (BasicDBObject) logstashElement.toDb();
        logStashDbo.put("_id", requestId);
        logStashDbo.put("maxDocs", context.getStandaloneMaxDocs());
        logStashDbo.put("sourceKey", source.getKey());
        logStashDbo.put("isAdmin", isAdmin);
 
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
      if (!bIsPublic) { // Cleanse URLs, remove processing pipeline information
        source.setPartiallyPublished(true); //TESTED
       
        // Copy URL info from px pipeline into the main source
        if ((null != source.getProcessingPipeline()) && !source.getProcessingPipeline().isEmpty()) {         
          SourcePipelinePojo firstEl = source.getProcessingPipeline().iterator().next();
          if (null != firstEl.web) {
            source.setRssConfig(firstEl.web);
          }
          else if (null != firstEl.feed) {
            source.setRssConfig(firstEl.feed);
View Full Code Here
691
692
693
694
695
696
697
698
699
700
701
          logStashMessage.put("deleteOnlyCommunityId", communityId);
          logStashMessage.put("sourceKey", source.getKey());
          logStashMessage.put("deleteDocsOnly", bDocsOnly);
         
          if ((null != source.getProcessingPipeline()) && !source.getProcessingPipeline().isEmpty()) {
            SourcePipelinePojo px = source.getProcessingPipeline().iterator().next();
            if ((null != px.logstash) && (null != px.logstash.distributed) && px.logstash.distributed) {
              logStashMessage.put("distributed", true);
            }
          }//TESTED (by hand)
          DbManager.getIngest().getLogHarvesterQ().save(logStashMessage);
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
 
  public void enrichSource_processingPipeline(SourcePojo source, List<DocumentPojo> toAdd, List<DocumentPojo> toUpdate, List<DocumentPojo> toRemove)
  {
    Iterator<SourcePipelinePojo> pxPipeIt = source.getProcessingPipeline().iterator(); // (must be non null if here)
    while (pxPipeIt.hasNext()) {
      SourcePipelinePojo pxPipe = pxPipeIt.next();
        // (note can edit the iterator/list because this bit of the pipeline only runs once, not over each doc)
     
      // 1] Is this a starting point:
     
      if ((null != pxPipe.database) || (null != pxPipe.nosql) || (null != pxPipe.file) || (null != pxPipe.feed) || (null != pxPipe.web))
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
    // Ignore anything malformed:
    if (null == src.getProcessingPipeline() || src.getProcessingPipeline().isEmpty()) {
      setSourceError(src.getId(), "Internal logic error: no processing pipeline");
      return;
    }
    SourcePipelinePojo px = src.getProcessingPipeline().iterator().next();
    if ((null == px.logstash) || (null == px.logstash.config)) {
      setSourceError(src.getId(), "Internal logic error: no logstash block");
      return;
    }
View Full Code Here
TOP

Related Classes of com.ikanow.infinit.e.data_model.store.config.source.SourcePipelinePojo

  • com.ikanow.infinit.e.api.config.source.SourceHandler
  • com.ikanow.infinit.e.application.handlers.polls.LogstashHarvestPollHandler
  • com.ikanow.infinit.e.data_model.api.config.SourcePojoApiMap$SourcePojoSerializer
  • com.ikanow.infinit.e.harvest.extraction.document.file.FileHarvester
  • com.ikanow.infinit.e.harvest.extraction.document.logstash.LogstashHarvester
  • com.ikanow.infinit.e.harvest.HarvestControllerPipeline
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.