Examples of com.cloudera.flume.core.CompositeSink

com.cloudera.flume.core.CompositeSink
This sink takes a data flow spec string as a constructor argument. It parses the construction and instantiates the sink. During execution, all the sink API calls are forwarded to the instantiated sink. Currently, there no builder method because this should only be used by sinks/decorators that need to embed sinks specs as arguments. In the future there may be due to different spec syntax.

        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent1);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 2\",\"collector 3\",\"collector 1\")",
        agent1);
    CompositeSink snk1 = new CompositeSink(new Context(), agent1);
    snk1.open();
    snk1.close();


    String agent2 = am.getFailChainSinkSpec("agent2",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent2);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 1\",\"collector 4\",\"collector 5\")",
        agent2);
    CompositeSink snk2 = new CompositeSink(new Context(), agent2);
    snk2.open();
    snk2.close();


    String agent3 = am.getFailChainSinkSpec("agent3",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent3);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 1\",\"collector 2\",\"collector 5\")",
        agent3);
    CompositeSink snk3 = new CompositeSink(new Context(), agent3);
    snk3.open();
    snk3.close();


  }

View Full Code Here

        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent1);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 5\",\"collector 1\",\"collector 3\")",
        agent1);
    CompositeSink snk1 = new CompositeSink(new Context(), agent1);
    snk1.open();
    snk1.close();


    String agent2 = am.getFailChainSinkSpec("agent2",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent2);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 2\",\"collector 1\",\"collector 3\")",
        agent2);
    CompositeSink snk2 = new CompositeSink(new Context(), agent2);
    snk2.open();
    snk2.close();


    String agent3 = am.getFailChainSinkSpec("agent3",
        "{lazyOpen => counter(\"%s\")}");
    LOG.info(agent3);
    assertEquals(
        "failChain(\"{lazyOpen => counter(\\\"%s\\\")}\",\"collector 5\",\"collector 1\",\"collector 2\")",
        agent3);
    CompositeSink snk3 = new CompositeSink(new Context(), agent3);
    snk3.open();
    snk3.close();


  }

View Full Code Here

  }


  @Test(expected = IllegalArgumentException.class)
  public void testRollRollConflict() throws IOException, FlumeSpecException,
      InterruptedException {
    EventSink snk = new CompositeSink(new Context(),
        "{value(\"rolltag\",\"foofoo\") =>   roll(10000) {null} } ");
    Event e = new EventImpl("foo".getBytes());
    snk.open();
    snk.append(e); // should bork.
  }

View Full Code Here

    snk.append(e); // should bork.
  }


  @Test(expected = IllegalArgumentException.class)
  public void testRollRollBork() throws IOException, FlumeSpecException, InterruptedException {
    EventSink snk = new CompositeSink(new Context(),
        "roll(10000) { roll(10000) { null } } ");
    Event e = new EventImpl("foo".getBytes());
    snk.open();
    snk.append(e); // should bork.
  }

View Full Code Here

    snk.append(e); // should bork.
  }


  @Test
  public void testRollRollNoConflict() throws IOException, FlumeSpecException, InterruptedException {
    EventSink snk = new CompositeSink(new Context(),
        "{value(\"rolltag\",\"foofoo\") =>  "
            + "{ mask(\"rolltag\")=>  roll(10000) { null}}} ");


    Event e = new EventImpl("foo".getBytes());
    snk.open();
    snk.append(e); // should not bork.
  }

View Full Code Here

    snk.append(e); // should not bork.
  }


  @Test
  public void testRollRollNoBork() throws IOException, FlumeSpecException, InterruptedException {
    EventSink snk = new CompositeSink(new Context(),
        "roll(10000) {{ mask(\"rolltag\") => roll(10000) {null} }} ");
    Event e = new EventImpl("foo".getBytes());
    snk.open();
    snk.append(e); // should not bork.
  }

View Full Code Here

  public void testAgentCollector() throws FlumeSpecException, IOException, InterruptedException {
    BenchmarkHarness.setupLocalWriteDir();
    File path = File.createTempFile("collector", ".tmp");
    path.deleteOnExit();


    EventSink snk = new CompositeSink(LogicalNodeContext.testingContext(),
        "{ ackedWriteAhead => roll(1000) { dfs(\"file:///"
            + StringEscapeUtils.escapeJava(path.getAbsolutePath()) + "\") } }");
    Event e = new EventImpl("foo".getBytes());
    snk.open();
    snk.append(e); // should not bork.
    snk.close();
    BenchmarkHarness.cleanupLocalWriteDir();
  }

View Full Code Here

      InterruptedException {
    BenchmarkHarness.setupLocalWriteDir();
    File path = File.createTempFile("collector", ".tmp");
    path.deleteOnExit();


    EventSink snk = new CompositeSink(LogicalNodeContext.testingContext(),
        "{ ackedWriteAhead => { mask(\"rolltag\") => roll(1000) { dfs(\"file:///"
            + StringEscapeUtils.escapeJava(path.getAbsolutePath())
            + "\") } } }");
    Event e = new EventImpl("foo".getBytes());
    snk.open();
    snk.append(e); // should not bork.
    snk.close();
    BenchmarkHarness.cleanupLocalWriteDir();
  }

View Full Code Here

  @Test
  public void testTailPermissionDenied() throws IOException,
      FlumeSpecException, InterruptedException {
    File f;
    final EventSource eventSource;
    final CompositeSink eventSink;
    final AtomicBoolean workerFailed;
    Thread workerThread;
    FileWriter writer;
    long sleepTime;
    long eventCount;


    f = File.createTempFile("temp", ".tmp");
    f.setReadable(false, false);


    f.deleteOnExit();


    eventSource = TailSource.builder().build(f.getAbsolutePath());
    eventSink = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    workerFailed = new AtomicBoolean(false);
    workerThread = new Thread() {


      @Override
      public void run() {
        try {
          eventSource.open();
          eventSink.open();


          EventUtil.dumpN(10, eventSource, eventSink);
          Clock.sleep(500);
          eventSource.close();
          eventSink.close();
        } catch (Exception e) {
          LOG.error("Unexpected exception", e);
        }
      }

View Full Code Here

  @Test
  public void testTailSource() throws IOException, FlumeSpecException,
      InterruptedException {
    File f = File.createTempFile("temp", ".tmp");
    f.deleteOnExit();
    final CompositeSink snk = new CompositeSink(new ReportTestingContext(),
        "{ delay(50) => counter(\"count\") }");
    final EventSource src = TailSource.builder().build(f.getAbsolutePath());
    final CountDownLatch done = new CountDownLatch(1);
    final int count = 30;
    runDriver(src, snk, done, count);

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of com.cloudera.flume.core.CompositeSink

com.cloudera.flume.agent.AgentFailChainSink

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.agent.LogicalNodeManager

com.cloudera.flume.agent.TestAgentFailChainSink

com.cloudera.flume.collector.CollectorSink

com.cloudera.flume.conf.Context

com.cloudera.flume.handlers.debug.BenchmarkReportDecorator

com.cloudera.flume.handlers.debug.BloomCheckDecorator

com.cloudera.flume.handlers.rolling.RollSink

com.cloudera.flume.handlers.rolling.TestRollRollTags

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.