Examples of com.cloudera.flume.handlers.debug.NoNlASCIISynthSource

Package com.cloudera.flume.handlers.debug

Examples of com.cloudera.flume.handlers.debug.NoNlASCIISynthSource

com.cloudera.flume.handlers.debug.NoNlASCIISynthSource
This converts all of the body data to bytes that are ascii printable with no newlines. Good for testing and generating arbitrary workloads

      new Thread() {
        @Override
        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.
            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);
            src.open();
            snk.open();


            start.await();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {

View Full Code Here

      new Thread() {
        @Override
        public void run() {
          start.countDown();
          try {
            EventSource src = new NoNlASCIISynthSource(count, 100);
            Context ctx = new ReportTestingContext();
            EventSink snk = new CompositeSink(ctx, "counter(\"total." + idx
                + "\")");
            // use the same wal, but different counter.


            snk = new NaiveFileWALDeco(ctx, snk, wal, new TimeTrigger(
                new ProcessTagger(), 1000000), new AckListener.Empty(), 1000000);


            start.await();


            // allow for contention on the open call.
            src.open();
            snk.open();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close();
          } catch (Exception e) {
            LOG.error("failure", e);
            // fail("e");
          } finally {

View Full Code Here

    final EventSink sink = new LazyOpenDecorator<EventSink>(insistent);
    sink.open();


    // create an endless stream of data
    final EventSource source = new LazyOpenSource<EventSource>(
        new NoNlASCIISynthSource(0, 100));
    source.open();


    DirectDriver driver = new DirectDriver(source, sink);
    driver.start();
    Clock.sleep(1000); // let the insistent open try a few times.

View Full Code Here

    final EventSink sink = new LazyOpenDecorator<EventSink>(append);
    sink.open();


    // create an endless stream of data
    final EventSource source = new LazyOpenSource<EventSource>(
        new NoNlASCIISynthSource(0, 100));
    source.open();


    DirectDriver driver = new DirectDriver(source, sink);
    driver.start();
    Clock.sleep(1000); // let the insistent open try a few times.

View Full Code Here

                .testingContext(), dfoMan, new TimeTrigger(100), 50);


            ReportManager.get().add(cnt1);
            // make each parallel instance send a slightly different number of
            // messages.
            EventSource src = new NoNlASCIISynthSource(events + idx, 100);


            src.open();
            snk.open();


            started.countDown();


            EventUtil.dumpAll(src, snk);
            src.close();
            snk.close(); // this triggers a flush of current file!?
            FileUtil.rmr(f1);
          } catch (Exception e) {
            LOG.error(e, e);
          } finally {

View Full Code Here

   * 
   * @throws InterruptedException
   */
  @Test
  public void testAvroSend() throws IOException, InterruptedException {
    EventSource txt = new NoNlASCIISynthSource(25, 100);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();


    FlumeConfiguration conf = FlumeConfiguration.get();
    final AvroEventSource tes = new AvroEventSource(conf.getCollectorPort() + 1);
    tes.open();

View Full Code Here

    return new HashMap<String, EventSource>() {
      {
        // 1337 is the rand seed.


        // this is *really* slow
        put("100000,10,0,0,0", new NoNlASCIISynthSource(100000, 10, 1337));
        put("100000,100,0,0,0", new NoNlASCIISynthSource(100000, 100, 1337));
        put("100000,1000,0,0,0", new NoNlASCIISynthSource(100000, 1000, 1337));
        put("100000,3000,0,0,0", new NoNlASCIISynthSource(100000, 3000, 1337));
        put("100000,10000,0,0,0", new NoNlASCIISynthSource(100000, 10000, 1337));
      }
    };
  }

View Full Code Here

    return synthInMem(1000000, 100, 1);
  }


  public static MemorySinkSource synthInMem(int count, int bodySz, int seed)
      throws IOException, InterruptedException {
    EventSource txt = new NoNlASCIISynthSource(count, bodySz, seed);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    return mem;
  }

View Full Code Here

 */
public class TestHadoopLogData implements ExampleData {


  @Test
  public void testLineCount() throws IOException, InterruptedException {
    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    src.open();


    SimpleRegexReporterBuilder b = new SimpleRegexReporterBuilder(getClass()
        .getClassLoader().getResource(HADOOP_REGEXES).getFile());
    Collection<RegexGroupHistogramSink> sinks = b.load();
    MultiReporter mr = new MultiReporter("apache_sinks", sinks);

View Full Code Here


    MultiGrepReporterSink<String> snk = new MultiGrepReporterSink<String>(
        "multi grep", aho);
    snk.open();


    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    src.open();


    EventUtil.dumpAll(src, snk);


    snk.append(new EventImpl(NPE.getBytes()));
    snk.append(new EventImpl(LOST.getBytes()));
    snk.append(new EventImpl(LOST.getBytes()));


    Histogram<String> h = snk.getHistogram();
    System.out.println(h);


    Assert.assertEquals(3, h.total());
    Assert.assertEquals(2, h.get(LOST));
    Assert.assertEquals(1, h.get(NPE));


    snk.close();
    src.close();


  }

View Full Code Here

0 1 2 3 4

TOP

Related Classes of com.cloudera.flume.handlers.debug.NoNlASCIISynthSource

com.cloudera.flume.agent.diskfailover.TestConcurrentDFOMan

com.cloudera.flume.agent.durability.TestConcurrentWALMan

com.cloudera.flume.agent.durability.TestNaiveFileWALManagerConcurrently

com.cloudera.flume.collector.TestCollectorSink

com.cloudera.flume.conf.SourceFactory.SourceBuilder

com.cloudera.flume.core.connector.TestDirectDriver

com.cloudera.flume.core.Event

com.cloudera.flume.FlumeBenchmarkHarness

com.cloudera.flume.handlers.avro.TestAvroSinks

com.cloudera.flume.handlers.thrift.TestThriftSinks

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.