Examples of org.apache.flume.Sink

org.apache.flume.Sink

A sink is connected to a Channel and consumes its contents, sending them to a configured destination that may vary according to the sink type.

Sinks can be grouped together for various behaviors using SinkGroup and SinkProcessor. They are polled periodically by a SinkRunner via the processor

Sinks are associated with unique names that can be used for separating configuration and working namespaces.

While the {@link Sink#process()} call is guaranteed to only be accessedby a single thread, other calls may be concurrently accessed and should thus be protected.
@see org.apache.flume.Channel @see org.apache.flume.SinkProcessor @see org.apache.flume.SinkRunner


  @Test
  public void testDuplicateCreate() {




    Sink avroSink1 = sinkFactory.create("avroSink1", "avro");
    Sink avroSink2 = sinkFactory.create("avroSink2", "avro");


    Assert.assertNotNull(avroSink1);
    Assert.assertNotNull(avroSink2);
    Assert.assertNotSame(avroSink1, avroSink2);
    Assert.assertTrue(avroSink1 instanceof AvroSink);
    Assert.assertTrue(avroSink2 instanceof AvroSink);


    Sink s1 = sinkFactory.create("avroSink1", "avro");
    Sink s2 = sinkFactory.create("avroSink2", "avro");


    Assert.assertSame(avroSink1, s1);
    Assert.assertSame(avroSink2, s2);
  }

View Full Code Here

    Assert.assertSame(avroSink2, s2);
  }


  private void verifySinkCreation(String name, String type, Class<?> typeClass)
    throws Exception {
    Sink sink = sinkFactory.create(name, type);
    Assert.assertNotNull(sink);
    Assert.assertTrue(typeClass.isInstance(sink));
  }

View Full Code Here

  }




  @Test
  public void testSinkRegistry() {
    Sink s1 = sinkFactory.create("s1", "avro");
    Map<Class<?>, Map<String, Sink>> sr =
        ((DefaultSinkFactory) sinkFactory).getRegistryClone();


    Assert.assertEquals(1, sr.size());
    Map<String, Sink> sinkMap = sr.get(AvroSink.class);
    Assert.assertNotNull(sinkMap);
    Assert.assertEquals(1, sinkMap.size());


    Sink sink = sinkMap.get("s1");
    Assert.assertNotNull(sink);
    Assert.assertSame(s1, sink);
  }

View Full Code Here

    if (sinkMap == null) {
      sinkMap = new HashMap<String, Sink>();
      sinks.put(sinkClass, sinkMap);
    }


    Sink sink = sinkMap.get(name);


    if (sink == null) {
      try {
        sink = sinkClass.newInstance();
        sink.setName(name);
        sinkMap.put(name,  sink);
      } catch (Exception ex) {
        // Clean up the sink map
        sinks.remove(sinkClass);
        throw new FlumeException("Unable to create sink: " + name

View Full Code Here

    sinkFactory = new DefaultSinkFactory();
  }


  private void verifySinkCreation(String name, String type,
      Class<?> typeClass) throws FlumeException {
    Sink sink = sinkFactory.create(name, type);
    Assert.assertNotNull(sink);
    Assert.assertTrue(typeClass.isInstance(sink));
  }

View Full Code Here

    sinkFactory = new DefaultSinkFactory();
  }


  private void verifySinkCreation(String name, String type,
      Class<?> typeClass) throws FlumeException {
    Sink sink = sinkFactory.create(name, type);
    Assert.assertNotNull(sink);
    Assert.assertTrue(typeClass.isInstance(sink));
  }

View Full Code Here

    Map<String, Sink> sinks = new HashMap<String, Sink>();
    for (String sinkName : sinkNames) {
      ComponentConfiguration comp = compMap.get(sinkName);
      if(comp != null) {
        SinkConfiguration config = (SinkConfiguration) comp;
        Sink sink = getSinkFactory().create(comp.getComponentName(),
            comp.getType());


        Configurables.configure(sink, config);


        sink.setChannel(conf.getChannels().get(config.getChannel()));
        sinks.put(comp.getComponentName(), sink);
      }
    }


    Map<String, Context> sinkContexts = agentConf.getSinkContext();
    for (String sinkName : sinkNames) {
      Context context = sinkContexts.get(sinkName);
      if(context != null) {
        Sink sink = getSinkFactory().create(sinkName, context.getString(
            BasicConfigurationConstants.CONFIG_TYPE));
        Configurables.configure(sink, context);


        sink.setChannel(conf.getChannels().get(context.getString(
            BasicConfigurationConstants.CONFIG_CHANNEL)));
        sinks.put(sinkName, sink);
      }
    }

View Full Code Here

      if(comp != null) {
        SinkGroupConfiguration groupConf = (SinkGroupConfiguration) comp;
        List<String> groupSinkList = groupConf.getSinks();
        List<Sink> groupSinks = new ArrayList<Sink>();
        for (String sink : groupSinkList) {
          Sink s = sinks.remove(sink);
          if (s == null) {
            String sinkUser = usedSinks.get(sink);
            if (sinkUser != null) {
              throw new InstantiationException(String.format(
                  "Sink %s of group %s already " +

View Full Code Here


    for (ComponentConfiguration comp : agentConf.getSinks()) {
      Context context = new Context();


      String type = comp.getConfiguration().get("type");
      Sink sink = getSinkFactory().create(type);
      if(sink == null) {
        throw new InstantiationException("Can't instantiate sink with type " + type + " (it's probably " +
          "unknown type)");
      }


      for (Entry<String, String> entry : comp.getConfiguration().entrySet()) {
        context.put(entry.getKey(), entry.getValue());
      }


      Configurables.configure(sink, context);


      sink.setChannel(conf.getChannels().get(
          comp.getConfiguration().get("channel")));
      conf.getSinkRunners().put(comp.getComponentName(),
          SinkRunner.forSink(sink));
    }
  }

View Full Code Here

    Configurables.configure(channel, new Context());


    Source generatorSource = new SequenceGeneratorSource();
    generatorSource.setChannel(channel);


    Sink nullSink = new NullSink();
    nullSink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(generatorSource));
    nodeManager.add(SinkRunner.forSink(nullSink));


    nodeManager.start();

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.flume.Sink

com.salesforce.phoenix.flume.TestPhoenixSink

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.sink.DefaultSinkFactory

org.apache.flume.sink.elasticsearch.TestElasticSearchSinkCreation

org.apache.flume.sink.hbase.TestHBaseSinkCreation

org.apache.flume.sink.LoadBalancingSinkProcessor

org.apache.flume.sink.TestDefaultSinkFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.