Examples of org.apache.flume.Sink

org.apache.flume.Sink

A sink is connected to a Channel and consumes its contents, sending them to a configured destination that may vary according to the sink type.

Sinks can be grouped together for various behaviors using SinkGroup and SinkProcessor. They are polled periodically by a SinkRunner via the processor

Sinks are associated with unique names that can be used for separating configuration and working namespaces.

While the {@link Sink#process()} call is guaranteed to only be accessedby a single thread, other calls may be concurrently accessed and should thus be protected.
@see org.apache.flume.Channel @see org.apache.flume.SinkProcessor @see org.apache.flume.SinkRunner

     */
    for (String sinkName : sinkNames) {
      ComponentConfiguration comp = compMap.get(sinkName);
      if(comp != null) {
        SinkConfiguration config = (SinkConfiguration) comp;
        Sink sink = sinkFactory.create(comp.getComponentName(),
            comp.getType());
        try {
          Configurables.configure(sink, config);
          ChannelComponent channelComponent = channelComponentMap.
              get(config.getChannel());
          if(channelComponent == null) {
            String msg = String.format("Sink %s is not connected to a " +
                "channel",  sinkName);
            throw new IllegalStateException(msg);
          }
          sink.setChannel(channelComponent.channel);
          sinks.put(comp.getComponentName(), sink);
          channelComponent.components.add(sinkName);
        } catch (Exception e) {
          String msg = String.format("Sink %s has been removed due to an " +
              "error during configuration", sinkName);
          LOGGER.error(msg, e);
        }
      }
    }
    /*
     * Components which DO NOT have a ComponentConfiguration object
     * and use only Context
     */
    Map<String, Context> sinkContexts = agentConf.getSinkContext();
    for (String sinkName : sinkNames) {
      Context context = sinkContexts.get(sinkName);
      if(context != null) {
        Sink sink = sinkFactory.create(sinkName, context.getString(
            BasicConfigurationConstants.CONFIG_TYPE));
        try {
          Configurables.configure(sink, context);
          ChannelComponent channelComponent = channelComponentMap.
              get(context.getString(BasicConfigurationConstants.CONFIG_CHANNEL));
          if(channelComponent == null) {
            String msg = String.format("Sink %s is not connected to a " +
                "channel",  sinkName);
            throw new IllegalStateException(msg);
          }
          sink.setChannel(channelComponent.channel);
          sinks.put(sinkName, sink);
          channelComponent.components.add(sinkName);
        } catch (Exception e) {
          String msg = String.format("Sink %s has been removed due to an " +
              "error during configuration", sinkName);

View Full Code Here

      ComponentConfiguration comp = compMap.get(groupName);
      if(comp != null) {
        SinkGroupConfiguration groupConf = (SinkGroupConfiguration) comp;
        List<Sink> groupSinks = new ArrayList<Sink>();
        for (String sink : groupConf.getSinks()) {
          Sink s = sinks.remove(sink);
          if (s == null) {
            String sinkUser = usedSinks.get(sink);
            if (sinkUser != null) {
              throw new InstantiationException(String.format(
                  "Sink %s of group %s already " +

View Full Code Here


  @Test
  public void testDuplicateCreate() {




    Sink avroSink1 = sinkFactory.create("avroSink1", "avro");
    Sink avroSink2 = sinkFactory.create("avroSink2", "avro");


    Assert.assertNotNull(avroSink1);
    Assert.assertNotNull(avroSink2);
    Assert.assertNotSame(avroSink1, avroSink2);
    Assert.assertTrue(avroSink1 instanceof AvroSink);
    Assert.assertTrue(avroSink2 instanceof AvroSink);


    Sink s1 = sinkFactory.create("avroSink1", "avro");
    Sink s2 = sinkFactory.create("avroSink2", "avro");


    Assert.assertNotSame(avroSink1, s1);
    Assert.assertNotSame(avroSink2, s2);
  }

View Full Code Here

    Assert.assertNotSame(avroSink2, s2);
  }


  private void verifySinkCreation(String name, String type, Class<?> typeClass)
    throws Exception {
    Sink sink = sinkFactory.create(name, type);
    Assert.assertNotNull(sink);
    Assert.assertTrue(typeClass.isInstance(sink));
  }

View Full Code Here

        final Map<String, Sink> sinks = new HashMap<String, Sink>();
        for (final String sinkName : sinkNames) {
            final ComponentConfiguration comp = compMap.get(sinkName);
            if (comp != null) {
                final SinkConfiguration config = (SinkConfiguration) comp;
                final Sink sink = sinkFactory.create(comp.getComponentName(), comp.getType());


                Configurables.configure(sink, config);


                sink.setChannel(conf.getChannels().get(config.getChannel()));
                sinks.put(comp.getComponentName(), sink);
            }
        }


        final Map<String, Context> sinkContexts = agentConf.getSinkContext();
        for (final String sinkName : sinkNames) {
            final Context context = sinkContexts.get(sinkName);
            if (context != null) {
                final Sink sink = sinkFactory.create(sinkName,
                    context.getString(BasicConfigurationConstants.CONFIG_TYPE));
                Configurables.configure(sink, context);


                sink.setChannel(conf.getChannels().get(context.getString(BasicConfigurationConstants.CONFIG_CHANNEL)));
                sinks.put(sinkName, sink);
            }
        }


        loadSinkGroups(agentConf, sinks, conf);

View Full Code Here

            if (comp != null) {
                final SinkGroupConfiguration groupConf = (SinkGroupConfiguration) comp;
                final List<String> groupSinkList = groupConf.getSinks();
                final List<Sink> groupSinks = new ArrayList<Sink>();
                for (final String sink : groupSinkList) {
                    final Sink s = sinks.remove(sink);
                    if (s == null) {
                        final String sinkUser = usedSinks.get(sink);
                        if (sinkUser != null) {
                            throw new ConfigurationException(String.format(
                                "Sink %s of group %s already in use by group %s", sink, groupName, sinkUser));

View Full Code Here

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    generatorSource.setChannelProcessor(new ChannelProcessor(rcs));


    Sink nullSink = new NullSink();
    nullSink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(generatorSource));
    SinkProcessor processor = new DefaultSinkProcessor();
    List<Sink> sinks = new ArrayList<Sink>();
    sinks.add(nullSink);

View Full Code Here

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);


    source.setChannelProcessor(new ChannelProcessor(rcs));


    Sink sink = new NullSink();
    sink.setChannel(channel);


    nodeManager.add(SourceRunner.forSource(source));
    SinkProcessor processor = new DefaultSinkProcessor();
    List<Sink> sinks = new ArrayList<Sink>();
    sinks.add(sink);

View Full Code Here

    for (ComponentConfiguration comp : agentConf.getSinks()) {
      Context context = new Context();
      Map<String, String> componentConfig = comp.getConfiguration();




      Sink sink = getSinkFactory().create(comp.getComponentName(),
          componentConfig.get("type"));


      for (Entry<String, String> entry : comp.getConfiguration().entrySet()) {
        context.put(entry.getKey(), entry.getValue());
      }


      Configurables.configure(sink, context);


      sink.setChannel(conf.getChannels().get(
          componentConfig.get("channel")));
      sinks.put(comp.getComponentName(), sink);
    }


    loadSinkGroups(agentConf, sinks, conf);

View Full Code Here

      String groupSinkList = groupConf.get("sinks");
      StringTokenizer sinkTokenizer = new StringTokenizer(groupSinkList, " \t");
      List<Sink> groupSinks = new ArrayList<Sink>();
      while(sinkTokenizer.hasMoreTokens()) {
        String sinkName = sinkTokenizer.nextToken();
        Sink s = sinks.remove(sinkName);
        if(s == null) {
          String sinkUser = usedSinks.get(sinkName);
          if(sinkUser != null) {
            throw new InstantiationException(String.format(
                "Sink %s of group %s already " +

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.flume.Sink

com.salesforce.phoenix.flume.TestPhoenixSink

org.apache.flume.conf.file.JsonFileConfigurationProvider

org.apache.flume.conf.properties.PropertiesFileConfigurationProvider

org.apache.flume.node.AbstractConfigurationProvider

org.apache.flume.node.TestAbstractLogicalNodeManager

org.apache.flume.sink.DefaultSinkFactory

org.apache.flume.sink.elasticsearch.TestElasticSearchSinkCreation

org.apache.flume.sink.hbase.TestHBaseSinkCreation

org.apache.flume.sink.LoadBalancingSinkProcessor

org.apache.flume.sink.TestDefaultSinkFactory

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.