Examples of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.conf.FlumeConfiguration
This is the configuration file for flume. It uses hadoop's xml schema for properties. The default configuration is in $FLUME_HOME/flume-conf.xml and is a singleton. NOTE: The logging system gets configuration information from this file and thus depends on this being initialized. So no logging events are allowed when this is being initialized. If logger is used, an infinite recursion happens and causes a stack overflow.

    try {
      tmpdir = FileUtil.mktempdir();
    } catch (Exception e) {
      Assert.fail("mk temp dir failed");
    }
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    mock = new MockMasterRPC();


    nodesMan = (nodesMan != null) ? nodesMan : new LogicalNodeManager(NetUtils
        .localhost());
    walMan = (walMan != null) ? walMan : new NaiveFileWALManager(new File(conf
        .getAgentLogsDir()));
    dfMan = (dfMan != null) ? dfMan : new NaiveFileFailoverManager(new File(
        conf.getAgentLogsDir()));
    colAck = (colAck != null) ? colAck : new CollectorAckListener(mock);
    liveman = (liveman != null) ? liveman : new LivenessManager(nodesMan, mock,
        walMan);


    node = new FlumeNode(NetUtils.localhost(), mock, nodesMan, walMan, dfMan,

View Full Code Here

    try {
      tmpdir = FileUtil.mktempdir();
    } catch (Exception e) {
      fail("mk temp dir failed");
    }
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    // This will register the FlumeNode with a MockMasterRPC so it doesn't go
    // across the network
    mock = new MockMasterRPC();
    node = new FlumeNode(mock, false /* starthttp */, false /* oneshot */);

View Full Code Here

   * 
   * @throws IOException
   */
  @Test
  public void testConnect() throws IOException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.MASTER_HEARTBEAT_SERVERS, "localhost:44444");
    conf.set(FlumeConfiguration.MASTER_HEARBEAT_RPC, "AVRO");
    MultiMasterRPC masterRPC = new MultiMasterRPC(conf, false);
    MockAvroServer server1 = new MockAvroServer();
    server1.serve(44444);
    runTest(masterRPC);
    server1.stop();


    conf.set(FlumeConfiguration.MASTER_HEARBEAT_RPC, "THRIFT");
    masterRPC = new MultiMasterRPC(conf, false);
    MockThriftServer server2 = new MockThriftServer();
    server2.setPort(44444);
    server2.serve();
    runTest(masterRPC);

View Full Code Here


    b.mark("local_disk_write done");


    sink.close();


    FlumeConfiguration conf = FlumeConfiguration.get();
    Path src = new Path(tmp.getAbsolutePath());
    Path dst = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = dst.getFileSystem(conf);
    hdfs.deleteOnExit(dst);

View Full Code Here

    mem.open();
    EventUtil.dumpAll(txt, mem);
    txt.close();
    b.mark("disk_loaded");


    FlumeConfiguration conf = FlumeConfiguration.get();
    Path path = new Path("hdfs://localhost/testfile");
    FileSystem hdfs = path.getFileSystem(conf);
    hdfs.deleteOnExit(path);


    Writer w = SequenceFile.createWriter(hdfs, conf, path,

View Full Code Here

   * Tries to connect to several servers in turn and compensate as masters fail.
   */
  @Test
  public void testConnect() throws TTransportException, IOException,
      InterruptedException {
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.MASTER_HEARTBEAT_SERVERS,
        "localhost:9999,localhost:56789,localhost:56790");
    conf.set(FlumeConfiguration.MASTER_HEARBEAT_RPC, "THRIFT");
    MultiMasterRPC masterRPC = new MultiMasterRPC(conf, false);
    MyThriftServer server1 = new MyThriftServer();
    server1.serve(56789);
    MyThriftServer server2 = new MyThriftServer();
    server2.serve(56790);

View Full Code Here

    try {
      tmpdir = FileUtil.mktempdir();
    } catch (Exception e) {
      Assert.fail("mk temp dir failed");
    }
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    // This will register the FlumeNode with a MockMasterRPC so it doesn't go
    // across the network
    MockMasterRPC mock = new MockMasterRPC();
    @SuppressWarnings("unused")

View Full Code Here

    try {
      tmpdir = FileUtil.mktempdir();
    } catch (Exception e) {
      Assert.fail("mk temp dir failed");
    }
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    // This will register the FlumeNode with a MockMasterRPC so it doesn't go
    // across the network
    mock = new MockMasterRPC();
    node = new FlumeNode(mock, false /* starthttp */, false /* oneshot */);

View Full Code Here

   */
  @Test
  public void testFlumeNodeReconnect() throws TTransportException, IOException,
      InterruptedException {


    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    Clock.resetDefault();
    // Set directory of webapps to build-specific dir
    cfg.set(FlumeConfiguration.WEBAPPS_PATH, "build/webapps");
    // Doesn't matter whether or not we use ZK - use memory for speed
    cfg.set(FlumeConfiguration.MASTER_STORE, "memory");


    FlumeNode node = new FlumeNode(cfg);
    node.start();


    // avoiding gossip ack manager until it shuts down cleanly.

View Full Code Here

   * 
   * @throws IOException
   */
  @Test
  public void testLogDirsCorrect() throws IOException {
    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    Clock.resetDefault();
    // Set directory of webapps to build-specific dir
    cfg.set(FlumeConfiguration.WEBAPPS_PATH, "build/webapps");
    // Doesn't matter whether or not we use ZK - use memory for speed
    cfg.set(FlumeConfiguration.MASTER_STORE, "memory");


    File tmpdir = FileUtil.mktempdir();
    cfg.set(FlumeConfiguration.AGENT_LOG_DIR_NEW, tmpdir.getAbsolutePath());


    FlumeMaster master = FlumeMaster.getInstance();
    FlumeNode node = new FlumeNode(cfg, "foo", new DirectMasterRPC(master),
        false, false);


    node.getAddDFOManager("foo").open();
    node.getAddWALManager("foo").open();


    File defaultDir = new File(new File(cfg.getAgentLogsDir()), node
        .getPhysicalNodeName());
    File walDir = new File(defaultDir, NaiveFileWALManager.WRITINGDIR);
    assertTrue(walDir.isDirectory());


    File dfoDir = new File(defaultDir, NaiveFileFailoverManager.WRITINGDIR);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of com.cloudera.flume.conf.FlumeConfiguration

com.cloudera.flume.agent.AgentSink

com.cloudera.flume.agent.diskfailover.DiskFailoverDeco

com.cloudera.flume.agent.durability.NaiveFileWALDeco

com.cloudera.flume.agent.FaultyFlumeLocalAgentOOME

com.cloudera.flume.agent.FlumeNode

com.cloudera.flume.agent.TestAckedWALDecorator

com.cloudera.flume.agent.TestAgentSink

com.cloudera.flume.agent.TestAvroMultiMasterRPC

com.cloudera.flume.agent.TestFlumeNode

com.cloudera.flume.agent.TestRPCMechanisms

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.