Package org.apache.hadoop.mapreduce.v2

Examples of org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster

65
66
67
68
69
70
71
72
73
74
75
        + " not found. Not running test.");
      return;
    }

    if (mrCluster == null) {
      mrCluster = new MiniMRYarnCluster(getClass().getSimpleName());
      mrCluster.init(new Configuration());
      mrCluster.start();
    }
    // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
    // workaround the absent public discache.
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
            Configuration config = new Configuration();
            m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
            m_fileSys = m_dfs.getFileSystem();
            m_dfs_conf = m_dfs.getConfiguration(0);
           
            m_mr = new MiniMRYarnCluster("PigMiniCluster", taskTrackers);
            m_mr.init(m_dfs_conf);
            //m_mr.init(m_dfs_conf);
            m_mr.start();
           
            // Write the necessary config info to hadoop-site.xml
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
          + " not found. Not running test.");
      return;
    }

    if (mr == null) {
      mr = new MiniMRYarnCluster(TestMiniMRChildTask.class.getName());
      Configuration conf = new Configuration();
      mr.init(conf);
      mr.start();
    }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
    job.addFileToClassPath(appJar);
    String callerJar = JarFinder.getJar(caller);
    job.setJar(callerJar);

    MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller
        .getName(), noOfNMs);
    miniMRYarnCluster.init(job.getConfiguration());
    miniMRYarnCluster.start();

    return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
    fs = FileSystem.get(cconf);
    Assert.assertTrue(fs instanceof DistributedFileSystem);
    DistributedFileSystem dfs = (DistributedFileSystem) fs;

    JobClient client = null;
    MiniMRYarnCluster miniMRYarnCluster = null;
    try {
      // This will test RPC to the NameNode only.
      // could we test Client-DataNode connections?
      Path filePath = new Path("/dir");

      Assert.assertFalse(directDfs.exists(filePath));
      Assert.assertFalse(dfs.exists(filePath));

      directDfs.mkdirs(filePath);
      Assert.assertTrue(directDfs.exists(filePath));
      Assert.assertTrue(dfs.exists(filePath));

      // This will test RPC to a Resource Manager
      fs = FileSystem.get(sconf);
      JobConf jobConf = new JobConf();
      FileSystem.setDefaultUri(jobConf, fs.getUri().toString());
      miniMRYarnCluster = initAndStartMiniMRYarnCluster(jobConf);
      JobConf jconf = new JobConf(miniMRYarnCluster.getConfig());
      jconf.set("hadoop.rpc.socket.factory.class.default",
                "org.apache.hadoop.ipc.DummySocketFactory");
      jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
      String rmAddress = jconf.get("yarn.resourcemanager.address");
      String[] split = rmAddress.split(":");
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
      shutdownDFSCluster(cluster);
    }
  }

  private MiniMRYarnCluster initAndStartMiniMRYarnCluster(JobConf jobConf) {
    MiniMRYarnCluster miniMRYarnCluster;
    miniMRYarnCluster = new MiniMRYarnCluster(this.getClass().getName(), 1);
    miniMRYarnCluster.init(jobConf);
    miniMRYarnCluster.start();
    return miniMRYarnCluster;
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));
    job.addFileToClassPath(remoteCallerJar);

    MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller
        .getName(), noOfNMs);
    job.getConfiguration().set("minimrclientcluster.caller.name",
        caller.getName());
    job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
        noOfNMs);
    miniMRYarnCluster.init(job.getConfiguration());
    miniMRYarnCluster.start();

    return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
  }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
            m_dfs_conf = m_dfs.getConfiguration(0);
           
            //Create user home directory
            m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());

            m_mr = new MiniMRYarnCluster("PigMiniCluster", taskTrackers);
            m_mr.init(m_dfs_conf);
            //m_mr.init(m_dfs_conf);
            m_mr.start();

            // Write the necessary config info to hadoop-site.xml
View Full Code Here
235
236
237
238
239
240
241
242
243
244
245
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
    WaitForItHandler waitForItHandler = new WaitForItHandler();
   
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    JobId jobId =  TypeConverter.toYarn(
        TypeConverter.fromYarn(attemptid.getApplicationId()));
   
    WaitForItHandler waitForItHandler = new WaitForItHandler();
   
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileContext
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.ipc.TestMRCJCSocketFactory
  • org.apache.hadoop.ipc.TestSocketFactory
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.MiniMRClientClusterFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.