Examples of org.apache.hadoop.fs.FileSystem.listStatus()

org.apache.hadoop.fs.FileSystem.listStatus()
List the statuses of the files/directories in the given path if the path is a directory. @param f given path @return the statuses of the files/directories in the given patch @throws IOException

      } else if (matches.length == 0) {
        errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
      } else {
        for (FileStatus globStat: matches) {
          if (globStat.isDir()) {
            for(FileStatus stat: fs.listStatus(globStat.getPath(),
                inputFilter)) {
              result.add(stat);
            }          
          } else {
            result.add(globStat);

View Full Code Here

    if (!inputIsDirectory) {
      input = maybeUncompressedPath(inputPath);
    } else {
      inputDirectoryPath = inputPath;
      FileSystem fs = inputPath.getFileSystem(getConf());
      FileStatus[] statuses = fs.listStatus(inputPath);
      inputDirectoryFiles = new String[statuses.length];


      for (int i = 0; i < statuses.length; ++i) {
        inputDirectoryFiles[i] = statuses[i].getPath().getName();
      }

View Full Code Here

      cluster = new MiniDFSCluster(conf, 4, true, null);
      fs = cluster.getFileSystem();
      final String fileName = "/srcdat";
      util.createFiles(fs, fileName);
      util.waitReplication(fs, fileName, (short)3);
      FileStatus[] stats = fs.listStatus(new Path(fileName));
      assertFalse(0==stats.length);
      final Path file = stats[0].getPath();
      long aTime = fs.getFileStatus(file).getAccessTime();
      Thread.sleep(2*precision);
      String outStr = runFsck(conf, 0, true, "/");

View Full Code Here

     * Delete job conf from the history folder.
     */
    static void deleteConfFiles() throws IOException {
      LOG.info("Cleaning up config files from the job history folder");
      FileSystem fs = new Path(LOG_DIR).getFileSystem(jtConf);
      FileStatus[] status = fs.listStatus(new Path(LOG_DIR), CONF_FILTER);
      for (FileStatus s : status) {
        LOG.info("Deleting conf file " + s.getPath());
        fs.delete(s.getPath(), false);
      }
    }

View Full Code Here

  public static String readOutput(Path outDir, Configuration conf) 
      throws IOException {
    FileSystem fs = outDir.getFileSystem(conf);
    StringBuffer result = new StringBuffer();


    Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
        new Utils.OutputFileUtils.OutputFilesFilter()));
    for (Path outputFile : fileList) {
      LOG.info("Path" + ": "+ outputFile);
      BufferedReader file = 
        new BufferedReader(new InputStreamReader(fs.open(outputFile)));

View Full Code Here

            "SUBMIT_JOB on queue " + queue));
        // check if the system directory gets cleaned up or not
        JobTracker jobtracker = miniMRCluster.getJobTrackerRunner().getJobTracker();
        Path sysDir = new Path(jobtracker.getSystemDir());
        FileSystem fs = sysDir.getFileSystem(conf);
        int size = fs.listStatus(sysDir).length;
        while (size > 1) { // ignore the jobtracker.info file
          System.out.println("Waiting for the job files in sys directory to be cleaned up");
          UtilsForTests.waitFor(100);
          size = fs.listStatus(sysDir).length;
        }

View Full Code Here

        FileSystem fs = sysDir.getFileSystem(conf);
        int size = fs.listStatus(sysDir).length;
        while (size > 1) { // ignore the jobtracker.info file
          System.out.println("Waiting for the job files in sys directory to be cleaned up");
          UtilsForTests.waitFor(100);
          size = fs.listStatus(sysDir).length;
        }
      }
    } finally {
      // tearDownCluster();
    }

View Full Code Here

                                  JobConf conf) throws IOException {
    FileSystem fs = outDir.getFileSystem(conf);
    StringBuffer result = new StringBuffer();
    {
      
      Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
          new Utils.OutputFileUtils.OutputFilesFilter()));


      for(int i=0; i < fileList.length; ++i) {
        LOG.info("File list[" + i + "]" + ": "+ fileList[i]);
        BufferedReader file =

View Full Code Here

        jobCleanupTasks.length == 2);
    assertTrue("Cleanup progress is " + runningJob.cleanupProgress()
        + " and not 1.0", runningJob.cleanupProgress() == 1.0);


    assertTrue("Job output directory doesn't exit!", fs.exists(outDir));
    FileStatus[] list = fs.listStatus(outDir, 
        new Utils.OutputFileUtils.OutputFilesFilter());
    assertTrue("Number of part-files is " + list.length + " and not "
        + numReduces, list.length == numReduces);


    // cleanup

View Full Code Here

    FileSystem fs = FileSystem.get(hconf);
    Path dir = new Path(store);
    if (!fs.exists(dir)) {
      throw new IOException("can not found path:" + store);
    }
    FileStatus[] filelist = fs.listStatus(dir);
  
    Long bytesRead = 0l;
    long maxsize = 1024l * 1024 * 1024 * 10;
  
    String[] result=null;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.