Examples of FileSplit

com.cloudera.iterativereduce.yarn.avro.generated.FileSplit
edu.uci.ics.hyracks.dataflow.std.file.FileSplit
org.apache.hadoop.mapred.FileSplit
A section of an input file. Returned by {@link InputFormat#getSplits(JobConf,int)} and passed to{@link InputFormat#getRecordReader(InputSplit,JobConf,Reporter)}.
org.apache.hadoop.mapreduce.lib.input.FileSplit
A section of an input file. Returned by {@link InputFormat#getSplits(JobContext)} and passed to{@link InputFormat#createRecordReader(InputSplit,TaskAttemptContext)}.
org.apache.hama.bsp.FileSplit

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

  public static class InputSplitDetailMapper
    extends Mapper<NullWritable, NullWritable, Text, LongWritable> {
    @Override
    protected void map(NullWritable key, NullWritable value, Context context) 
        throws IOException, InterruptedException {
      FileSplit split = (FileSplit)context.getInputSplit();
      context.write(new Text(split.getPath().toString()), 
          new LongWritable(split.getLength()));
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        }
        
        @Override
        public void initialize(InputSplit genericSplit, TaskAttemptContext context)
                throws IOException, InterruptedException {
            FileSplit split = (FileSplit) genericSplit;
            Configuration job = context.getConfiguration();
 
            start = split.getStart();
            end = start + split.getLength();
            final Path file = split.getPath();
 
            // open the file and seek to the start of the split
            FileSystem fs = file.getFileSystem(job);
            FSDataInputStream fileIn = fs.open(split.getPath());
        
            this.xmlLoaderBPIS = new XMLLoaderBufferedPositionedInputStream(fileIn);
        }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        }


        @Override
        public void initialize(InputSplit split, TaskAttemptContext context)
                throws IOException, InterruptedException {
            FileSplit fSplit = (FileSplit) split; 
            Path p = fSplit.getPath();
            location = p.toString();
            LOG.info("location: " + location);    
            conf = context.getConfiguration();
        }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit


      @Override
      public void map(LongWritable key, Text value, Context context)
      throws IOException, InterruptedException {
        if (filePath == null) {
          FileSplit split = (FileSplit) context.getInputSplit();
          filePath = split.getPath().toString();
        }
        String line = value.toString();
        StringTokenizer st = new StringTokenizer(line, " ");
        while (st.hasMoreElements()) {
          byte[] word = st.nextToken().getBytes();

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

  static class RandomInputFormat extends InputFormat<Text, LongWritable> {
    public List<InputSplit> getSplits(JobContext job) throws IOException {
      List<InputSplit> result = new ArrayList<InputSplit>();
      int numSplits = job.getConfiguration().getInt(NUM_MAPS_KEY, NUM_MAPS);
      for (int i = 0; i < numSplits; ++i) {
        result.add(new FileSplit(new Path("/tmp", "dummy-split-" + i), 0, 1, null));
      }
      return result;
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

  }


  private Path getCurrentFile(Context context) throws IOException {
    InputSplit split = context.getInputSplit();
    if (split != null) {
      FileSplit inputSplit = (FileSplit) split;
      Path path = inputSplit.getPath();
      return path.makeQualified(path.getFileSystem(context.getConfiguration()));
    }
    return null;
  }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

        long pos = 0;
        int n;
        try {
          while ((n = reader.readLine(key)) > 0) {
            String[] hosts = getStoreDirHosts(fs, path);
            splits.add(new FileSplit(path, pos, n, hosts));
            pos += n;
          }
        } finally {
          reader.close();
        }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

    private FSDataInputStream fileIn = null;
    private static InterSedes sedes = InterSedesFactory.getInterSedesInstance();


    public void initialize(InputSplit genericSplit, TaskAttemptContext context)
                    throws IOException {
        FileSplit split = (FileSplit) genericSplit;
        Configuration job = context.getConfiguration();
        start = split.getStart();
        end = start + split.getLength();
        final Path file = split.getPath();


        // open the file and seek to the start of the split
        FileSystem fs = file.getFileSystem(job);
        fileIn = fs.open(split.getPath());
        reader = new Reader(fileIn, fs.getFileStatus(file).getLen(), job);
        scanner = reader.createScannerByByteRange(start, split.getLength());
    }

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

  public static final int RECORD_3 = 0x03;
  private DataInputStream inData = null;


  public void initialize(InputSplit genericSplit,
                         TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();


    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    if (start != 0) {
        fileIn.seek(start);
    }
    in = new BufferedPositionedInputStream(fileIn, start);
    inData = new DataInputStream(in);

View Full Code Here

Examples of org.apache.hadoop.mapreduce.lib.input.FileSplit

  private DataInputStream inData = null;
  private static InterSedes sedes = InterSedesFactory.getInterSedesInstance();


  public void initialize(InputSplit genericSplit,
                         TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();


    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    if (start != 0) {
        fileIn.seek(start);
    }
    in = new BufferedPositionedInputStream(fileIn, start);
    inData = new DataInputStream(in);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.