Examples of org.apache.flink.core.fs.Path

org.apache.flink.core.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

  public DataSink<T> write(FileOutputFormat<T> outputFormat, String filePath, WriteMode writeMode) {
    Validate.notNull(filePath, "File path must not be null.");
    Validate.notNull(writeMode, "Write mode must not be null.");
    Validate.notNull(outputFormat, "Output format must not be null.");


    outputFormat.setOutputFilePath(new Path(filePath));
    outputFormat.setWriteMode(writeMode);
    return output(outputFormat);
  }

View Full Code Here

   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<String> readTextFile(String filePath) {
    Validate.notNull(filePath, "The file path may not be null.");
    
    return new DataSource<String>(this, new TextInputFormat(new Path(filePath)), BasicTypeInfo.STRING_TYPE_INFO );
  }

View Full Code Here

   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<String> readTextFile(String filePath, String charsetName) {
    Validate.notNull(filePath, "The file path may not be null.");


    TextInputFormat format = new TextInputFormat(new Path(filePath));
    format.setCharsetName(charsetName);
    return new DataSource<String>(this, format, BasicTypeInfo.STRING_TYPE_INFO );
  }

View Full Code Here

   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<StringValue> readTextFileWithValue(String filePath) {
    Validate.notNull(filePath, "The file path may not be null.");
    
    return new DataSource<StringValue>(this, new TextValueInputFormat(new Path(filePath)), new ValueTypeInfo<StringValue>(StringValue.class) );
  }

View Full Code Here

  private JobGraph getJobGraph(OptimizedPlan optPlan, List<File> jarFiles) {
    NepheleJobGraphGenerator gen = new NepheleJobGraphGenerator();
    JobGraph job = gen.compileJobGraph(optPlan);
    
    for (File jar : jarFiles) {
      job.addJar(new Path(jar.getAbsolutePath()));
    }
    
    return job;
  }

View Full Code Here

   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<StringValue> readTextFileWithValue(String filePath, String charsetName, boolean skipInvalidLines) {
    Validate.notNull(filePath, "The file path may not be null.");
    
    TextValueInputFormat format = new TextValueInputFormat(new Path(filePath));
    format.setCharsetName(charsetName);
    format.setSkipInvalidLines(skipInvalidLines);
    return new DataSource<StringValue>(this, format, new ValueTypeInfo<StringValue>(StringValue.class) );
  }

View Full Code Here

    }
    if (filePath == null) {
      throw new IllegalArgumentException("The file path must not be null.");
    }
    
    inputFormat.setFilePath(new Path(filePath));
    try {
      return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat));
    }
    catch (Exception e) {
      throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " +

View Full Code Here

      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);


      @SuppressWarnings("unchecked")
      CsvOutputFormat outFormat = new CsvOutputFormat("\n", " ", LongValue.class, LongValue.class, LongValue.class);
      outFormat.setOutputFilePath(new Path(resultPath));
      
      taskConfig.setStubWrapper(new UserCodeObjectWrapper<CsvOutputFormat>(outFormat));
    }


    return output;

View Full Code Here

      Class<F> inputFormatClass, String path, Configuration configuration)
    throws IOException
  {
    configuration = configuration == null ? new Configuration() : configuration;


    Path normalizedPath = normalizePath(new Path(path));
    final F inputFormat = ReflectionUtil.newInstance(inputFormatClass);


    inputFormat.setFilePath(normalizedPath);
    inputFormat.setOpenTimeout(0);
    inputFormat.configure(configuration);


    final FileSystem fs = FileSystem.get(normalizedPath.toUri());
    FileStatus fileStatus = fs.getFileStatus(normalizedPath);


    BlockLocation[] blocks = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    inputFormat.open(new FileInputSplit(0, new Path(path), 0, fileStatus.getLen(), blocks[0].getHosts()));
    return inputFormat;
  }

View Full Code Here

   *         if an I/O error occurred while accessing the files or initializing the InputFormat.
   */
  @SuppressWarnings("unchecked")
  public static <T, F extends FileInputFormat<T>> List<F> openAllInputs(
      Class<F> inputFormatClass, String path, Configuration configuration) throws IOException {
    Path nephelePath = new Path(path);
    FileSystem fs = nephelePath.getFileSystem();
    FileStatus fileStatus = fs.getFileStatus(nephelePath);
    if (!fileStatus.isDir()) {
      return Arrays.asList(openInput(inputFormatClass, path, configuration));
    }
    FileStatus[] list = fs.listStatus(nephelePath);

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.flink.core.fs.Path

org.apache.flink.api.avro.testjar.AvroExternalJarProgram

org.apache.flink.api.common.cache.DistributedCache

org.apache.flink.api.common.io.BinaryInputFormat

org.apache.flink.api.common.io.DelimitedInputFormat

org.apache.flink.api.common.io.DelimitedInputFormatTest

org.apache.flink.api.common.io.FileInputFormat

org.apache.flink.api.common.io.FileOutputFormat

org.apache.flink.api.common.io.FileOutputFormatTest

org.apache.flink.api.common.io.FormatUtil

org.apache.flink.api.common.io.GenericCsvInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.