Examples of textFile()

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

100
101
102
103
104
105
106
107
108
109
110
      JavaPairRDD<LongWritable, DataInputRecord> source = ctx.hadoopFile(sourceFile.getPath(), HBINInputFormat.class, LongWritable.class, DataInputRecord.class);
      base = (JavaRDD<Indexed>) (JavaRDD) source.map(new Function<Tuple2<LongWritable, DataInputRecord>, DataInputRecord>() {
        public DataInputRecord call(Tuple2<LongWritable, DataInputRecord> pair) throws Exception {return pair._2;}
      });
    } else {
      JavaRDD<String> source = ctx.textFile(sourceFile.getCanonicalPath());
      base = source.map(new StringToIndexed("\\s*,\\s*"));
    }

    Glypher<G,I> glypher = new Glypher<G,I>(dataset.shaper,dataset.valuer);
    GlyphsetRDD<G, I> glyphs = new GlyphsetRDD<>(base.map(glypher), true, partition);
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

47
48
49
50
51
52
53
54
55
56
57
    String outFile= arg(args, "-out", null);
    String sparkhome = arg(args,  "-spark", System.getenv("SPARK_HOME"));
    String jars[] = arg(args, "-jars", "AR.jar:ARApp.jar:ARExt.jar").split(":");
   
    JavaSparkContext ctx = new JavaSparkContext(host, "Abstract-Rendering", sparkhome, jars);
    JavaRDD<String> source = ctx.textFile(inFile);
    JavaRDD<Indexed> base = source.map(new StringToIndexed("\\s*,\\s*"));
    Shaper<Indexed, Rectangle2D> shaper = new ToRect(.1, .1, false, 2, 3);
    Valuer<Indexed,Integer> valuer = new Valuer.Constant<Indexed,Integer>(1);

    GlyphsetRDD<Rectangle2D, Integer> glyphs = new GlyphsetRDD<>(base.map(new Glypher<>(shaper, valuer)));
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

68
69
70
71
72
73
74
75
76
77
78
    // Loads in input file. It should be in format of:
    //     URL         neighbor URL
    //     URL         neighbor URL
    //     URL         neighbor URL
    //     ...
    JavaRDD<String> lines = ctx.textFile(args[0], 1);

    // Loads all URLs from input file and initialize their neighbors.
    JavaPairRDD<String, Iterable<String>> links = lines.mapToPair(new PairFunction<String, String, String>() {
      @Override
      public Tuple2<String, String> call(String s) {
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

40
41
42
43
44
45
46
47
48
49
50
      System.exit(1);
    }

    SparkConf sparkConf = new SparkConf().setAppName("JavaWordCount");
    JavaSparkContext ctx = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = ctx.textFile(args[0], 1);

    JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
      @Override
      public Iterable<String> call(String s) {
        return Arrays.asList(SPACE.split(s));
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

109
110
111
112
113
114
115
116
117
118
119
      System.exit(1);
    }

    SparkConf sparkConf = new SparkConf().setAppName("JavaHdfsLR");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = sc.textFile(args[0]);
    JavaRDD<DataPoint> points = lines.map(new ParsePoint()).cache();
    int ITERATIONS = Integer.parseInt(args[1]);

    // Initialize w to a random value
    double[] w = new double[D];
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

102
103
104
105
106
107
108
109
110
111
112
  public static void main(String[] args) {

    SparkConf sparkConf = new SparkConf().setAppName("JavaLogQuery");
    JavaSparkContext jsc = new JavaSparkContext(sparkConf);

    JavaRDD<String> dataSet = (args.length == 1) ? jsc.textFile(args[0]) : jsc.parallelize(exampleApacheLogs);

    JavaPairRDD<Tuple3<String, String, String>, Stats> extracted = dataSet.mapToPair(new PairFunction<String, Tuple3<String, String, String>, Stats>() {
      @Override
      public Tuple2<Tuple3<String, String, String>, Stats> call(String s) {
        return new Tuple2<Tuple3<String, String, String>, Stats>(extractKey(s), extractStats(s));
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

57
58
59
60
61
62
63
64
65
66
67
    JavaSparkContext ctx = new JavaSparkContext(sparkConf);
    JavaSQLContext sqlCtx = new JavaSQLContext(ctx);

    System.out.println("=== Data source: RDD ===");
    // Load a text file and convert each line to a Java Bean.
    JavaRDD<Person> people = ctx.textFile("examples/src/main/resources/people.txt").map(
      new Function<String, Person>() {
        @Override
        public Person call(String line) {
          String[] parts = line.split(",");
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

71
72
73
74
75
76
77
78
79
80
81
    if (args.length == 5) {
      blocks = Integer.parseInt(args[4]);
    }

    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = sc.textFile(args[0]);

    JavaRDD<Rating> ratings = lines.map(new ParseRating());

    MatrixFactorizationModel model = ALS.train(ratings.rdd(), rank, iterations, 0.01, blocks);
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

56
57
58
59
60
61
62
63
64
65
66
      System.err.println("Usage: JavaLR <input_dir> <step_size> <niters>");
      System.exit(1);
    }
    SparkConf sparkConf = new SparkConf().setAppName("JavaLR");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = sc.textFile(args[0]);
    JavaRDD<LabeledPoint> points = lines.map(new ParsePoint()).cache();
    double stepSize = Double.parseDouble(args[1]);
    int iterations = Integer.parseInt(args[2]);

    // Another way to configure LogisticRegression
View Full Code Here

Examples of org.apache.spark.api.java.JavaSparkContext.textFile()

62
63
64
65
66
67
68
69
70
71
72
    if (args.length >= 4) {
      runs = Integer.parseInt(args[3]);
    }
    SparkConf sparkConf = new SparkConf().setAppName("JavaKMeans");
    JavaSparkContext sc = new JavaSparkContext(sparkConf);
    JavaRDD<String> lines = sc.textFile(inputFile);

    JavaRDD<Vector> points = lines.map(new ParsePoint());

    KMeansModel model = KMeans.train(points.rdd(), k, iterations, runs, KMeans.K_MEANS_PARALLEL());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.