Package org.apache.crunch.impl.spark

Examples of org.apache.crunch.impl.spark.SparkPipeline.done()

54
55
56
57
58
59
60
61
        return j.join(input.first(), j.join(input.second()));
      }
    }, strings()).materialize();
    assertEquals(ImmutableList.of("one,[-5,10],[1,1],[2,-3]", "three,[0,-1]", "two,[1,7],[2,6],[4,5]"),
        ImmutableList.copyOf(lines));
    p.done();
  }
}
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
    assertTrue(Iterables.isEmpty(p.emptyPCollection(Writables.strings())
        .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs()))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }

  @Test
  public void testUnionWithEmptyMR() throws Exception {
    Pipeline p = new SparkPipeline("local", "empty");
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
            p.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")))
                .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs())))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }

  @Test
  public void testUnionTableWithEmptyMR() throws Exception {
    Pipeline p = new SparkPipeline("local", "empty");
View Full Code Here
76
77
78
79
80
81
82
83
            p.read(From.textFile(tempDir.copyResourceFileName("shakes.txt")))
                .parallelDo(new SplitFn(), Writables.tableOf(Writables.strings(), Writables.longs())))
        .groupByKey()
        .combineValues(Aggregators.SUM_LONGS())
        .materialize()));
    p.done();
  }
}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.