Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.JobContext

82
83
84
85
86
87
88
89
  /**
   * Test no max version configured.
   */
  @Test
  public void testNoMaxVersion() {
    JobContext job = new JobContext(new Configuration(), new JobID());
    assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
   * Check that the iterator configuration is getting stored in the Job conf correctly.
   */
  @SuppressWarnings("deprecation")
  @Test
  public void testSetIterator() {
    JobContext job = new JobContext(new Configuration(), new JobID());
   
    AccumuloInputFormat.setIterator(job, 1, "org.apache.accumulo.core.iterators.WholeRowIterator", "WholeRow");
    Configuration conf = job.getConfiguration();
    String iterators = conf.get("AccumuloInputFormat.iterators");
    assertEquals("1:org.apache.accumulo.core.iterators.WholeRowIterator:WholeRow", iterators);
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    assertEquals("1:org.apache.accumulo.core.iterators.WholeRowIterator:WholeRow", iterators);
  }
 
  @Test
  public void testAddIterator() {
    JobContext job = new JobContext(new Configuration(), new JobID());
   
    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
    IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
    iter.addOption("v1", "1");
    iter.addOption("junk", "\0omg:!\\xyzzy");
    AccumuloInputFormat.addIterator(job.getConfiguration(), iter);
   
    List<AccumuloIterator> list = AccumuloInputFormat.getIterators(job.getConfiguration());
   
    // Check the list size
    assertTrue(list.size() == 3);
   
    // Walk the list and make sure our settings are correct
    AccumuloIterator setting = list.get(0);
    assertEquals(1, setting.getPriority());
    assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
    assertEquals("WholeRow", setting.getIteratorName());
   
    setting = list.get(1);
    assertEquals(2, setting.getPriority());
    assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
    assertEquals("Versions", setting.getIteratorName());
   
    setting = list.get(2);
    assertEquals(3, setting.getPriority());
    assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
    assertEquals("Count", setting.getIteratorName());
   
    List<AccumuloIteratorOption> iteratorOptions = AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
    assertEquals(2, iteratorOptions.size());
    assertEquals("Count", iteratorOptions.get(0).getIteratorName());
    assertEquals("Count", iteratorOptions.get(1).getIteratorName());
    assertEquals("v1", iteratorOptions.get(0).getKey());
    assertEquals("1", iteratorOptions.get(0).getValue());
View Full Code Here
184
185
186
187
188
189
190
191
192
193
194
   * Test getting iterator settings for multiple iterators set
   */
  @SuppressWarnings("deprecation")
  @Test
  public void testGetIteratorSettings() {
    JobContext job = new JobContext(new Configuration(), new JobID());
   
    AccumuloInputFormat.setIterator(job, 1, "org.apache.accumulo.core.iterators.WholeRowIterator", "WholeRow");
    AccumuloInputFormat.setIterator(job, 2, "org.apache.accumulo.core.iterators.VersioningIterator", "Versions");
    AccumuloInputFormat.setIterator(job, 3, "org.apache.accumulo.core.iterators.CountingIterator", "Count");
   
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
   * Check that the iterator options are getting stored in the Job conf correctly.
   */
  @SuppressWarnings("deprecation")
  @Test
  public void testSetIteratorOption() {
    JobContext job = new JobContext(new Configuration(), new JobID());
    AccumuloInputFormat.setIteratorOption(job, "someIterator", "aKey", "aValue");
   
    Configuration conf = job.getConfiguration();
    String options = conf.get("AccumuloInputFormat.iterators.options");
    assertEquals(new String("someIterator:aKey:aValue"), options);
  }
View Full Code Here
233
234
235
236
237
238
239
240
241
242
243
   * Test getting iterator options for multiple options set
   */
  @SuppressWarnings("deprecation")
  @Test
  public void testGetIteratorOption() {
    JobContext job = new JobContext(new Configuration(), new JobID());
   
    AccumuloInputFormat.setIteratorOption(job, "iterator1", "key1", "value1");
    AccumuloInputFormat.setIteratorOption(job, "iterator2", "key2", "value2");
    AccumuloInputFormat.setIteratorOption(job, "iterator3", "key3", "value3");
   
View Full Code Here
264
265
266
267
268
269
270
271
272
273
274
  }
 
  @SuppressWarnings("deprecation")
  @Test
  public void testSetRegex() {
    JobContext job = new JobContext(new Configuration(), new JobID());
   
    String regex = ">\"*%<>\'\\";
   
    AccumuloInputFormat.setRegex(job, RegexType.ROW, regex);
   
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
   
    JobContext job = new JobContext(new Configuration(), new JobID());
    AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable2", new Authorizations());
    AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
    AccumuloInputFormat input = new AccumuloInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
    RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
    rr.initialize(ris, tac);
   
    TestMapper mapper = new TestMapper();
    Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), tac.getTaskAttemptID(), rr, null, null, null, ris);
    while (rr.nextKeyValue()) {
      mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), context);
    }
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  static TaskAttemptContext tac;
  static Path f = null;
 
  @Before
  public void setup() {
    job = new JobContext(new Configuration(), new JobID());
   
    Path file = new Path(System.getenv("ACCUMULO_HOME") + "/target/");
    f = new Path(file, "_temporary");
    job.getConfiguration().set("mapred.output.dir", file.toString());
   
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    insertList(bw, row1);
    insertList(bw, row2);
    insertList(bw, row3);
    bw.close();
   
    JobContext job = new JobContext(new Configuration(), new JobID());
    AccumuloRowInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations());
    AccumuloRowInputFormat.setMockInstance(job.getConfiguration(), "instance1");
    AccumuloRowInputFormat crif = new AccumuloRowInputFormat();
    RangeInputSplit ris = new RangeInputSplit();
    TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
    RecordReader<Text,PeekingIterator<Entry<Key,Value>>> rr = crif.createRecordReader(ris, tac);
    rr.initialize(ris, tac);
   
    assertTrue(rr.nextKeyValue());
    assertEquals(new Text("row1"), rr.getCurrentKey());
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.JobContext

  • co.cask.cdap.hive.datasets.DatasetInputFormat
  • co.gridport.kafka.hadoop.FakeHadoopJob
  • com.atlantbh.hadoop.s3.io.S3InputFormat
  • com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs
  • com.datasalt.pangool.tuplemr.mapred.lib.output.ProxyOutputFormat$ProxyOutputCommitter
  • com.facebook.giraph.hive.impl.input.benchmark.InputBenchmark
  • com.facebook.hiveio.benchmark.InputBenchmark
  • com.facebook.hiveio.input.HiveInput
  • com.facebook.hiveio.output.CheckOutputSpecsTest
  • com.facebook.hiveio.output.FaultyCheckOutputSpecsTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.