Package org.apache.hcatalog.data.schema

Examples of org.apache.hcatalog.data.schema.HCatSchema

159
160
161
162
163
164
165
166
167
168
169
      // else - this means pig's optimizer never invoked the pushProjection
      // method - so we need all fields and hence we should not call the
      // setOutputSchema on HCatInputFormat
      if (HCatUtil.checkJobContextIfRunningFromBackend(job)) {
        try {
          HCatSchema hcatTableSchema = (HCatSchema) udfProps.get(HCatConstants.HCAT_TABLE_SCHEMA);
          outputSchema = hcatTableSchema;
          HCatInputFormat.setOutputSchema(job, outputSchema);
        } catch (Exception e) {
          throw new IOException(e);
        }
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
      .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);

    Table table = phutil.getTable(location,
      hcatServerUri != null ? hcatServerUri : PigHCatUtil.getHCatServerUri(job),
      PigHCatUtil.getHCatServerPrincipal(job));
    HCatSchema hcatTableSchema = HCatUtil.getTableSchemaWithPtnCols(table);
    try {
      PigHCatUtil.validateHCatTableSchemaFollowsPigRules(hcatTableSchema);
    } catch (IOException e) {
      throw new PigException(
        "Table schema incompatible for reading through HCatLoader :" + e.getMessage()
          + ";[Table schema was " + hcatTableSchema.toString() + "]"
        , PigHCatUtil.PIG_EXCEPTION_CODE, e);
    }
    storeInUDFContext(signature, HCatConstants.HCAT_TABLE_SCHEMA, hcatTableSchema);
    outputSchema = hcatTableSchema;
    return PigHCatUtil.getResourceSchema(hcatTableSchema);
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
      return null;
    }

    Properties props = UDFContext.getUDFContext().getUDFProperties(
      classForUDFCLookup, new String[]{signature});
    HCatSchema hcatTableSchema = (HCatSchema) props.get(HCatConstants.HCAT_TABLE_SCHEMA);

    ArrayList<HCatFieldSchema> fcols = new ArrayList<HCatFieldSchema>();
    for (RequiredField rf : fields) {
      fcols.add(hcatTableSchema.getFields().get(rf.getIndex()));
    }
    return new HCatSchema(fcols);
  }
View Full Code Here
406
407
408
409
410
411
412
413
414
415
416
    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
      HCatRecord record = new DefaultHCatRecord(3);
      HCatSchema schema = jobInfo.getOutputSchema();
      String vals[] = value.toString().split(",");
      record.setInteger("key", schema, Integer.parseInt(vals[0]));
      for (int i = 1; i < vals.length; i++) {
        String pair[] = vals[i].split(":");
        record.set(pair[0], schema, pair[1]);
View Full Code Here
451
452
453
454
455
456
457
458
459
460
461
    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
      OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO));
      HCatRecord record = new DefaultHCatRecord(3);
      HCatSchema schema = jobInfo.getOutputSchema();
      String vals[] = value.toString().split(",");
      record.setInteger("key", schema, Integer.parseInt(vals[0]));
      synchronized (MapWriteAbortTransaction.class) {
        if (count == 2) {
          failedKey = vals[0];
View Full Code Here
269
270
271
272
273
274
275
276
277
278
279
    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(DefaultHCatRecord.class);

    job.setNumReduceTasks(0);

    HCatOutputFormat.setSchema(job, new HCatSchema(partitionColumns));

    //new HCatOutputCommitter(null).setupJob(job);
    job.waitForCompletion(true);
    new HCatOutputCommitter(job,null).cleanupJob(job);
    if (assertWrite){
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
        //Handle duplicate publish
        handleDuplicatePublish(job, outputInfo, client, table);

        StorageDescriptor tblSD = table.getSd();
        HCatSchema tableSchema = HCatUtil.extractSchemaFromStorageDescriptor(tblSD);
        StorerInfo storerInfo = InitializeInput.extractStorerInfo(tblSD,table.getParameters());

        List<String> partitionCols = new ArrayList<String>();
        for(FieldSchema schema : table.getPartitionKeys()) {
          partitionCols.add(schema.getName());
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
        Constants.STRING_TYPE_NAME, "")));
    columns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("emp_dob",
        Constants.STRING_TYPE_NAME, "")));
    columns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("emp_sex",
        Constants.STRING_TYPE_NAME, "")));
    schema = new HCatSchema(columns);

    fs = new LocalFileSystem();
    fs.initialize(fs.getWorkingDirectory().toUri(), new Configuration());
    inputLocation = new Path(fs.getWorkingDirectory(), "tmp/exports");
    outputLocation = new Path(fs.getWorkingDirectory(), "tmp/data");
View Full Code Here
283
284
285
286
287
288
289
290
291
292
293
      readColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("emp_id",
            Constants.INT_TYPE_NAME, "")));
      readColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("emp_name",
            Constants.STRING_TYPE_NAME, "")));

      HCatEximInputFormat.setOutputSchema(job, new HCatSchema(readColumns));
      job.waitForCompletion(true);

      assertEquals(4, TestImport.empRecords.size());
      assertEmpDetail(TestImport.empRecords.get(237), "Krishna", null, null, null, null);
      assertEmpDetail(TestImport.empRecords.get(238), "Kalpana", null, null, null, null);
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
      }

      List<HCatFieldSchema> partKeys = new ArrayList<HCatFieldSchema>(2);
      partKeys.add(new HCatFieldSchema("emp_country", HCatFieldSchema.Type.STRING, ""));
      partKeys.add(new HCatFieldSchema("emp_state", HCatFieldSchema.Type.STRING, ""));
      partSchema = new HCatSchema(partKeys);

      runPartExport("237,Krishna,01/01/1990,M,IN,TN", "in", "tn");
      setUp();
      runPartExport("238,Kalpana,01/01/2000,F,IN,KA\n", "in", "ka");
      setUp();
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.data.schema.HCatSchema

  • org.apache.giraph.io.hcatalog.GiraphHCatInputFormat
  • org.apache.hcatalog.common.HCatUtil
  • org.apache.hcatalog.common.TestHCatUtil
  • org.apache.hcatalog.data.JsonSerDe
  • org.apache.hcatalog.data.schema.TestHCatSchema
  • org.apache.hcatalog.data.schema.TestHCatSchemaUtils
  • org.apache.hcatalog.data.TestDefaultHCatRecord
  • org.apache.hcatalog.data.TestLazyHCatRecord
  • org.apache.hcatalog.hbase.HBaseHCatStorageHandler
  • org.apache.hcatalog.hbase.HBaseRevisionManagerUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.