Examples of org.apache.hcatalog.data.schema.HCatSchema

org.apache.hcatalog.data.schema.HCatSchema
HCatSchema. This class is NOT thread-safe.

      }


      List<HCatFieldSchema> partKeys = new ArrayList<HCatFieldSchema>(2);
      partKeys.add(new HCatFieldSchema("emp_country", HCatFieldSchema.Type.STRING, ""));
      partKeys.add(new HCatFieldSchema("emp_state", HCatFieldSchema.Type.STRING, ""));
      partSchema = new HCatSchema(partKeys);


      runPartExport("237,Krishna,01/01/1990,M,IN,TN", "in", "tn");
      setUp();
      runPartExport("238,Kalpana,01/01/2000,F,IN,KA\n", "in", "ka");
      setUp();
      runPartExport("239,Satya,01/01/2001,M,US,TN\n", "us", "tn");
      setUp();
      runPartExport("240,Kavya,01/01/2002,F,US,KA\n", "us", "ka");


      setUp();
      setupMRImport();
      HCatEximInputFormat.setInput(job, "tmp/exports", null);


      List<HCatFieldSchema> colsPlusPartKeys = new ArrayList<HCatFieldSchema>();
      colsPlusPartKeys.addAll(columns);
      colsPlusPartKeys.addAll(partKeys);


      HCatBaseInputFormat.setOutputSchema(job, new HCatSchema(colsPlusPartKeys));
      job.waitForCompletion(true);


      assertEquals(4, TestImport.empRecords.size());
      assertEmpDetail(TestImport.empRecords.get(237), "Krishna", "01/01/1990", "M", "in", "tn");
      assertEmpDetail(TestImport.empRecords.get(238), "Kalpana", "01/01/2000", "F", "in", "ka");

View Full Code Here

      }


      List<HCatFieldSchema> partKeys = new ArrayList<HCatFieldSchema>(2);
      partKeys.add(new HCatFieldSchema("emp_country", HCatFieldSchema.Type.STRING, ""));
      partKeys.add(new HCatFieldSchema("emp_state", HCatFieldSchema.Type.STRING, ""));
      partSchema = new HCatSchema(partKeys);


      runPartExport("237,Krishna,01/01/1990,M,IN,TN", "in", "tn");
      setUp();
      runPartExport("238,Kalpana,01/01/2000,F,IN,KA\n", "in", "ka");
      setUp();

View Full Code Here


    storageDriver.setPartitionValues(context, partitionInfo.getPartitionValues());


    //Set the output schema. Use the schema given by user if set, otherwise use the
    //table level schema
    HCatSchema outputSchema = null;
    String outputSchemaString = context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_SCHEMA);
    if( outputSchemaString != null ) {
      outputSchema = (HCatSchema) HCatUtil.deserialize(outputSchemaString);
    } else {
      outputSchema = tableSchema;

View Full Code Here




  //test that new columns gets added to table schema
  private void tableSchemaTest() throws Exception {


    HCatSchema tableSchema = getTableSchema();


    assertEquals(3, tableSchema.getFields().size());


    //Update partition schema to have 3 fields
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", Constants.STRING_TYPE_NAME, "")));


    writeRecords = new ArrayList<HCatRecord>();


    for(int i = 0;i < 20;i++) {
      List<Object> objList = new ArrayList<Object>();


      objList.add(i);
      objList.add("strvalue" + i);
      objList.add("str2value" + i);


      writeRecords.add(new DefaultHCatRecord(objList));
    }


    Map<String, String> partitionMap = new HashMap<String, String>();
    partitionMap.put("part1", "p1value5");


    runMRCreate(partitionMap, partitionColumns, writeRecords, 10,true);


    tableSchema = getTableSchema();


    //assert that c3 has got added to table schema
    assertEquals(4, tableSchema.getFields().size());
    assertEquals("c1", tableSchema.getFields().get(0).getName());
    assertEquals("c2", tableSchema.getFields().get(1).getName());
    assertEquals("c3", tableSchema.getFields().get(2).getName());
    assertEquals("part1", tableSchema.getFields().get(3).getName());


    //Test that changing column data type fails
    partitionMap.clear();
    partitionMap.put("part1", "p1value6");

View Full Code Here

  }


  //check behavior while change the order of columns
  private void columnOrderChangeTest() throws Exception {


    HCatSchema tableSchema = getTableSchema();


    assertEquals(4, tableSchema.getFields().size());


    partitionColumns = new ArrayList<HCatFieldSchema>();
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", Constants.INT_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", Constants.STRING_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", Constants.STRING_TYPE_NAME, "")));

View Full Code Here

    HiveMetaStoreClient client = null;


    try {
      client = createHiveMetaClient(job.getConfiguration(),inputInfo);
      Table table = client.getTable(inputInfo.getDatabaseName(), inputInfo.getTableName());
      HCatSchema tableSchema = HCatUtil.getTableSchemaWithPtnCols(table);


      List<PartInfo> partInfoList = new ArrayList<PartInfo>();


      if( table.getPartitionKeys().size() != 0 ) {
        //Partitioned table

View Full Code Here


    return ptnKeyValues;
  }


  static PartInfo extractPartInfo(StorageDescriptor sd, Map<String,String> parameters) throws IOException{
    HCatSchema schema = HCatUtil.extractSchemaFromStorageDescriptor(sd);
    String inputStorageDriverClass = null;
    Properties hcatProperties = new Properties();
    if (parameters.containsKey(HCatConstants.HCAT_ISD_CLASS)){
      inputStorageDriverClass = parameters.get(HCatConstants.HCAT_ISD_CLASS);
    }else{

View Full Code Here

    // We also need to update the output Schema with these deletions.
    
    // Note that, output storage drivers never sees partition columns in data
    // or schema.


    HCatSchema schemaWithoutParts = new HCatSchema(schema.getFields());
    for(String partKey : partMap.keySet()){
      Integer idx;
      if((idx = schema.getPosition(partKey)) != null){
        posOfPartCols.add(idx);
        schemaWithoutParts.remove(schema.get(partKey));
      }
    }


    // Also, if dynamic partitioning is being used, we want to
    // set appropriate list of columns for the columns to be dynamically specified.
    // These would be partition keys too, so would also need to be removed from 
    // output schema and partcols


    if (jobInfo.getTableInfo().isDynamicPartitioningUsed()){
      for (String partKey : jobInfo.getTableInfo().getDynamicPartitioningKeys()){
        Integer idx;
        if((idx = schema.getPosition(partKey)) != null){
          posOfPartCols.add(idx);
          posOfDynPartCols.add(idx);
          schemaWithoutParts.remove(schema.get(partKey));
        }
      }
    }
    
    HCatUtil.validatePartitionSchema(jobInfo.getTable(), schemaWithoutParts);

View Full Code Here

  public void testCannotAddFieldMoreThanOnce() throws HCatException {
    List<HCatFieldSchema> fieldSchemaList = new ArrayList<HCatFieldSchema>();
    fieldSchemaList.add(new HCatFieldSchema("name", HCatFieldSchema.Type.STRING, "What's your handle?"));
    fieldSchemaList.add(new HCatFieldSchema("age", HCatFieldSchema.Type.INT, "So very old"));


    HCatSchema schema = new HCatSchema(fieldSchemaList);


    assertTrue(schema.getFieldNames().contains("age"));
    assertEquals(2, schema.getFields().size());


    try {
      schema.append(new HCatFieldSchema("age", HCatFieldSchema.Type.INT, "So very old"));
      fail("Was able to append field schema with same name");
    } catch(HCatException he) {
      assertTrue(he.getMessage().contains("Attempt to append HCatFieldSchema with already existing name: age."));
    }


    assertTrue(schema.getFieldNames().contains("age"));
    assertEquals(2, schema.getFields().size());


    // Should also not be able to add fields of different types with same name
    try {
      schema.append(new HCatFieldSchema("age", HCatFieldSchema.Type.STRING, "Maybe spelled out?"));
      fail("Was able to append field schema with same name");
    } catch(HCatException he) {
      assertTrue(he.getMessage().contains("Attempt to append HCatFieldSchema with already existing name: age."));
    }


    assertTrue(schema.getFieldNames().contains("age"));
    assertEquals(2, schema.getFields().size());
  }

View Full Code Here


      fieldSchemaList.add(new HCatFieldSchema("memberID", HCatFieldSchema.Type.INT, "as a number"));
      fieldSchemaList.add(new HCatFieldSchema("location", HCatFieldSchema.Type.STRING, "there's Waldo"));


      // No duplicate names.  This should be ok
      HCatSchema schema = new HCatSchema(fieldSchemaList);


      fieldSchemaList.add(new HCatFieldSchema("memberID", HCatFieldSchema.Type.STRING, "as a String"));


      // Now a duplicated field name.  Should fail
      try {
        HCatSchema schema2 = new HCatSchema(fieldSchemaList);
        fail("Able to add duplicate field name");
      } catch (IllegalArgumentException iae) {
        assertTrue(iae.getMessage().contains("Field named memberID already exists"));
      }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hcatalog.data.schema.HCatSchema

org.apache.giraph.io.hcatalog.GiraphHCatInputFormat

org.apache.hcatalog.common.HCatUtil

org.apache.hcatalog.common.TestHCatUtil

org.apache.hcatalog.data.JsonSerDe

org.apache.hcatalog.data.schema.TestHCatSchema

org.apache.hcatalog.data.schema.TestHCatSchemaUtils

org.apache.hcatalog.data.TestDefaultHCatRecord

org.apache.hcatalog.data.TestLazyHCatRecord

org.apache.hcatalog.hbase.HBaseHCatStorageHandler

org.apache.hcatalog.hbase.HBaseRevisionManagerUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.