Package org.apache.hcatalog.data.schema

Examples of org.apache.hcatalog.data.schema.HCatSchema

156
157
158
159
160
161
162
163
164
165
166
167
168
            return null;
        }

        Properties props = UDFContext.getUDFContext().getUDFProperties(
            classForUDFCLookup, new String[]{signature});
        HCatSchema hcatTableSchema = (HCatSchema) props.get(HCatConstants.HCAT_TABLE_SCHEMA);

        ArrayList<HCatFieldSchema> fcols = new ArrayList<HCatFieldSchema>();
        for (RequiredField rf : fields) {
            fcols.add(hcatTableSchema.getFields().get(rf.getIndex()));
        }
        return new HCatSchema(fcols);
    }
View Full Code Here
162
163
164
165
166
167
168
            return result;
        }
    }

    public static HCatSchema extractSchema(Table table) throws HCatException {
        return new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
    }
View Full Code Here
166
167
168
169
170
171
172
    public static HCatSchema extractSchema(Table table) throws HCatException {
        return new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
    }

    public static HCatSchema extractSchema(Partition partition) throws HCatException {
        return new HCatSchema(HCatUtil.getHCatFieldSchemaList(partition.getCols()));
    }
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
        throws NoSuchObjectException, TException, MetaException {
        return new Table(client.getTable(dbName, tableName));
    }

    public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
        HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));

        if (table.getPartitionKeys().size() != 0) {

            // add partition keys to table schema
            // NOTE : this assumes that we do not ever have ptn keys as columns
            // inside the table schema as well!
            for (FieldSchema fs : table.getPartitionKeys()) {
                tableSchema.append(HCatSchemaUtils.getHCatFieldSchema(fs));
            }
        }
        return tableSchema;
    }
View Full Code Here
210
211
212
213
214
215
216
217
218
219
220
221
222
     * @param table the instance to extract partition columns from
     * @return HCatSchema instance which contains the partition columns
     * @throws IOException
     */
    public static HCatSchema getPartitionColumns(Table table) throws IOException {
        HCatSchema cols = new HCatSchema(new LinkedList<HCatFieldSchema>());
        if (table.getPartitionKeys().size() != 0) {
            for (FieldSchema fs : table.getPartitionKeys()) {
                cols.append(HCatSchemaUtils.getHCatFieldSchema(fs));
            }
        }
        return cols;
    }
View Full Code Here
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
            for (int i = 0; i < splits.length; i++) {
                if (!splits[i].equals(HBaseSerDe.HBASE_KEY_COL))
                    builder.append(splits[i]).append(" ");
            }
        } else {
            HCatSchema outputSchema = (HCatSchema) HCatUtil.deserialize(outputColSchema);
            HCatSchema tableSchema = tableInfo.getDataColumns();
            List<String> outputFieldNames = outputSchema.getFieldNames();
            List<Integer> outputColumnMapping = new ArrayList<Integer>();
            for (String fieldName : outputFieldNames) {
                int position = tableSchema.getPosition(fieldName);
                outputColumnMapping.add(position);
            }
            List<String> columnFamilies = new ArrayList<String>();
            List<String> columnQualifiers = new ArrayList<String>();
            HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies, null,
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
152
        // Explicitly use {@link org.apache.hadoop.hive.ql.metadata.Table} when getting the schema,
        // but store @{link org.apache.hadoop.hive.metastore.api.Table} as this class is serialized
        // into the job conf.
        org.apache.hadoop.hive.ql.metadata.Table mTable =
            new org.apache.hadoop.hive.ql.metadata.Table(table);
        HCatSchema schema = HCatUtil.extractSchema(mTable);
        StorerInfo storerInfo =
            InternalUtil.extractStorerInfo(table.getSd(), table.getParameters());
        HCatSchema partitionColumns = HCatUtil.getPartitionColumns(mTable);
        return new HCatTableInfo(table.getDbName(), table.getTableName(), schema,
            partitionColumns, storerInfo, table);
    }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
                }

                outputJobInfo.setPartitionValues(valueMap);
            }

            HCatSchema tableSchema = HCatUtil.extractSchema(table);
            StorerInfo storerInfo =
                InternalUtil.extractStorerInfo(table.getTTable().getSd(), table.getParameters());

            List<String> partitionCols = new ArrayList<String>();
            for (FieldSchema schema : table.getPartitionKeys()) {
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
        for (PartInfo partitionInfo : partitionInfoList) {
            jobConf = HCatUtil.getJobConfFromContext(jobContext);
            setInputPath(jobConf, partitionInfo.getLocation());
            Map<String, String> jobProperties = partitionInfo.getJobProperties();

            HCatSchema allCols = new HCatSchema(new LinkedList<HCatFieldSchema>());
            for (HCatFieldSchema field :
                inputJobInfo.getTableInfo().getDataColumns().getFields())
                allCols.append(field);
            for (HCatFieldSchema field :
                inputJobInfo.getTableInfo().getPartitionColumns().getFields())
                allCols.append(field);

            HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);

            storageHandler = HCatUtil.getStorageHandler(
                jobConf, partitionInfo);
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
    /**
     * gets values for fields requested by output schema which will not be in the data
     */
    private static Map<String, String> getColValsNotInDataColumns(HCatSchema outputSchema,
                                                                  PartInfo partInfo) {
        HCatSchema dataSchema = partInfo.getPartitionSchema();
        Map<String, String> vals = new HashMap<String, String>();
        for (String fieldName : outputSchema.getFieldNames()) {
            if (dataSchema.getPosition(fieldName) == null) {
                // this entry of output is not present in the output schema
                // so, we first check the table schema to see if it is a part col

                if (partInfo.getPartitionValues().containsKey(fieldName)) {
                    vals.put(fieldName, partInfo.getPartitionValues().get(fieldName));
View Full Code Here
TOP

Related Classes of org.apache.hcatalog.data.schema.HCatSchema

  • org.apache.giraph.io.hcatalog.GiraphHCatInputFormat
  • org.apache.hcatalog.common.HCatUtil
  • org.apache.hcatalog.common.TestHCatUtil
  • org.apache.hcatalog.data.JsonSerDe
  • org.apache.hcatalog.data.schema.TestHCatSchema
  • org.apache.hcatalog.data.schema.TestHCatSchemaUtils
  • org.apache.hcatalog.data.TestDefaultHCatRecord
  • org.apache.hcatalog.data.TestLazyHCatRecord
  • org.apache.hcatalog.hbase.HBaseHCatStorageHandler
  • org.apache.hcatalog.hbase.HBaseRevisionManagerUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.