Package org.apache.pig.experimental.logical.relational.LogicalSchema

Examples of org.apache.pig.experimental.logical.relational.LogicalSchema.LogicalFieldSchema


        assertEquals(1, ls.getField(0).uid);
        assertEquals(4, ls.getField(1).uid);
        assertEquals(5, ls.getField(2).uid);
       
        LogicalSchema expected = new LogicalSchema();
        expected.addField(new LogicalFieldSchema("id", null, DataType.BYTEARRAY));
        expected.addField(new LogicalFieldSchema("s", null, DataType.BYTEARRAY));
        expected.addField(new LogicalFieldSchema("v", null, DataType.BYTEARRAY));
        assertTrue(expected.isEqual(ls));
       
       
        PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
       
View Full Code Here


        @Override
        public void visitLOLoad(LOLoad load) throws IOException {
            if( load.getSchema() != null ) {
                Map<Integer,Set<String>> annotation = new HashMap<Integer,Set<String>>();
                for( int i=0; i<load.getSchema().size(); i++) {
                    LogicalFieldSchema field = load.getSchema().getField(i);
                    if( inputUids.containsKey( field.uid ) ) {
                        annotation.put(i, inputUids.get( field.uid ) );
                    }
                }
                load.annotate(REQUIRED_MAPKEYS, annotation);
View Full Code Here

                }
            }
           
            // if type is primitive, just add to schema
            if (t != DataType.TUPLE && t != DataType.BAG) {
                LogicalFieldSchema f = new LogicalSchema.LogicalFieldSchema(alias, fieldSchema, t, exp.getUid());               
                schema.addField(f);
                continue;
            }
           
            // if flatten is set, set schema of tuple field to this schema
            if (flattenFlags[i]) {
                if (t == DataType.BAG) {
                    // if it is bag of tuples, get the schema of tuples
                    if (fieldSchema != null && fieldSchema.size() == 1
                        && fieldSchema.getField(0).type == DataType.TUPLE) {
                       
                        fieldSchema = fieldSchema.getField(0).schema;
                    }else {
                        fieldSchema = null;
                    }
                }
               
                if (fieldSchema != null) {
                    List<LogicalFieldSchema> ll = fieldSchema.getFields();
                    for(LogicalFieldSchema f: ll) {
                        LogicalFieldSchema nf = new LogicalSchema.LogicalFieldSchema(alias+"::"+f.alias, f.schema, f.type, f.uid);
                        schema.addField(nf);
                    }                              
                } else {
                    schema = null;
                    break;
                }
            } else {
                 LogicalFieldSchema f = new LogicalSchema.LogicalFieldSchema(alias, fieldSchema, t, exp.getUid());                
                 schema.addField(f)
            }                                                     
        }
        return schema;
    }
View Full Code Here

                fieldSchema = s.getField(((ProjectExpression)sourceExp).getColNum()).schema;
                alias = s.getField(((ProjectExpression)sourceExp).getColNum()).alias;
            }
        }
       
        return new LogicalFieldSchema(alias, fieldSchema, sourceType, sourceExp.getUid());
    }
View Full Code Here

                for( Integer key : keySet ) {
                    Collection<LogicalExpressionPlan> plans =
                        mExpressionPlans.get(key);

                    for( LogicalExpressionPlan plan : plans ) {
                        LogicalFieldSchema fieldSchema = getPlanSchema(plan);
                        // if any plan schema is null, that means we can't calculate
                        // further schemas so we bail out
                        if( fieldSchema == null ) {
                            schema = null;
                            return schema;
                        }
                        // Change the uid of this field
                        fieldSchema.uid = LogicalExpression.getNextUid();
                        keySchema.addField(fieldSchema);
                    }
                    // We only need fields from one input and not all
                    break;
                }
                groupKeySchema = new LogicalFieldSchema(GROUP_COL_NAME, keySchema, DataType.TUPLE,
                        LogicalExpression.getNextUid() );
            } else {
                // We sort here to maintain the correct order of inputs
                TreeSet<Integer> keySet = new TreeSet<Integer>();
                keySet.addAll( mExpressionPlans.keySet() );
                for( Integer key : keySet ) {
                    Collection<LogicalExpressionPlan> plans = mExpressionPlans.get(key);
                    for( LogicalExpressionPlan plan : plans ) {
                        groupKeySchema = getPlanSchema(plan);
                        // if any plan schema is null, that means we can't calculate
                        // further schemas so we bail out
                        if( groupKeySchema == null ) {
                            schema = null;
                            return schema;
                        }
                        // Change the uid of this field
                        groupKeySchema.alias = GROUP_COL_NAME;
                        groupKeySchema.uid = LogicalExpression.getNextUid();
                        break;
                    }
                    break;
                }
            }
        }
       
        fieldSchemaList.add( groupKeySchema );

        // Generate the Bag Schema
        int counter = 0;
        for (Operator op : inputs) {
            LogicalSchema inputSchema = ((LogicalRelationalOperator)op).getSchema();
            // the schema of one input is unknown, so the join schema is unknown, just return
            if (inputSchema == null) {
                schema = null;
                return schema;
            }
          
            // Check if we already have calculated Uid for this bag for given
            // input operator
            long bagUid = -1;
            if( generatedInputUids.containsKey(counter) ) {
                bagUid = generatedInputUids.get(counter);
            } else {
                bagUid = LogicalExpression.getNextUid();
                generatedInputUids.put( counter, bagUid );
            }
           
            LogicalFieldSchema newBagSchema = new LogicalFieldSchema(
                    ((LogicalRelationalOperator)op).getAlias(), inputSchema,
                    DataType.BAG, bagUid);

            fieldSchemaList.add( newBagSchema );
            counter ++;
View Full Code Here

TOP

Related Classes of org.apache.pig.experimental.logical.relational.LogicalSchema.LogicalFieldSchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.