Examples of LineageTracer


Examples of org.apache.pig.pen.util.LineageTracer

    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if(illustrator != null) {
            ExampleTuple tIn = (ExampleTuple) in;
            LineageTracer lineage = illustrator.getLineage();
            lineage.insert(tIn);
            if (!isTmpStore)
                illustrator.getEquivalenceClasses().get(eqClassIndex).add(tIn);
            illustrator.addData((Tuple) out);
        }
        return (Tuple) out;
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        if (illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            illustrator.addData(tOut);
            illustrator.getEquivalenceClasses().get(eqClassIndex).add(
                    (Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            for (int i = 0; i < data.length; i++) {
                lineageTracer.union(tOut, data[i]);
            }
            return tOut;
        } else {
            return (Tuple) out;
        }
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        // All customized packagers are introduced during MRCompilaition.
        // Illustrate happens before that, so we only have to focus on the basic
        // POPackage
        if (illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            boolean synthetic = false;
            if (illustrator.getEquivalenceClasses() == null) {
                LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                for (int i = 0; i < numInputs; ++i) {
                    IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                    equivalenceClasses.add(equivalenceClass);
                }
                illustrator.setEquivalenceClasses(equivalenceClasses, parent);
            }

            if (isDistinct()) {
                int count = 0;
                for (Tuple tmp : bags[0]){
                    count++;
                    if (!tmp.equals(tOut))
                        lineageTracer.union(tOut, tmp);
                }
                if (count > 1) // only non-distinct tuples are inserted into the
                    // equivalence class
                    illustrator.getEquivalenceClasses().get(eqClassIndex)
                    .add(tOut);
                illustrator.addData((Tuple) tOut);
                return (Tuple) tOut;
            }
            boolean outInEqClass = true;
            try {
                for (int i = 1; i < numInputs + 1; i++) {
                    DataBag dbs = (DataBag) ((Tuple) out).get(i);
                    Iterator<Tuple> iter = dbs.iterator();
                    if (dbs.size() <= 1 && outInEqClass) // all inputs have >= 2
                        // records
                        outInEqClass = false;
                    while (iter.hasNext()) {
                        Tuple tmp = iter.next();
                        // any of synthetic data in bags causes the output tuple
                        // to be synthetic
                        if (!synthetic && ((ExampleTuple) tmp).synthetic)
                            synthetic = true;
                        lineageTracer.union(tOut, tmp);
                    }
                }
            } catch (ExecException e) {
                // TODO better exception handling
                throw new RuntimeException("Illustrator exception :"
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

    public void init() throws IOException, InterruptedException {

        Map<Operator, DataBag> data = eg.getData();

        LineageTracer lineage = eg.getLineage();
        Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = eg.getLoToEqClassMap();
        for (Operator leaf : plan.getSinks()) {
            Lineage.put(leaf, lineage);
            AffinityGroups.put(leaf, eg.getEqClasses());
        }
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        if (continueTrimming) {
            try {

                continueTrimming = checkCompleteness(cg);
               
                LineageTracer lineage = null;
                // create affinity groups
                if (cg.getInputs(plan).size() == 1) {
                    lineage = eg.getLineage();
                    AffinityGroups.put(cg.getInputs(plan).get(0), eg.getEqClasses());
                    Lineage.put(cg.getInputs(plan).get(0), lineage);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

          throw new FrontendException("Exception: "+e.getMessage());
        }
    }

    private boolean checkCompleteness(LogicalRelationalOperator op) throws Exception {
        LineageTracer lineage = Lineage.get(op);
        Lineage.remove(op);

        Collection<IdentityHashSet<Tuple>> affinityGroups = AffinityGroups
                .get(op);
        AffinityGroups.remove(op);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

    }
   
    private Map<Operator, DataBag> getData(PhysicalPlan plan) throws PigException, IOException, InterruptedException
    {
        // get data on a physical plan possibly trimmed of one branch
        lineage = new LineageTracer();
        IllustratorAttacher attacher = new IllustratorAttacher(plan, lineage, MAX_RECORDS, poLoadToSchemaMap, pigContext);
        attacher.visit();
        if (oriLimitMap != null) {
            for (Map.Entry<LOLimit, Long> entry : oriLimitMap.entrySet()) {
                logToPhyMap.get(entry.getKey()).getIllustrator().setOriginalLimit(entry.getValue());
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        // All customized packagers are introduced during MRCompilaition.
        // Illustrate happens before that, so we only have to focus on the basic
        // POPackage
        if (illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            boolean synthetic = false;
            if (illustrator.getEquivalenceClasses() == null) {
                LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                for (int i = 0; i < numInputs; ++i) {
                    IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                    equivalenceClasses.add(equivalenceClass);
                }
                illustrator.setEquivalenceClasses(equivalenceClasses, parent);
            }

            if (isDistinct()) {
                int count = 0;
                for (Tuple tmp : bags[0]){
                    count++;
                    if (!tmp.equals(tOut))
                        lineageTracer.union(tOut, tmp);
                }
                if (count > 1) // only non-distinct tuples are inserted into the
                    // equivalence class
                    illustrator.getEquivalenceClasses().get(eqClassIndex)
                    .add(tOut);
                illustrator.addData((Tuple) tOut);
                return (Tuple) tOut;
            }
            boolean outInEqClass = true;
            try {
                for (int i = 1; i < numInputs + 1; i++) {
                    DataBag dbs = (DataBag) ((Tuple) out).get(i);
                    Iterator<Tuple> iter = dbs.iterator();
                    if (dbs.size() <= 1 && outInEqClass) // all inputs have >= 2
                        // records
                        outInEqClass = false;
                    while (iter.hasNext()) {
                        Tuple tmp = iter.next();
                        // any of synthetic data in bags causes the output tuple
                        // to be synthetic
                        if (!synthetic && ((ExampleTuple) tmp).synthetic)
                            synthetic = true;
                        lineageTracer.union(tOut, tmp);
                    }
                }
            } catch (ExecException e) {
                // TODO better exception handling
                throw new RuntimeException("Illustrator exception :"
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        if (illustrator != null) {
            ExampleTuple tOut = new ExampleTuple((Tuple) out);
            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            if (illustrator.getEquivalenceClasses() == null) {
                LinkedList<IdentityHashSet<Tuple>> equivalenceClasses = new LinkedList<IdentityHashSet<Tuple>>();
                for (int i = 0; i < numInputs; ++i) {
                    IdentityHashSet<Tuple> equivalenceClass = new IdentityHashSet<Tuple>();
                    equivalenceClasses.add(equivalenceClass);
View Full Code Here

Examples of org.apache.pig.pen.util.LineageTracer

        if(illustrator != null){
            ExampleTuple tOut = new ExampleTuple((Tuple)out);
            illustrator.addData((Tuple)out);
            illustrator.getEquivalenceClasses().get(eqClassIndex).add((Tuple)in);

            LineageTracer lineageTracer = illustrator.getLineage();
            lineageTracer.insert(tOut);
            return tOut;
        }
        return (Tuple) out;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.