Package org.apache.pig.newplan

Examples of org.apache.pig.newplan.PColFilterExtractor


                null,
                TestLoader.partFilter);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);

        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());

        String actual = extractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().
                getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "(((f5 >= 20) and (f2 == 'us')) and (f3 == 15))", actual);
View Full Code Here


                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);

        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());

        String actual = extractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "((age >= 20) and (f3 == 15))", actual);
    }
View Full Code Here

    throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();

        if(expPartFilterString == null) {
            Assert.assertEquals("Checking partition column filter:", null,
                    pColExtractor.getPColCondition());
        } else  {
            Assert.assertEquals("Checking partition column filter:",
                    expPartFilterString,
                    pColExtractor.getPColCondition().toString());
        }

        // The getExpression() in PColFilterExtractor was written to get the
        // pushdown filter expression and does not have support for columns of
        // type tuple or map as partition columns are expected to be of
        // primitive data type. But we are using the method in the tests for forming
        // trimmed filter after pushdown. So skip check in cases where we expect a
        // trimmed filter to have a map or tuple in the condition.
        if (!skipTrimmedFilterCheck) {
            if (expFilterString == null) {
                Assert.assertTrue("Check that filter can be removed:",
                        pColExtractor.isFilterRemovable());
            } else {
                String actual = pColExtractor
                        .getExpression(
                                (LogicalExpression) filter.getFilterPlan().getSources().get(0))
                        .toString();
                Assert.assertEquals("checking trimmed filter expression:", expFilterString, actual);
            }
View Full Code Here

    private void negativeTest(String query, List<String> partitionCols) throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        pColExtractor.visit();
        Assert.assertFalse(pColExtractor.canPushDown());
        Assert.assertNull(pColExtractor.getPColCondition());
    }
View Full Code Here

            int expectedErrorCode) throws Exception {
        PigServer pigServer = new PigServer( pc );
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
        PColFilterExtractor pColExtractor = new PColFilterExtractor(
                filter.getFilterPlan(), partitionCols);
        try {
            pColExtractor.visit();
        } catch(Exception e) {
            Assert.assertEquals("Checking if exception has right error code",
                    expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
            return;
        }
View Full Code Here

          // original filter plan only if the transformation is successful
          // (i.e. partition filter can be pushed down)
          LogicalExpressionPlan filterExpr = loFilter.getFilterPlan();
          LogicalExpressionPlan filterExprCopy = filterExpr.deepCopy();
         
          PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                  filterExprCopy, getMappedKeys( partitionKeys ) );
          pColFilterFinder.visit();
          Expression partitionFilter = pColFilterFinder.getPColCondition();
         
          if(partitionFilter != null) {
            // the column names in the filter may be the ones provided by
            // the user in the schema in the load statement - we may need
            // to replace them with partition column names as given by
            // LoadFunc.getSchema()
            updateMappedColNames(partitionFilter);
            try {
          loadMetadata.setPartitionFilter(partitionFilter);
        } catch (IOException e) {
          throw new FrontendException( e );
        }
            if(pColFilterFinder.isFilterRemovable()) { 
              currentPlan.removeAndReconnect( loFilter );
            } else {
                    loFilter.setFilterPlan(filterExprCopy);
                }
            }
View Full Code Here

          // original filter plan only if the transformation is successful
          // (i.e. partition filter can be pushed down)
          LogicalExpressionPlan filterExpr = loFilter.getFilterPlan();
          LogicalExpressionPlan filterExprCopy = filterExpr.deepCopy();
         
          PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
                  filterExprCopy, getMappedKeys( partitionKeys ) );
          pColFilterFinder.visit();
          Expression partitionFilter = pColFilterFinder.getPColCondition();
         
          if(partitionFilter != null) {
            // the column names in the filter may be the ones provided by
            // the user in the schema in the load statement - we may need
            // to replace them with partition column names as given by
            // LoadFunc.getSchema()
            updateMappedColNames(partitionFilter);
            try {
          loadMetadata.setPartitionFilter(partitionFilter);
        } catch (IOException e) {
          throw new FrontendException( e );
        }
            if(pColFilterFinder.isFilterRemovable()) { 
              currentPlan.removeAndReconnect( loFilter );
            } else {
                loFilter.setFilterPlan(filterExprCopy);
            }
          }
View Full Code Here

                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
       
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
       
        String actual = extractor.getExpression(
                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "((f5 >= 20) and (f3 == 15))", actual);
    }
View Full Code Here

                null,
                TestLoader.partFilter);
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
       
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
       
        String actual = extractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().
                getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "(((f5 >= 20) and (f2 == 'us')) and (f3 == 15))", actual);
View Full Code Here

                "((mrkt == 'us') and (srcid == 10))",
                TestLoader.partFilter.toString());
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
       
        PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
       
        String actual = extractor.getExpression(
                (LogicalExpression) filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:",
                "((age >= 20) and (f3 == 15))", actual);
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.PColFilterExtractor

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.