Package org.apache.flink.compiler.plan

Examples of org.apache.flink.compiler.plan.DualInputPlanNode

64
65
66
67
68
69
70
71
72
73
74
75
76
   
    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
     
      DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
      Channel in1 = crossPlanNode.getInput1();
      Channel in2 = crossPlanNode.getInput2();
     
      assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy());
      assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy());
    } catch(CompilerException ce) {
      ce.printStackTrace();
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
   
    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
     
      DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
      Channel in1 = crossPlanNode.getInput1();
      Channel in2 = crossPlanNode.getInput2();
     
      assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
    } catch(CompilerException ce) {
      ce.printStackTrace();
View Full Code Here
907
908
909
910
911
912
913
914
915
916
917
918
919
      toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin1.setLocalStrategy(LocalStrategy.NONE);
      toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin2.setLocalStrategy(LocalStrategy.NONE);
     
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
     
      FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
      assertEquals(NO_PARTIAL_SOLUTION, report);
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
      SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
     
      Channel toJoin1 = new Channel(map1);
      Channel toJoin2 = new Channel(map2);
     
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
     
      Channel toAfterJoin = new Channel(join);
      toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
     
      // attach some properties to the non-relevant input
      {
        toMap2.setShipStrategy(ShipStrategyType.BROADCAST);
        toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(2, 7), new boolean[] {true, true});
       
        RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
        joinGp.setFullyReplicated();
       
        RequestedLocalProperties joinLp = new RequestedLocalProperties();
        joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
       
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        toJoin2.setRequiredGlobalProps(joinGp);
        toJoin2.setRequiredLocalProps(joinLp);
      }
     
      // ------------------------------------------------------------------------------------
     
      // no properties from the partial solution, no required properties
      {
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.EMPTY;
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // some properties from the partial solution, no required properties
      {
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
     
      // produced properties match relevant input
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2));
       
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
       
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // produced properties do not match relevant input
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(1, 2, 3));
       
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
       
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
     
      // produced properties overridden before join
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
       
        toMap1.setRequiredGlobalProps(rgp);
        toMap1.setRequiredLocalProps(rlp);
       
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
       
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1));
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(MET, report);
      }
     
      // produced properties before join match, after join match as well
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
       
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
       
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
       
        toMap1.setRequiredGlobalProps(null);
        toMap1.setRequiredLocalProps(null);
       
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
       
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
     
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
       
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
       
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
     
      // produced properties before join match, after join do not match
      {
View Full Code Here
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
     
      Channel toJoin2 = new Channel(map2);
      toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin2.setLocalStrategy(LocalStrategy.NONE);
     
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
     
      Channel toAfterJoin = new Channel(join);
      toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
      Plan plan = getJavaTestPlan(false, true);
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
      DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
      SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
      SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
     
      // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
     
     
      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
      Plan plan = getJavaTestPlan(false, false);
     
      OptimizedPlan oPlan = compileNoStats(plan);
     
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
      DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
      SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
     
      // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
     
      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
     
      // verify solution delta
      assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
     
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
     
      OptimizedPlan oPlan = compileNoStats(plan);
 
     
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
      DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
      DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
      SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
     
      // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
      // the in-loop partitioning is before the final reducer
     
      // verify joinWithInvariant
      assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
      assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
     
      // verify joinWithSolutionSet
      assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
      assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
     
      // verify reducer
      assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
     
     
      // verify solution delta
      assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
      assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
     
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
View Full Code Here
320
321
322
323
324
325
326
327
328
329
330
331
332
    oPlan.accept(new Visitor<PlanNode>() {
     
      @Override
      public boolean preVisit(PlanNode visitable) {
        if (visitable instanceof DualInputPlanNode) {
          DualInputPlanNode node = (DualInputPlanNode) visitable;
          Channel c1 = node.getInput1();
          Channel c2 = node.getInput2();
         
          Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.FORWARD, c1.getShipStrategy());
          Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.PARTITION_HASH, c2.getShipStrategy());
          return false;
        }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
public class JoinTranslationTest extends CompilerTestBase {

  @Test
  public void testBroadcastHashFirstTest() {
    try {
      DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_FIRST);
      assertEquals(ShipStrategyType.BROADCAST, node.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, node.getInput2().getShipStrategy());
      assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getClass().getSimpleName() + ": " + e.getMessage());
    }
View Full Code Here
TOP

Related Classes of org.apache.flink.compiler.plan.DualInputPlanNode

  • org.apache.flink.compiler.AdditionalOperatorsTest
  • org.apache.flink.compiler.CachedMatchStrategyCompilerTest
  • org.apache.flink.compiler.CoGroupSolutionSetFirstTest
  • org.apache.flink.compiler.dag.TwoInputNode
  • org.apache.flink.compiler.deadlockdetect.DeadlockPreventer
  • org.apache.flink.compiler.DOPChangeTest
  • org.apache.flink.compiler.FeedbackPropertiesMatchTest
  • org.apache.flink.compiler.GroupOrderTest
  • org.apache.flink.compiler.java.JoinTranslationTest
  • org.apache.flink.compiler.java.WorksetIterationsJavaApiCompilerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.