Examples of lupos.optimizations.logical.statistics.VarBucket

Package lupos.optimizations.logical.statistics

Examples of lupos.optimizations.logical.statistics.VarBucket

lupos.optimizations.logical.statistics.VarBucket

          result = map;
        } else {
          // look, which histogram is based on more elements and just choose this one
          for(final Entry<Variable, VarBucket> entry: map.entrySet()){
            final Variable var = entry.getKey();
            final VarBucket histogram = result.get(var);
            final VarBucket otherHistogram = entry.getValue();
            if(histogram == null || otherHistogram.getSum()>histogram.getSum()) {
              result.put(var, otherHistogram);
            }
          }
        }
      }

View Full Code Here

          }
          if (flag) {
            intermediate.keySet().retainAll(joinPartnersTP);
            if (classBindings == BindingsArrayVarMinMax.class) {
              for (final Variable v : intermediate.keySet()) {
                final VarBucket vb = intermediate.get(v);
                final Literal l[] = new Literal[vb.selectivityOfInterval
                    .size()];
                int indexLiteral = 0;
                for (final Entry entry : vb.selectivityOfInterval) {
                  l[indexLiteral] = entry.literal;
                  indexLiteral++;
                }
                tp.addHistogram(v, l, vb.getSum());
              }
            }
            return intermediate;
          }
        }
      }
    }
    // first determine the result size of the triple pattern!
    final Map<Variable, VarBucket> result = new HashMap<Variable, VarBucket>();
    final Collection<TriplePattern> ztp = this.getTriplePattern();
    final Collection<TriplePattern> ctp = new LinkedList<TriplePattern>();
    ctp.add(tp);
    this.setTriplePatterns(ctp);
    if (this instanceof RDF3XIndexScan) {
      ((RDF3XIndexScan) this).setCollationOrder(RDF3XOperatorGraphGenerator.getCollationOrder(tp, joinPartners));
      ((RDF3XIndexScan) this).setMinimaMaxima(minima, maxima);


    }
    final QueryResult qrSize = this.join(this.root.dataset);
    if (qrSize == null) {
      // System.out.println("No result for " + tp);
      return null;
    }


    int size = 0;


    if (classBindings == BindingsArrayVarMinMax.class) {
      final Map<Variable, Integer> hmvi = this.bindingsFactory.getPosVariables();
      final Integer[] minArray = new Integer[hmvi.size()];
      final Integer[] maxArray = new Integer[hmvi.size()];


      final Iterator<Bindings> itbSize = qrSize.oneTimeIterator();
      while (itbSize.hasNext()) {
        final Bindings b = itbSize.next();
        for (final Variable v : b.getVariableSet()) {
          final LazyLiteral ll = (LazyLiteral) b.get(v);
          final int pos = hmvi.get(v);
          if (minArray[pos] == null || minArray[pos] > ll.getCode()) {
            minArray[pos] = ll.getCode();
          }
          if (maxArray[pos] == null || maxArray[pos] < ll.getCode()) {
            maxArray[pos] = ll.getCode();
          }
        }
        size++;


        for (int i = 0; i < minArray.length; i++) {
          if (minArray[i] != null) {
            tp.addMinMaxLazyLiteral(i, minArray[i], maxArray[i]);
          }
        }
      }
    } else {
      int maxId = 1;
      if (qrSize instanceof IdIteratorQueryResult) {
        maxId = ((IdIteratorQueryResult) qrSize).getMaxId();
      }


      final int[][] min = new int[maxId][];
      final int[][] max = new int[maxId][];


      size = qrSize.oneTimeSize();


      for (int id = 0; id < maxId; id++) {
        if (min[id] != null) {
          for (int i = 0; i < min[id].length; i++) {
            tp.addMinMaxPresortingNumbers(i, min[id].length,
                id, min[id][i], max[id][i]);
          }
        }
      }
    }


    qrSize.release();


    if (size == 0) {
      System.out.println("No result for " + tp);
      return null;
    }


    // System.out.println("(Statistics) " + tp + ", " + size
    // + " triples retrieved");
    tp.setCardinality(size);


    for (final Variable v : joinPartnersTP) {
      if (intermediate != null && intermediate.containsKey(v)) {
        boolean flag = true;
        final Literal min = minima.get(v);
        if (intermediate.get(v) == null
            || min != null
            && (intermediate.get(v).minimum == null || !min
                .equals(intermediate.get(v).minimum))) {
          flag = false;
        }
        final Literal max = maxima.get(v);
        if (max != null
            && (intermediate.get(v).maximum == null || !max
                .equals(intermediate.get(v).maximum))) {
          flag = false;
        }
        if (flag) {
          result.put(v, intermediate.get(v));
          continue;
        }
      }
      // get result of triple pattern in the correct sorted way!
      final Collection<Variable> cv = new LinkedList<Variable>();
      cv.add(v);
      if (this instanceof RDF3XIndexScan) {
        ((RDF3XIndexScan) this).setCollationOrder(CollationOrder.getCollationOrder(tp, cv));
      }


      QueryResult qr = this.join(this.root.dataset);


      if (this instanceof MemoryIndexScan) {
        // additional sorting phase according to variable v needed
        // for memory index approach!
        final DBMergeSortedBag<Bindings> sort = new DBMergeSortedBag<Bindings>(
            new SortConfiguration(), new Comparator<Bindings>() {
              @Override
              public int compare(final Bindings arg0, final Bindings arg1) {
                return arg0.get(v).compareToNotNecessarilySPARQLSpecificationConform(arg1.get(v));
              }
            }, Bindings.class);
        final Iterator<Bindings> itb = qr.oneTimeIterator();
        while (itb.hasNext()) {
          sort.add(itb.next());
        }
        qr = QueryResult.createInstance(sort.iterator());
      }
      if (qr == null) {
        return result;
      }


      if (size == 0) {
        qr.release();
        return result;
      }
      final double bucketSize = (double) size / MaxNumberBuckets;
      final Iterator<Bindings> ib = qr.oneTimeIterator();
      final VarBucket vb = new VarBucket();
      vb.minimum = minima.get(v);
      vb.maximum = maxima.get(v);
      Entry currentEntry = new Entry();
      Literal lastLiteral = null;
      while (ib.hasNext()) {
        final Bindings b = ib.next();
        final Literal next = b.get(v);
        if (lastLiteral == null
            || lastLiteral
                .compareToNotNecessarilySPARQLSpecificationConform(next) != 0) {
          currentEntry.distinctLiterals += 1.0;
          lastLiteral = next;
        }
        currentEntry.selectivity += 1.0;
        if (currentEntry.selectivity >= bucketSize) {
          currentEntry.literal = b.get(v);
          if (ib.hasNext()) {
            Bindings b2 = ib.next();
            while (ib.hasNext()
                && b2
                    .get(v)
                    .compareToNotNecessarilySPARQLSpecificationConform(
                        currentEntry.literal) == 0) {
              currentEntry.selectivity += 1.0;
              b2 = ib.next();
            }
            vb.selectivityOfInterval.add(currentEntry);
            currentEntry = new Entry();
            currentEntry.distinctLiterals = 1.0;
            currentEntry.selectivity = 1.0;
            if (!ib.hasNext()
                && b2
                    .get(v)
                    .compareToNotNecessarilySPARQLSpecificationConform(
                        next) != 0) {
              currentEntry.literal = b2.get(v);
              vb.selectivityOfInterval.add(currentEntry);
            }
          } else {
            vb.selectivityOfInterval.add(currentEntry);
          }
        }
      }
      qr.release();
      result.put(v, vb);
      if (intermediate != null) {
        intermediate.put(v, vb);
      }
    }
    this.setTriplePatterns(ztp);


    if (classBindings == BindingsArrayVarMinMax.class) {
      for (final Variable v : result.keySet()) {
        final VarBucket vb = result.get(v);
        final Literal l[] = new Literal[vb.selectivityOfInterval.size()];
        int indexLiteral = 0;
        for (final Entry entry : vb.selectivityOfInterval) {
          l[indexLiteral] = entry.literal;
          indexLiteral++;
        }
        tp.addHistogram(v, l, vb.getSum());
      }
    }
    if (Indices.usedDatastructure == Indices.DATA_STRUCT.DBBPTREE) {
      if (intermediate != null) {
        this.storeVarBuckets(tp, intermediate, key);

View Full Code Here

          }
          if (flag) {
            intermediate.keySet().retainAll(joinPartnersTP);
            // if (classBindings == BindingsArrayVarMinMax.class) {
            for (final Variable v : intermediate.keySet()) {
              final VarBucket vb = intermediate.get(v);
              final Literal l[] = new Literal[vb.selectivityOfInterval
                  .size()];
              int indexLiteral = 0;
              for (final Entry entry : vb.selectivityOfInterval) {
                l[indexLiteral] = entry.literal;
                indexLiteral++;
              }
              tp.addHistogram(v, l, vb.getSum());
            }
            // }
            return intermediate;
          }
        }
      }
    }
    final Map<Variable, VarBucket> result = new HashMap<Variable, VarBucket>();
    for (final Variable v : joinPartnersTP) {
      final Literal min = (minima == null) ? null : minima.get(v);
      final Literal max = (maxima == null) ? null : maxima.get(v);
      if (intermediate != null && intermediate.containsKey(v)) {
        boolean flag = true;
        if (intermediate.get(v) == null
            || min != null
            && (intermediate.get(v).minimum == null || min
                .compareToNotNecessarilySPARQLSpecificationConform(intermediate
                    .get(v).minimum) != 0)) {
          flag = false;
        }
        if (max != null
            && (intermediate.get(v).maximum == null || max
                .compareToNotNecessarilySPARQLSpecificationConform(intermediate
                    .get(v).maximum) != 0)) {
          flag = false;
        }
        if (flag) {
          result.put(v, intermediate.get(v));
          continue;
        }
      }
      try {


        // get the graph constraint from the super class.
        // If it is null, a default graph is used, if not null a named
        // one
        // is used
        final Item graphConstraintItem = this.getGraphConstraint();


        // get a collection of indices using the determined graph
        // constraint
        final Collection<Indices> indicesC = this.root.dataset.indexingRDFGraphs(
            graphConstraintItem, false, false, this.root);
        if ((indicesC != null) && !(indicesC.size() == 0)) {
          final Triple key = getKey(tp, null);
          final Collection<URILiteral> namedGraphs = new ArrayList<URILiteral>();


          // if the graph constraint is not null (which means that a
          // named
          // graph is used)
          if (graphConstraintItem != null) {


            if (graphConstraintItem instanceof Variable) {


              final Variable graphConstraint = (Variable) graphConstraintItem;


              // check if named graphs were provided at query time
              if (this.root.namedGraphs != null
                  && this.root.namedGraphs.size() > 0) {


                // Convert the named graphs' names into
                // URILiterals
                // to be applicable
                // later on
                for (final String name : this.root.namedGraphs) {


                  final Indices indices = this.root.dataset
                      .getNamedGraphIndices(LiteralFactory
                          .createURILiteralWithoutLazyLiteral(name));


                  final URILiteral rdfName = indices
                      .getRdfName();
                  if (namedGraphs.contains(rdfName)) {
                    final TriplePattern ztp = new TriplePattern(
                        graphConstraint.equals(tp
                            .getPos(0)) ? rdfName
                            : tp.getPos(0),
                        graphConstraint.equals(tp
                            .getPos(1)) ? rdfName
                            : tp.getPos(1),
                        graphConstraint.equals(tp
                            .getPos(2)) ? rdfName
                            : tp.getPos(2));
                    final Triple zkey = getKey(ztp, null);
                    final Triple keyMinimum = this.getKey(ztp,
                        null, minima);
                    final Triple keyMaximum = this.getKey(ztp,
                        null, maxima);


                    final VarBucket vb = this.getVarBucket(v,
                        ztp, zkey, keyMinimum,
                        keyMaximum,
                        (SixIndices) indices);
                    if (vb != null) {
                      final VarBucket previous_vb = result
                          .get(v);
                      if (previous_vb != null) {
                        vb.add(previous_vb);
                      }
                      vb.minimum = (minima == null) ? null
                          : minima.get(v);
                      vb.maximum = (maxima == null) ? null
                          : maxima.get(v);
                      result.put(v, vb);
                    }
                  }


                }


              }


              // otherwise there might have been named graphs
              // added
              // during the evaluation
              else {


                // get all indices of named graphs and bind them
                // to
                // the graph constraint
                final Collection<Indices> dataSetIndices = this.root.dataset
                    .getNamedGraphIndices();
                if (dataSetIndices != null) {


                  for (final Indices indices : dataSetIndices) {
                    final TriplePattern ztp = new TriplePattern(
                        graphConstraint.equals(tp
                            .getPos(0)) ? indices
                            .getRdfName() : tp
                            .getPos(0),
                        graphConstraint.equals(tp
                            .getPos(1)) ? indices
                            .getRdfName() : tp
                            .getPos(1),
                        graphConstraint.equals(tp
                            .getPos(2)) ? indices
                            .getRdfName() : tp
                            .getPos(2));
                    final Triple zkey = getKey(ztp, null);
                    final Triple keyMinimum = this.getKey(ztp,
                        null, minima);
                    final Triple keyMaximum = this.getKey(ztp,
                        null, maxima);
                    final VarBucket vb = this.getVarBucket(v,
                        ztp, zkey, keyMinimum,
                        keyMaximum,
                        (SixIndices) indices);
                    if (vb != null) {
                      final VarBucket previous_vb = result
                          .get(v);
                      if (previous_vb != null) {
                        vb.add(previous_vb);
                      }
                      vb.minimum = (minima == null) ? null
                          : minima.get(v);
                      vb.maximum = (maxima == null) ? null
                          : maxima.get(v);
                      result.put(v, vb);
                    }
                  }
                }
              }
            }


            // if the graph constraint is an URILiteral fetch the
            // matching indices object
            // but do not bind anything
            else {
              final Triple keyMinimum = this.getKey(tp, null, minima);
              final Triple keyMaximum = this.getKey(tp, null, maxima);
              for (final Indices indices : indicesC) {


                final URILiteral rdfName = indices.getRdfName();
                if (namedGraphs.contains(rdfName)) {
                  final VarBucket vb = this.getVarBucket(v, tp,
                      key, keyMinimum, keyMaximum,
                      (SixIndices) indices);
                  if (vb != null) {
                    final VarBucket previous_vb = result
                        .get(v);
                    if (previous_vb != null) {
                      vb.add(previous_vb);
                    }
                    vb.minimum = (minima == null) ? null
                        : minima.get(v);
                    vb.maximum = (maxima == null) ? null
                        : maxima.get(v);
                    result.put(v, vb);
                  }
                }


              }
            }
          } else {
            if (indicesC != null) {
              // deal with special case: several default graphs!
              // if (triplePatterns.size() != 1) {
              //log.error("Can only process one triple pattern!");
              // }
              final Triple keyMinimum = this.getKey(tp, null, minima);
              final Triple keyMaximum = this.getKey(tp, null, maxima);


              for (final Indices indices : indicesC) {
                final VarBucket vb = this.getVarBucket(v, tp, key,
                    keyMinimum, keyMaximum,
                    (SixIndices) indices);
                if (vb != null) {
                  vb.minimum = (minima == null) ? null
                      : minima.get(v);
                  vb.maximum = (maxima == null) ? null
                      : maxima.get(v);
                  final VarBucket previous_vb = result.get(v);
                  if (previous_vb != null) {
                    vb.add(previous_vb);
                  }
                  result.put(v, vb);
                }
              }
            }
          }
        }
      } catch (final Exception e) {
        System.err.println("Error while joining triple patterns: "+ e);
        e.printStackTrace();
        return null;
      }


    }
    // if (classBindings == BindingsArrayVarMinMax.class) {
    for (final Variable v : result.keySet()) {
      final VarBucket vb = result.get(v);
      final Literal l[] = new Literal[vb.selectivityOfInterval.size()];
      int indexLiteral = 0;
      for (final Entry entry : vb.selectivityOfInterval) {
        l[indexLiteral] = entry.literal;
        indexLiteral++;
      }
      tp.addHistogram(v, l, vb.getSum());
    }
    // }
    // if (intermediate != null)
    // storeVarBuckets(tp, intermediate, keyHistogram);
    // else

View Full Code Here

   * @param varBucketJson the json object containing the histogram
   * @return the histogram object
   * @throws JSONException
   */
  public static VarBucket createVarBucketFromJSON(final JSONObject varBucketJson) throws JSONException {
    final VarBucket result = new VarBucket();


    result.minimum = Helper.createLiteralFromJSON(varBucketJson.getJSONObject("min"));
    result.maximum = Helper.createLiteralFromJSON(varBucketJson.getJSONObject("max"));


    final JSONArray entriesJson = (JSONArray) varBucketJson.get("histogram");

View Full Code Here

      return new InnerNodeInPartitionTree(pcoll);
    }
  }


  public final static VarBucket readLuposVarBucket(final InputStream is) throws IOException {
    final VarBucket vb = new VarBucket();
    final int size = InputHelper.readLuposIntVariableBytes(is);
    final byte minMax = InputHelper.readLuposByte(is);
    if (minMax >= 2) {
      vb.minimum = LiteralFactory.readLuposLiteral(is);
    }

View Full Code Here

    return null;
  }


  public VarBucket getVarBucket(final TripleKey key, final TripleKey minimum,
      final TripleKey maximum, final int pos) {
    final VarBucket result = new VarBucket();
    final int distance = (minimum == null) ? this.getDistance(key, key)
        : (maximum == null) ? this.getDistance(minimum, minimum)
            : this.getDistance(minimum, maximum);
    if (distance < 0) {
      return null;

View Full Code Here

TOP

Related Classes of lupos.optimizations.logical.statistics.VarBucket

lupos.datastructures.items.literal.Literal

lupos.datastructures.paged_dbbptree.LazyLiteralTripleKeyDBBPTreeStatistics

lupos.distributed.operator.format.Helper

lupos.distributed.query.operator.histogramsubmission.AbstractHistogramExecutor

lupos.engine.operators.index.adaptedRDF3X.RDF3XIndexScan

lupos.engine.operators.index.BasicIndexScan

lupos.io.helper.InputHelper

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.