Package org.apache.lucene.util

Examples of org.apache.lucene.util.AttributeSource


        return a.length - b.length;
      }
    });

    for (int i = 0; i < tokens.length; i++) {
      AttributeSource token = tokens[i];
      final NamedList<Object> tokenNamedList = new SimpleOrderedMap<Object>();
      final String rawText = token.addAttribute(CharTermAttribute.class).toString();

      String text = fieldType.indexedToReadable(rawText);
      tokenNamedList.add("text", text);
      if (!text.equals(rawText)) {
        tokenNamedList.add("raw_text", rawText);
      }

      if (context.getTermsToMatch().contains(rawText)) {
        tokenNamedList.add("match", true);
      }

      token.reflectWith(new AttributeReflector() {
        public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
          // leave out position and term
          if (CharTermAttribute.class.isAssignableFrom(attClass))
            return;
          if (PositionIncrementAttribute.class.isAssignableFrom(attClass))
View Full Code Here


    @Override
    public boolean incrementToken() throws IOException {
      if (tokenIterator.hasNext()) {
        clearAttributes();
        AttributeSource next = tokenIterator.next();
        Iterator<Class<? extends Attribute>> atts = next.getAttributeClassesIterator();
        while (atts.hasNext()) // make sure all att impls in the token exist here
          addAttribute(atts.next());
        next.copyTo(this);
        return true;
      } else {
        return false;
      }
    }
View Full Code Here

        copy(this, replacement.next());
        return true;
      }

      // common case fast-path of first token not matching anything
      AttributeSource firstTok = nextTok();
      if (firstTok == null) return false;
      CharTermAttribute termAtt = firstTok.addAttribute(CharTermAttribute.class);
      SynonymMap result = map.submap!=null ? map.submap.get(termAtt.buffer(), 0, termAtt.length()) : null;
      if (result == null) {
        copy(this, firstTok);
        return true;
      }

      // fast-path failed, clone ourselves if needed
      if (firstTok == this)
        firstTok = cloneAttributes();
      // OK, we matched a token, so find the longest match.

      matched = new LinkedList<AttributeSource>();

      result = match(result);

      if (result==null) {
        // no match, simply return the first token read.
        copy(this, firstTok);
        return true;
      }

      // reuse, or create new one each time?
      ArrayList<AttributeSource> generated = new ArrayList<AttributeSource>(result.synonyms.length + matched.size() + 1);

      //
      // there was a match... let's generate the new tokens, merging
      // in the matched tokens (position increments need adjusting)
      //
      AttributeSource lastTok = matched.isEmpty() ? firstTok : matched.getLast();
      boolean includeOrig = result.includeOrig();

      AttributeSource origTok = includeOrig ? firstTok : null;
      PositionIncrementAttribute firstPosIncAtt = firstTok.addAttribute(PositionIncrementAttribute.class);
      int origPos = firstPosIncAtt.getPositionIncrement()// position of origTok in the original stream
      int repPos=0; // curr position in replacement token stream
      int pos=0// current position in merged token stream

      for (int i=0; i<result.synonyms.length; i++) {
        Token repTok = result.synonyms[i];
        AttributeSource newTok = firstTok.cloneAttributes();
        CharTermAttribute newTermAtt = newTok.addAttribute(CharTermAttribute.class);
        OffsetAttribute newOffsetAtt = newTok.addAttribute(OffsetAttribute.class);
        PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);

        OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);

        newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
        newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
View Full Code Here

  private SynonymMap match(SynonymMap map) throws IOException {
    SynonymMap result = null;

    if (map.submap != null) {
      AttributeSource tok = nextTok();
      if (tok != null) {
        // clone ourselves.
        if (tok == this)
          tok = cloneAttributes();
        // check for positionIncrement!=1?  if>1, should not match, if==0, check multiple at this level?
        CharTermAttribute termAtt = tok.getAttribute(CharTermAttribute.class);
        SynonymMap subMap = map.submap.get(termAtt.buffer(), 0, termAtt.length());

        if (subMap != null) {
          // recurse
          result = match(subMap);
View Full Code Here

                copy(this, replacement.next());
                return true;
            }

            // common case fast-path of first token not matching anything
            AttributeSource firstTok = nextTok();
            if (firstTok == null) return false;
            CharTermAttribute termAtt = firstTok.addAttribute(CharTermAttribute.class);
            SynonymMap result = map.submap != null ? map.submap.get(termAtt.buffer(), 0, termAtt.length()) : null;
            if (result == null) {
                copy(this, firstTok);
                return true;
            }

            // fast-path failed, clone ourselves if needed
            if (firstTok == this)
                firstTok = cloneAttributes();
            // OK, we matched a token, so find the longest match.

            matched = new LinkedList<AttributeSource>();

            result = match(result);

            if (result == null) {
                // no match, simply return the first token read.
                copy(this, firstTok);
                return true;
            }

            // reuse, or create new one each time?
            ArrayList<AttributeSource> generated = new ArrayList<AttributeSource>(result.synonyms.length + matched.size() + 1);

            //
            // there was a match... let's generate the new tokens, merging
            // in the matched tokens (position increments need adjusting)
            //
            AttributeSource lastTok = matched.isEmpty() ? firstTok : matched.getLast();
            boolean includeOrig = result.includeOrig();

            AttributeSource origTok = includeOrig ? firstTok : null;
            PositionIncrementAttribute firstPosIncAtt = firstTok.addAttribute(PositionIncrementAttribute.class);
            int origPos = firstPosIncAtt.getPositionIncrement()// position of origTok in the original stream
            int repPos = 0; // curr position in replacement token stream
            int pos = 0// current position in merged token stream

            for (int i = 0; i < result.synonyms.length; i++) {
                Token repTok = result.synonyms[i];
                AttributeSource newTok = firstTok.cloneAttributes();
                CharTermAttribute newTermAtt = newTok.addAttribute(CharTermAttribute.class);
                OffsetAttribute newOffsetAtt = newTok.addAttribute(OffsetAttribute.class);
                PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);

                OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);

                newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
                newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
View Full Code Here

    private SynonymMap match(SynonymMap map) throws IOException {
        SynonymMap result = null;

        if (map.submap != null) {
            AttributeSource tok = nextTok();
            if (tok != null) {
                // clone ourselves.
                if (tok == this)
                    tok = cloneAttributes();
                // check for positionIncrement!=1?  if>1, should not match, if==0, check multiple at this level?
                CharTermAttribute termAtt = tok.getAttribute(CharTermAttribute.class);
                SynonymMap subMap = map.submap.get(termAtt.buffer(), 0, termAtt.length());

                if (subMap != null) {
                    // recurse
                    result = match(subMap);
View Full Code Here

   */
  private DocsEnum getOther(DocsEnum de) {
    if (de == null) {
      return null;
    } else {
      final AttributeSource atts = de.attributes();
      return atts.addAttribute(PulsingEnumAttribute.class).enums().get(this);
    }
  }
View Full Code Here

  /**
   * for a docsenum, sets the 'other' reused enum.
   * see getOther for an example.
   */
  private DocsEnum setOther(DocsEnum de, DocsEnum other) {
    final AttributeSource atts = de.attributes();
    return atts.addAttribute(PulsingEnumAttribute.class).enums().put(this, other);
  }
View Full Code Here

  protected TermsEnum() {
  }

  /** Returns the related attributes. */
  public AttributeSource attributes() {
    if (atts == null) atts = new AttributeSource();
    return atts;
  }
View Full Code Here

      if (!processedTerms.contains(term)) {
        processedTerms.add(term);
        ScoreTermQueue variantsQ = new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
        float minScore = 0;
        Term startTerm = new Term(f.fieldName, term);
        AttributeSource atts = new AttributeSource();
        MaxNonCompetitiveBoostAttribute maxBoostAtt =
            atts.addAttribute(MaxNonCompetitiveBoostAttribute.class);
        SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength);
        //store the df so all variants use same idf
        int df = reader.docFreq(startTerm);
        int numVariants = 0;
        int totalVariantDocFreqs = 0;
View Full Code Here

TOP

Related Classes of org.apache.lucene.util.AttributeSource

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.