Examples of TokenList

com.lastcalc.TokenList
com.plainsource.commons.text.template.token.TokenList
edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList
net.sourceforge.chaperon.grammar.token.TokenList
This class contains a list of tokens. Like
identifier := [A-Za-z][A-Za-z0-9_]
number := [0-9]+ @author Stephan Michels @version $Revision: 1.7 $ $Date: 2002/05/07 11:01:49 $
org.hsqldb.cmdline.sqltool.TokenList
A list of SqlFile Tokens

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList

              sb.append(doclabDimmed);
          else
            sb.append(doclab);
          counter++;
          
          TokenList lhs = line.getLeftHandSide();
          sb.append("<td class='right'>");
          for (Iterator<Token> iterator = lhs.iterator(); iterator.hasNext();) {
            Token word = iterator.next();
            sb.append(escapeXML(word.getText()) + " ");  
          }
          sb.append("</td><td><b>");
          sb.append(escapeXML(line.getTarget().getText()));
          TokenList rhs = line.getRightHandSide();
          sb.append("</b>");
          for (Iterator<Token> iterator = rhs.iterator(); iterator.hasNext();) {
            Token word = iterator.next();
            sb.append(" " + escapeXML(word.getText()));  
          }
          sb.append("</td></tr>\n"); 
        }

View Full Code Here

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList

          sb.append(toXML(doc));
          Concordance conc = map.get(doc);
          for (Iterator<ConcordanceLine> iter = conc.iterator(); iter.hasNext();) {
            ConcordanceLine line = iter.next();
            sb.append("<line>"); 
            TokenList lhs = line.getLeftHandSide();
            for (Iterator<Token> iterator = lhs.iterator(); iterator.hasNext();) {
              Token word = iterator.next();
              sb.append("<w txt=\"" + escapeXML(word.getText()) + "\"/>");  
            }
            sb.append( "<w txt=\"" + escapeXML(line.getTarget().getText()) +  
            "\" target=\"true\"/>"); 
            TokenList rhs = line.getRightHandSide();
            for (Iterator<Token> iterator = rhs.iterator(); iterator.hasNext();) {
              Token word = iterator.next();
              sb.append("<w txt=\"" + escapeXML(word.getText()) + "\"/>");  
            }
            sb.append("</line>\n"); 
          }

View Full Code Here

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList

        sb.append(" windowsize=\"" + conc.getWindowSize() + "\"");  
        sb.append(" creationdate=\"" + (new java.util.Date()) + "\">\n"); 
        for (Iterator<ConcordanceLine> iter = conc.iterator(); iter.hasNext();) {
            ConcordanceLine line = iter.next();
            sb.append("    <line>"); 
            TokenList lhs = line.getLeftHandSide();
            for (Iterator<Token> iterator = lhs.iterator(); iterator.hasNext();) {
                Token word = iterator.next();
                sb.append("<w txt=\"" + escapeXML(word.getText()) + "\"/>");  
            }
            sb.append( "<w txt=\"" + escapeXML(line.getTarget().getText()) +  
            "\" target=\"true\"/>"); 
            TokenList rhs = line.getRightHandSide();
            for (Iterator<Token> iterator = rhs.iterator(); iterator.hasNext();) {
                Token word = iterator.next();
                sb.append("<w txt=\"" + escapeXML(word.getText()) + "\"/>");  
            }
            sb.append("</line>\n"); 
        }

View Full Code Here

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList

        }
    }


    public void endElement(String uri, String localName, String qName){                                                  
        if (qName.equals("line")) { //$NON-NLS-1$
            TokenList tl = new TokenListImpl();
            for (Iterator iter = lhs.iterator(); iter.hasNext();) {
                String lhs = (String) iter.next();
                Token t = new TokenImpl(lhs, 0, 0);
                tl.add(t);
            }
            TokenList rl = new TokenListImpl();
            for (Iterator iter = rhs.iterator(); iter.hasNext();) {
                String rhs = (String) iter.next();
                Token t = new TokenImpl(rhs, 0, 0);
                rl.add(t);
            }
            Token targ = new TokenImpl(target, 0, 0);
            ConcordanceLine line = new ConcordanceLineImpl(tl, targ, rl);
            clist.add(line);
        }

View Full Code Here

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList

        
        TaskWorker tworker = new TaskWorker(yoshikoder){
            YKDictionaryReportDialog dia;
          protected void doWork() throws Exception {
                TokenizationCache tcache = yoshikoder.getTokenizationCache();
                TokenList tl = tcache.getTokenList(doc);
                if (tl == null){
                    tl = TokenizationService.getTokenizationService().tokenize(doc);
                    tcache.putTokenList(doc, tl);
                }

View Full Code Here

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenList

        Node n = yoshikoder.getSelectedNode();
        YKDocument doc = yoshikoder.getSelectedDocument();
        if (n == null || doc == null)
            return;
        
        TokenList toclist = null;
        try {
            TokenizationCache tcache = yoshikoder.getTokenizationCache();
            toclist = tcache.getTokenList(doc);
            if (toclist == null){
                TokenList tl =
                    TokenizationService.getTokenizationService().tokenize(doc);
                tcache.putTokenList(doc, tl);
                toclist = tl;
            }
            Set<Token> matches = yoshikoder.getDictionary().getMatchingTokens(toclist, n);

View Full Code Here

Examples of net.sourceforge.chaperon.grammar.token.TokenList

        grammar.setURI(atts.getValue(URI_ATTRIBUTE));
        stack.push(grammar);
      }
      else if (localName.equals(TOKENLIST_ELEMENT))
      {
        stack.push(new TokenList());
      }
      else if (localName.equals(IGNORABLETOKENLIST_ELEMENT))
      {
        stack.push(new TokenList());
      }
      else if (localName.equals(TOKEN_ELEMENT))
      {
        Token token = new Token(TerminalSymbol.valueOf(atts.getValue(TOKEN_SYMBOL_ATTRIBUTE)));

View Full Code Here

Examples of net.sourceforge.chaperon.grammar.token.TokenList

        // stack.push(new Grammar());
        grammar = (Grammar) stack.pop();
      }
      else if (localName.equals(TOKENLIST_ELEMENT))
      {
        TokenList tokens = (TokenList) stack.pop();
        Grammar grammar = (Grammar) stack.peek();


        grammar.getTokenList().addToken(tokens);
      }
      else if (localName.equals(IGNORABLETOKENLIST_ELEMENT))
      {
        TokenList tokens = (TokenList) stack.pop();
        Grammar grammar = (Grammar) stack.peek();


        grammar.getIgnorableTokenList().addToken(tokens);
      }
      else if (localName.equals(TOKEN_ELEMENT))
      {
        Token token = (Token) stack.pop();
        TokenList tokens = (TokenList) stack.peek();


        tokens.addToken(token);
      }
      else if ((localName.equals(ALTERNATION_ELEMENT))
               || (localName.equals(CONCATENATION_ELEMENT))
               || (localName.equals(CHARACTERSEQUENCE_ELEMENT))
               || (localName.equals(CHARACTERCLASS_ELEMENT))

View Full Code Here

Examples of org.hsqldb.cmdline.sqltool.TokenList

            setEncoding(encoding);
            this.interactive = interactive;
            continueOnError = this.interactive;


            if (interactive) {
                history = new TokenList();
                maxHistoryLength = DEFAULT_HISTORY_SIZE;
            }
            updateUserSettings();
            // Updates local vars basd on * shared.userVars
            // even when (like now) these are all defaults.

View Full Code Here

Examples of org.hsqldb.cmdline.sqltool.TokenList

     * the very top.
     */
    private TokenList seekTokenSource(String nestingCommand)
            throws BadSpecial, IOException {
        Token token;
        TokenList newTS = new TokenList();
        Pattern endPattern = Pattern.compile("end\\s+" + nestingCommand);
        String subNestingCommand;


        while ((token = scanner.yylex()) != null) {
            if (token.type == Token.PL_TYPE
                    && endPattern.matcher(token.val).matches()) {
                return newTS;
            }
            subNestingCommand = nestingCommand(token);
            if (subNestingCommand != null) {
                token.nestedBlock = seekTokenSource(subNestingCommand);
            }
            newTS.add(token);
        }
        throw new BadSpecial(
                SqltoolRB.pl_block_unterminated.getString(nestingCommand));
    }

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.