Examples of WordIterator

be.bagofwords.text.WordIterator
Split a text into words.
cue.lang.WordIterator
@author Jonathan Feinberg
edu.ucla.sspace.text.WordIterator
An iterator over all of the tokens present in a {@link BufferedReader} thatare separated by any amount of white space.

Examples of be.bagofwords.text.WordIterator

            if (charsRead < 0) {
                throw new RuntimeException("Could not read the text");
            }
            long numOfWordsInText = 0;
            char[] actualTextBuffer = charsRead < textBuffer.length ? Arrays.copyOf(textBuffer, charsRead) : textBuffer;
            WordIterator wordIterator = new WordIterator(actualTextBuffer, Collections.<String>emptySet());
            String prev = null;
            while (wordIterator.hasNext()) {
                String word = wordIterator.next().toString().toLowerCase();
                if (prev != null) {
                    if (readData) {
                        if (dataType == DataType.LONG_COUNT) {
                            dataInterface.readCount(prev + " " + word);
                        } else {

View Full Code Here

Examples of cue.lang.WordIterator

    return block;
  }


  public static UnicodeBlock guessUnicodeBlock(final String text)
  {
    return guessUnicodeBlock(new Counter<String>(new WordIterator(text)));
  }

View Full Code Here

Examples of cue.lang.WordIterator

  }


    private Word[] countWords(String text) {
        Counter<String> counter = new Counter<String>();


        for (String word : new WordIterator(text)) {
            if (shouldCountWord(word)) {
                counter.note(word);
            }
        }

View Full Code Here

Examples of edu.ucla.sspace.text.WordIterator


        // Load any Parse the config file for test types.  The configuration
        // file formatted as pairs of evaluations paired with data
        // files with everything separated by spaces.
        if (configFile != null) {
            WordIterator it = new WordIterator(new BufferedReader(
                                                   new FileReader(configFile)));
            while (it.hasNext()) {
                String className = it.next();
                if (!it.hasNext()) {
                    throw new Error("test is not matched with data file: " + 
                                    className);
                }
                String[] dataFiles = it.next().split(",");
                // Base the number of constructor arguments on the number of
                // String parameters specified
                Class<?> clazz = Class.forName(className);
                Class[] constructorArgs = new Class[dataFiles.length];
                for (int i = 0; i < constructorArgs.length; ++i)

View Full Code Here

Examples of edu.ucla.sspace.text.WordIterator

            int docNum = 0;
            long startTime = System.currentTimeMillis();
            Iterator<Document> docs = getDocuments(options);
            while (docs.hasNext()) {
                Document doc = docs.next();
                Iterator<String> tokens = new WordIterator(doc.reader());
                String t1 = null;
                while (tokens.hasNext()) {
                    String t2 = tokens.next();
                    // Count the occurrence of this token if we're supposed to
                    // record it
                    if (terms.contains(t2)
                            && (stopWords == null || !stopWords.contains(t2))) {

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.