Examples of java.io.StreamTokenizer

java.io.StreamTokenizer
The StreamTokenizer class takes an input stream and parses it into "tokens", allowing the tokens to be read one at a time. The parsing process is controlled by a table and a number of flags that can be set to various states. The stream tokenizer can recognize identifiers, numbers, quoted strings, and various comment styles.
Each byte read from the input stream is regarded as a character in the range '\u0000' through '\u00FF'. The character value is used to look up five possible attributes of the character: white space, alphabetic, numeric, string quote, and comment character. Each character can have zero or more of these attributes.
In addition, an instance has four flags. These flags indicate:
- Whether line terminators are to be returned as tokens or treated as white space that merely separates tokens.
- Whether C-style comments are to be recognized and skipped.
- Whether C++-style comments are to be recognized and skipped.
- Whether the characters of identifiers are converted to lowercase.
A typical application first constructs an instance of this class, sets up the syntax tables, and then repeatedly loops calling the nextToken method in each iteration of the loop until it returns the value TT_EOF. @author James Gosling @version 1.47, 11/30/05 @see java.io.StreamTokenizer#nextToken() @see java.io.StreamTokenizer#TT_EOF @since JDK1.0

        public static java.awt.Shape load(String fileName) {
            GeneralPath s = null;
            try {
                FileReader f = new FileReader(fileName);
                s = new GeneralPath();
                StreamTokenizer t = new StreamTokenizer(f);
                int count = 0;
                int type = 0;
                float coords[] = new float[6];
                while(t.nextToken() != StreamTokenizer.TT_EOF) {
                    switch(t.ttype) {
                    case StreamTokenizer.TT_EOL:
                        break;
                    case StreamTokenizer.TT_WORD:
                        type = findString(typeName, t.sval);

View Full Code Here

            float dash[] = bs.getDashArray();
            float dashPhase = bs.getDashPhase();


            int prop = -1;
            int dashCount = 0;
            StreamTokenizer t = new StreamTokenizer(f);


            while(t.nextToken() != StreamTokenizer.TT_EOF) {
                switch(t.ttype) {
                case StreamTokenizer.TT_EOL:
                    break;
                case StreamTokenizer.TT_WORD:
                    switch(prop) {

View Full Code Here


            StreamTokenizer t;
            String buf;


            TextTokenizer(String text) {
                t = new StreamTokenizer(new StringReader(text));
                buf = text;
            }

View Full Code Here

            }


            boolean findString(String substr) {
                int pos = buf.indexOf(substr);
                if (pos != -1) {
                    t = new StreamTokenizer(new StringReader(buf.substring(pos + substr.length())));
                }
                return pos != -1;
            }

View Full Code Here

  /**
   * @tests java.io.StreamTokenizer#StreamTokenizer(java.io.InputStream)
   */
  @SuppressWarnings("deprecation")
    public void test_ConstructorLjava_io_InputStream() throws IOException {
    st = new StreamTokenizer(new StringBufferInputStream(
        "/comments\n d 8 'h'"));


    assertEquals("the next token returned should be the letter d",
           StreamTokenizer.TT_WORD, st.nextToken());
    assertEquals("the next token returned should be the letter d",

View Full Code Here

    assertTrue("Wrong Token type8: " + st.ttype, st.ttype == -1);


    final PipedInputStream pin = new PipedInputStream();
    PipedOutputStream pout = new PipedOutputStream(pin);
    pout.write("hello\n\r\r".getBytes());
    StreamTokenizer s = new StreamTokenizer(pin);
    s.eolIsSignificant(true);
    assertTrue("Wrong token 1,1",
         s.nextToken() == StreamTokenizer.TT_WORD
         && s.sval.equals("hello"));
    assertTrue("Wrong token 1,2", s.nextToken() == '\n');
    assertTrue("Wrong token 1,3", s.nextToken() == '\n');
    assertTrue("Wrong token 1,4", s.nextToken() == '\n');
    pout.close();
    assertTrue("Wrong token 1,5",
         s.nextToken() == StreamTokenizer.TT_EOF);
    StreamTokenizer tokenizer = new StreamTokenizer(
                new Support_StringReader("\n \r\n#"));
    tokenizer.ordinaryChar('\n'); // make \n ordinary
    tokenizer.eolIsSignificant(true);
    assertTrue("Wrong token 2,1", tokenizer.nextToken() == '\n');
    assertTrue("Wrong token 2,2", tokenizer.nextToken() == '\n');
    assertEquals("Wrong token 2,3", '#', tokenizer.nextToken());
  }

View Full Code Here

     * @tests java.io.StreamTokenizer#slashSlashComments(boolean)
     */
    public void test_slashSlashComments_withSSOpen() throws IOException {
        Reader reader = new CharArrayReader( "t // t t t".toCharArray());


        StreamTokenizer st = new StreamTokenizer(reader);
        st.slashSlashComments(true);


        assertEquals(StreamTokenizer.TT_WORD,st.nextToken());
        assertEquals(StreamTokenizer.TT_EOF,st.nextToken());
    }

View Full Code Here

     * @tests java.io.StreamTokenizer#slashSlashComments(boolean)
     */
    public void test_slashSlashComments_withSSOpen_NoComment() throws IOException {
        Reader reader = new CharArrayReader( "// t".toCharArray());


        StreamTokenizer st = new StreamTokenizer(reader);
        st.slashSlashComments(true);
        st.ordinaryChar('/');


        assertEquals(StreamTokenizer.TT_EOF,st.nextToken());
    }

View Full Code Here

     * @tests java.io.StreamTokenizer#slashSlashComments(boolean)
     */
    public void test_slashSlashComments_withSSClosed() throws IOException {
        Reader reader = new CharArrayReader( "// t".toCharArray());


        StreamTokenizer st = new StreamTokenizer(reader);
        st.slashSlashComments(false);
        st.ordinaryChar('/');


        assertEquals('/',st.nextToken());
        assertEquals('/',st.nextToken());
        assertEquals(StreamTokenizer.TT_WORD,st.nextToken());
    }

View Full Code Here

     * @tests java.io.StreamTokenizer#slashStarComments(boolean)
     */
    public void test_slashStarComments_withSTOpen() throws IOException {
        Reader reader = new CharArrayReader( "t /* t */ t".toCharArray());


        StreamTokenizer st = new StreamTokenizer(reader);
        st.slashStarComments(true);


        assertEquals(StreamTokenizer.TT_WORD,st.nextToken());
        assertEquals(StreamTokenizer.TT_WORD,st.nextToken());
        assertEquals(StreamTokenizer.TT_EOF,st.nextToken());
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of java.io.StreamTokenizer

co.cask.cdap.logging.filter.FilterParser

com.eteks.sweethome3d.j3d.OBJLoader

com.eva.properties.PropertiesParser

com.google.devtools.moe.client.parser.Parser

com.liferay.util.lang.ClassUtil

com.nick125.thingamabob.parser.Lexer

com.orange.wink.parse.ParserUtils

com.skaringa.json.parser.JsonParser

com.sun.tools.apt.main.CommandLine

com.sun.tools.javac.main.CommandLine

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.