Mercurial > hg > Members > kono > PLparser
view src/plparser/PropertyListStreamScanner.java @ 9:29e309b2f624
Try several Tokenizer
author | one |
---|---|
date | Thu, 02 Sep 2010 10:00:23 +0900 |
parents | |
children | 79d492bce828 |
line wrap: on
line source
package plparser; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.io.StringReader; import java.util.Scanner; import java.util.regex.Pattern; /** * delimiter に何を設定しても動いてくれない。 * */ public class PropertyListStreamScanner<T> extends PLScannerImpl<T> implements PLScanner<T> { private Scanner scan; public PropertyListStreamScanner( PLScanner<T> s, Dictionary<T> dict, Token<T> nullToken) { this.dict = dict; this.nullToken = nullToken; } public PropertyListStreamScanner(Dictionary<T> dict) { this.dict = dict; nullToken = new Token<T>("",TokenID.NULL); } public void init() { String pattern = "."; scan.useDelimiter(pattern); } // Pattern must contain exact 1 group private static Pattern tokenPat = Pattern.compile( "([={}(),;])" ); private static Pattern namePat = Pattern.compile("([_a-zA-Z][\\@\\w]*)"); private static final Pattern numPat = Pattern.compile("([0-9]+)"); private static final Pattern stringPat1 = Pattern.compile("\\\"([^\"]*)\\\""); private static final Pattern stringPat = Pattern.compile("\\'([^\\']*)\\'"); private static final Pattern stringPat1cont = Pattern.compile("\\\"([^\"]*)$"); private static final Pattern stringPatCont = Pattern.compile("\\'([^\\']*)$"); private static final Pattern stringPat1End = Pattern.compile("([^\"]*)\\\""); private static final Pattern stringPatEnd = Pattern.compile("([^\\']*)\'"); private static final Pattern commentPat = Pattern.compile("(//.*)"); private static final Pattern commentPat1 = Pattern.compile("(/\\*)"); private static final Pattern commentPat1End = Pattern.compile("(.*\\*/)"); private static final Pattern errorPat = Pattern.compile("([^\\s])"); private static final Pattern anyPat = Pattern.compile("(.)"); @Override public Token<T> nextToken() { String s; nextToken = nullToken; while(hasRemaining()) { if ((s=scan.next(tokenPat))!=null) { Token<T> t; if ((t = dict.get(s))==null) { dict.put(s, t = new Token<T>(s,TokenID.Any)); } return nextToken = t; } else if ((s=scan.next(stringPatCont))!=null) { // non terminated string String s1; while((s1=scan.next(stringPatEnd))==null) { s += scan.nextLine(); lineno++; } s += s1; Token<T> t; if ((t = dict.get(s))==null) { dict.put(s, t = new Token<T>(s,TokenID.VARIABLE)); } return nextToken = t; } else if ((s=scan.next(stringPat1cont))!=null) { // non terminated string String s1; while((s1=scan.next(stringPat1End))==null) { s += scan.nextLine(); lineno++; } s += s1; Token<T> t; if ((t = dict.get(s))==null) { dict.put(s, t = new Token<T>(s,TokenID.VARIABLE)); } return nextToken = t; } else if ((s=scan.next(stringPat))!=null||(s=scan.next(stringPat1))!=null||(s=scan.next(namePat))!=null) { Token<T> t; if ((t = dict.get(s))==null) { dict.put(s, t = new Token<T>(s,TokenID.VARIABLE)); } if (t.type!=TokenID.VARIABLE) { t = new Token<T>(s,TokenID.VARIABLE); } return nextToken = t; } else if ((s=scan.next(numPat))!=null) { return nextToken = new Token<T>(s,TokenID.NUMBER); } else if ((s=scan.next(commentPat))!=null) { scan.nextLine(); continue; } else if ((s=scan.next(commentPat1))!=null) { while(scan.next(commentPat1End)==null) { scan.nextLine(); lineno++; } continue; } else if ((s=scan.next(errorPat))!=null) { error("Don't understand '"+s+"'"); continue; } else if ((s=scan.next(anyPat))!=null) { // skip space continue; } else { lineno++; } } return nextToken; } @Override public boolean hasRemaining() { return scan.hasNext(anyPat); } @Override public PLScanner<T> pushScannerFile(InputStream newfile, String prompt) { return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(newfile,prompt); } @Override public PLScanner<T> pushScanner(String exp) { return new PropertyListStreamScanner<T>(this,dict,nullToken).set(exp); } @Override public PLScanner<T> pushScannerFile(String file) throws FileNotFoundException { return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(file); } @Override public PLScanner<T> set(String exp) { Reader reader = new StringReader(exp); scan = new Scanner(reader); return this; } @Override public PLScanner<T> setFile(String file) throws FileNotFoundException { Reader reader = new FileReader(file); scan = new Scanner(reader); return this; } @Override public PLScanner<T> set(InputStreamReader reader) { scan = new Scanner(reader); return this; } }