Mercurial > hg > Members > kono > PLparser
diff src/plparser/PropertyListStreamScanner.java @ 9:29e309b2f624
Try several Tokenizer
author | one |
---|---|
date | Thu, 02 Sep 2010 10:00:23 +0900 |
parents | |
children | 79d492bce828 |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plparser/PropertyListStreamScanner.java Thu Sep 02 10:00:23 2010 +0900 @@ -0,0 +1,174 @@ +package plparser; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.StringReader; +import java.util.Scanner; +import java.util.regex.Pattern; + +/** + * delimiter に何を設定しても動いてくれない。 + * */ + +public class PropertyListStreamScanner<T> extends PLScannerImpl<T> implements + PLScanner<T> { + + private Scanner scan; + + public PropertyListStreamScanner( + PLScanner<T> s, + Dictionary<T> dict, Token<T> nullToken) { + this.dict = dict; + this.nullToken = nullToken; + } + + + public PropertyListStreamScanner(Dictionary<T> dict) { + this.dict = dict; + nullToken = new Token<T>("",TokenID.NULL); + } + + + public void init() { + String pattern = "."; + scan.useDelimiter(pattern); + } + + // Pattern must contain exact 1 group + private static Pattern tokenPat = Pattern.compile( + "([={}(),;])" + ); + private static Pattern namePat = Pattern.compile("([_a-zA-Z][\\@\\w]*)"); + private static final Pattern numPat = Pattern.compile("([0-9]+)"); + private static final Pattern stringPat1 = Pattern.compile("\\\"([^\"]*)\\\""); + private static final Pattern stringPat = Pattern.compile("\\'([^\\']*)\\'"); + private static final Pattern stringPat1cont = Pattern.compile("\\\"([^\"]*)$"); + private static final Pattern stringPatCont = Pattern.compile("\\'([^\\']*)$"); + private static final Pattern stringPat1End = Pattern.compile("([^\"]*)\\\""); + private static final Pattern stringPatEnd = Pattern.compile("([^\\']*)\'"); + private static final Pattern commentPat = Pattern.compile("(//.*)"); + private static final Pattern commentPat1 = Pattern.compile("(/\\*)"); + private static final Pattern commentPat1End = Pattern.compile("(.*\\*/)"); + private static final Pattern errorPat = Pattern.compile("([^\\s])"); + private static final Pattern anyPat = Pattern.compile("(.)"); + + @Override + public Token<T> nextToken() { + String s; + nextToken = nullToken; + while(hasRemaining()) { + if ((s=scan.next(tokenPat))!=null) { + Token<T> t; + if ((t = dict.get(s))==null) { + dict.put(s, t = new Token<T>(s,TokenID.Any)); + } + return nextToken = t; + } else if ((s=scan.next(stringPatCont))!=null) { + // non terminated string + String s1; + while((s1=scan.next(stringPatEnd))==null) { + s += scan.nextLine(); + lineno++; + } + s += s1; + Token<T> t; + if ((t = dict.get(s))==null) { + dict.put(s, t = new Token<T>(s,TokenID.VARIABLE)); + } + return nextToken = t; + } else if ((s=scan.next(stringPat1cont))!=null) { + // non terminated string + String s1; + while((s1=scan.next(stringPat1End))==null) { + s += scan.nextLine(); + lineno++; + } + s += s1; + Token<T> t; + if ((t = dict.get(s))==null) { + dict.put(s, t = new Token<T>(s,TokenID.VARIABLE)); + } + return nextToken = t; + } else if ((s=scan.next(stringPat))!=null||(s=scan.next(stringPat1))!=null||(s=scan.next(namePat))!=null) { + Token<T> t; + if ((t = dict.get(s))==null) { + dict.put(s, t = new Token<T>(s,TokenID.VARIABLE)); + } + if (t.type!=TokenID.VARIABLE) { + t = new Token<T>(s,TokenID.VARIABLE); + } + return nextToken = t; + } else if ((s=scan.next(numPat))!=null) { + return nextToken = new Token<T>(s,TokenID.NUMBER); + } else if ((s=scan.next(commentPat))!=null) { + scan.nextLine(); + continue; + } else if ((s=scan.next(commentPat1))!=null) { + while(scan.next(commentPat1End)==null) { + scan.nextLine(); + lineno++; + } + continue; + } else if ((s=scan.next(errorPat))!=null) { + error("Don't understand '"+s+"'"); + continue; + } else if ((s=scan.next(anyPat))!=null) { + // skip space + continue; + } else { + lineno++; + } + } + return nextToken; + } + + + @Override + public boolean hasRemaining() { + return scan.hasNext(anyPat); + } + + @Override + public PLScanner<T> pushScannerFile(InputStream newfile, + String prompt) { + return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(newfile,prompt); + } + + @Override + public PLScanner<T> pushScanner(String exp) { + return new PropertyListStreamScanner<T>(this,dict,nullToken).set(exp); + } + + @Override + public PLScanner<T> pushScannerFile(String file) + throws FileNotFoundException { + return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(file); + } + + @Override + public PLScanner<T> set(String exp) { + Reader reader = new StringReader(exp); + scan = new Scanner(reader); + return this; + } + + @Override + public PLScanner<T> setFile(String file) + throws FileNotFoundException { + Reader reader = new FileReader(file); + scan = new Scanner(reader); + return this; + } + + @Override + public PLScanner<T> set(InputStreamReader reader) { + scan = new Scanner(reader); + return this; + } + + + +}