diff src/plparser/PropertyListStreamScanner.java @ 9:29e309b2f624

Try several Tokenizer
author one
date Thu, 02 Sep 2010 10:00:23 +0900
parents
children 79d492bce828
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plparser/PropertyListStreamScanner.java	Thu Sep 02 10:00:23 2010 +0900
@@ -0,0 +1,174 @@
+package plparser;
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.Scanner;
+import java.util.regex.Pattern;
+
+/**
+ * delimiter に何を設定しても動いてくれない。
+ * */
+
+public class PropertyListStreamScanner<T> extends PLScannerImpl<T> implements
+		PLScanner<T> {
+
+	private Scanner scan;
+	
+	public PropertyListStreamScanner(
+			PLScanner<T> s,
+			Dictionary<T> dict, Token<T> nullToken) {
+		this.dict = dict;
+		this.nullToken = nullToken;
+	}
+
+
+	public PropertyListStreamScanner(Dictionary<T> dict) {
+		this.dict = dict;
+		nullToken = new Token<T>("",TokenID.NULL);
+	}
+
+
+	public void init() {
+		String pattern = ".";
+		scan.useDelimiter(pattern);
+	}
+
+	// Pattern must contain exact 1 group
+	private static Pattern tokenPat = Pattern.compile(
+			"([={}(),;])"
+	);
+	private static Pattern namePat  = Pattern.compile("([_a-zA-Z][\\@\\w]*)");
+	private static final Pattern numPat  = Pattern.compile("([0-9]+)");
+	private static final Pattern stringPat1  = Pattern.compile("\\\"([^\"]*)\\\"");
+	private static final Pattern stringPat  = Pattern.compile("\\'([^\\']*)\\'");
+	private static final Pattern stringPat1cont  = Pattern.compile("\\\"([^\"]*)$");
+	private static final Pattern stringPatCont  = Pattern.compile("\\'([^\\']*)$");
+	private static final Pattern stringPat1End  = Pattern.compile("([^\"]*)\\\"");
+	private static final Pattern stringPatEnd  = Pattern.compile("([^\\']*)\'");
+	private static final Pattern commentPat  = Pattern.compile("(//.*)");
+	private static final Pattern commentPat1  = Pattern.compile("(/\\*)");
+	private static final Pattern commentPat1End  = Pattern.compile("(.*\\*/)");
+	private static final Pattern errorPat = Pattern.compile("([^\\s])");
+	private static final Pattern anyPat = Pattern.compile("(.)");
+	
+	@Override
+	public Token<T> nextToken() {
+		String s;
+		nextToken = nullToken;
+		while(hasRemaining()) {
+			if ((s=scan.next(tokenPat))!=null) {
+				Token<T> t;
+				if ((t = dict.get(s))==null) {
+					dict.put(s, t = new Token<T>(s,TokenID.Any));
+				}
+				return nextToken = t;
+			} else if ((s=scan.next(stringPatCont))!=null) {
+				// non terminated string
+				String s1;
+				while((s1=scan.next(stringPatEnd))==null) {
+					s += scan.nextLine();
+					lineno++;
+				}
+				s += s1;
+				Token<T> t;
+				if ((t = dict.get(s))==null) {
+					dict.put(s, t = new Token<T>(s,TokenID.VARIABLE));
+				}
+				return nextToken = t;
+			} else if ((s=scan.next(stringPat1cont))!=null) {
+				// non terminated string
+				String s1;
+				while((s1=scan.next(stringPat1End))==null) {
+					s += scan.nextLine();
+					lineno++;
+				}
+				s += s1;
+				Token<T> t;
+				if ((t = dict.get(s))==null) {
+					dict.put(s, t = new Token<T>(s,TokenID.VARIABLE));
+				}
+				return nextToken = t;
+			} else if ((s=scan.next(stringPat))!=null||(s=scan.next(stringPat1))!=null||(s=scan.next(namePat))!=null) {
+				Token<T> t;
+				if ((t = dict.get(s))==null) {
+					dict.put(s, t = new Token<T>(s,TokenID.VARIABLE));
+				}
+				if (t.type!=TokenID.VARIABLE) {
+					t = new Token<T>(s,TokenID.VARIABLE);
+				}
+				return nextToken = t;
+			} else if ((s=scan.next(numPat))!=null) {
+				return nextToken = new Token<T>(s,TokenID.NUMBER);
+			} else if ((s=scan.next(commentPat))!=null) {
+				scan.nextLine();
+				continue;
+			} else if ((s=scan.next(commentPat1))!=null) {
+				while(scan.next(commentPat1End)==null) {
+					scan.nextLine();
+					lineno++;
+				}
+				continue;
+			} else if ((s=scan.next(errorPat))!=null) {
+				error("Don't understand '"+s+"'");
+				continue;
+			} else if ((s=scan.next(anyPat))!=null) {
+				// skip space
+				continue;
+			} else {
+				lineno++;
+			}
+		}
+		return nextToken;
+	}
+
+
+	@Override
+	public boolean hasRemaining() {
+		return scan.hasNext(anyPat);
+	}
+
+	@Override
+	public PLScanner<T> pushScannerFile(InputStream newfile,
+			String prompt) {
+		return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(newfile,prompt);
+	}
+
+	@Override
+	public PLScanner<T> pushScanner(String exp) {
+		return new PropertyListStreamScanner<T>(this,dict,nullToken).set(exp);
+	}
+
+	@Override
+	public PLScanner<T> pushScannerFile(String file)
+			throws FileNotFoundException {
+		return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(file);
+	}
+
+	@Override
+	public PLScanner<T> set(String exp) {
+		Reader reader = new StringReader(exp);
+		scan = new Scanner(reader);
+		return this;
+	}
+
+	@Override
+	public PLScanner<T> setFile(String file)
+			throws FileNotFoundException {
+		Reader reader = new FileReader(file);
+		scan = new Scanner(reader);
+		return this;
+	}
+
+	@Override
+	public PLScanner<T> set(InputStreamReader reader) {
+		scan = new Scanner(reader);
+		return this;
+	}
+
+
+
+}