Mercurial > hg > Members > kono > PLparser

comparison src/plparser/PropertyListStreamScanner.java @ 9:29e309b2f624

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Try several Tokenizer
author one
date Thu, 02 Sep 2010 10:00:23 +0900
parents
children 79d492bce828
comparison
equal deleted inserted replaced
8:8d0f9c1816f5 9:29e309b2f624
1 package plparser;
2
3 import java.io.FileNotFoundException;
4 import java.io.FileReader;
5 import java.io.InputStream;
6 import java.io.InputStreamReader;
7 import java.io.Reader;
8 import java.io.StringReader;
9 import java.util.Scanner;
10 import java.util.regex.Pattern;
11
12 /**
13 * delimiter に何を設定しても動いてくれない。
14 * */
15
16 public class PropertyListStreamScanner<T> extends PLScannerImpl<T> implements
17 PLScanner<T> {
18
19 private Scanner scan;
20
21 public PropertyListStreamScanner(
22 PLScanner<T> s,
23 Dictionary<T> dict, Token<T> nullToken) {
24 this.dict = dict;
25 this.nullToken = nullToken;
26 }
27
28
29 public PropertyListStreamScanner(Dictionary<T> dict) {
30 this.dict = dict;
31 nullToken = new Token<T>("",TokenID.NULL);
32 }
33
34
35 public void init() {
36 String pattern = ".";
37 scan.useDelimiter(pattern);
38 }
39
40 // Pattern must contain exact 1 group
41 private static Pattern tokenPat = Pattern.compile(
42 "([={}(),;])"
43 );
44 private static Pattern namePat = Pattern.compile("([_a-zA-Z][\\@\\w]*)");
45 private static final Pattern numPat = Pattern.compile("([0-9]+)");
46 private static final Pattern stringPat1 = Pattern.compile("\\\"([^\"]*)\\\"");
47 private static final Pattern stringPat = Pattern.compile("\\'([^\\']*)\\'");
48 private static final Pattern stringPat1cont = Pattern.compile("\\\"([^\"]*)$");
49 private static final Pattern stringPatCont = Pattern.compile("\\'([^\\']*)$");
50 private static final Pattern stringPat1End = Pattern.compile("([^\"]*)\\\"");
51 private static final Pattern stringPatEnd = Pattern.compile("([^\\']*)\'");
52 private static final Pattern commentPat = Pattern.compile("(//.*)");
53 private static final Pattern commentPat1 = Pattern.compile("(/\\*)");
54 private static final Pattern commentPat1End = Pattern.compile("(.*\\*/)");
55 private static final Pattern errorPat = Pattern.compile("([^\\s])");
56 private static final Pattern anyPat = Pattern.compile("(.)");
57
58 @Override
59 public Token<T> nextToken() {
60 String s;
61 nextToken = nullToken;
62 while(hasRemaining()) {
63 if ((s=scan.next(tokenPat))!=null) {
64 Token<T> t;
65 if ((t = dict.get(s))==null) {
66 dict.put(s, t = new Token<T>(s,TokenID.Any));
67 }
68 return nextToken = t;
69 } else if ((s=scan.next(stringPatCont))!=null) {
70 // non terminated string
71 String s1;
72 while((s1=scan.next(stringPatEnd))==null) {
73 s += scan.nextLine();
74 lineno++;
75 }
76 s += s1;
77 Token<T> t;
78 if ((t = dict.get(s))==null) {
79 dict.put(s, t = new Token<T>(s,TokenID.VARIABLE));
80 }
81 return nextToken = t;
82 } else if ((s=scan.next(stringPat1cont))!=null) {
83 // non terminated string
84 String s1;
85 while((s1=scan.next(stringPat1End))==null) {
86 s += scan.nextLine();
87 lineno++;
88 }
89 s += s1;
90 Token<T> t;
91 if ((t = dict.get(s))==null) {
92 dict.put(s, t = new Token<T>(s,TokenID.VARIABLE));
93 }
94 return nextToken = t;
95 } else if ((s=scan.next(stringPat))!=null||(s=scan.next(stringPat1))!=null||(s=scan.next(namePat))!=null) {
96 Token<T> t;
97 if ((t = dict.get(s))==null) {
98 dict.put(s, t = new Token<T>(s,TokenID.VARIABLE));
99 }
100 if (t.type!=TokenID.VARIABLE) {
101 t = new Token<T>(s,TokenID.VARIABLE);
102 }
103 return nextToken = t;
104 } else if ((s=scan.next(numPat))!=null) {
105 return nextToken = new Token<T>(s,TokenID.NUMBER);
106 } else if ((s=scan.next(commentPat))!=null) {
107 scan.nextLine();
108 continue;
109 } else if ((s=scan.next(commentPat1))!=null) {
110 while(scan.next(commentPat1End)==null) {
111 scan.nextLine();
112 lineno++;
113 }
114 continue;
115 } else if ((s=scan.next(errorPat))!=null) {
116 error("Don't understand '"+s+"'");
117 continue;
118 } else if ((s=scan.next(anyPat))!=null) {
119 // skip space
120 continue;
121 } else {
122 lineno++;
123 }
124 }
125 return nextToken;
126 }
127
128
129 @Override
130 public boolean hasRemaining() {
131 return scan.hasNext(anyPat);
132 }
133
134 @Override
135 public PLScanner<T> pushScannerFile(InputStream newfile,
136 String prompt) {
137 return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(newfile,prompt);
138 }
139
140 @Override
141 public PLScanner<T> pushScanner(String exp) {
142 return new PropertyListStreamScanner<T>(this,dict,nullToken).set(exp);
143 }
144
145 @Override
146 public PLScanner<T> pushScannerFile(String file)
147 throws FileNotFoundException {
148 return new PropertyListStreamScanner<T>(this,dict,nullToken).setFile(file);
149 }
150
151 @Override
152 public PLScanner<T> set(String exp) {
153 Reader reader = new StringReader(exp);
154 scan = new Scanner(reader);
155 return this;
156 }
157
158 @Override
159 public PLScanner<T> setFile(String file)
160 throws FileNotFoundException {
161 Reader reader = new FileReader(file);
162 scan = new Scanner(reader);
163 return this;
164 }
165
166 @Override
167 public PLScanner<T> set(InputStreamReader reader) {
168 scan = new Scanner(reader);
169 return this;
170 }
171
172
173
174 }