Mercurial > hg > CbC > CbC_gcc

comparison contrib/unicode/from_glibc/utf8_gen.py @ 145:1830386684a0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
gcc-9.2.0
author anatofuz
date Thu, 13 Feb 2020 11:34:05 +0900
parents
children
comparison
equal deleted inserted replaced
131:84e7813d76e9 145:1830386684a0
1 #!/usr/bin/python3
2 # -*- coding: utf-8 -*-
3 # Copyright (C) 2014-2019 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
5 #
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
10 #
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
15 #
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <https://www.gnu.org/licenses/>.
19
20 '''glibc/localedata/charmaps/UTF-8 file generator script
21
22 This script generates a glibc/localedata/charmaps/UTF-8 file
23 from Unicode data.
24
25 Usage: python3 utf8_gen.py UnicodeData.txt EastAsianWidth.txt
26
27 It will output UTF-8 file
28 '''
29
30 import argparse
31 import sys
32 import re
33 import unicode_utils
34
35 # Auxiliary tables for Hangul syllable names, see the Unicode 3.0 book,
36 # sections 3.11 and 4.4.
37
38 JAMO_INITIAL_SHORT_NAME = (
39 'G', 'GG', 'N', 'D', 'DD', 'R', 'M', 'B', 'BB', 'S', 'SS', '', 'J', 'JJ',
40 'C', 'K', 'T', 'P', 'H'
41 )
42
43 JAMO_MEDIAL_SHORT_NAME = (
44 'A', 'AE', 'YA', 'YAE', 'EO', 'E', 'YEO', 'YE', 'O', 'WA', 'WAE', 'OE',
45 'YO', 'U', 'WEO', 'WE', 'WI', 'YU', 'EU', 'YI', 'I'
46 )
47
48 JAMO_FINAL_SHORT_NAME = (
49 '', 'G', 'GG', 'GS', 'N', 'NI', 'NH', 'D', 'L', 'LG', 'LM', 'LB', 'LS',
50 'LT', 'LP', 'LH', 'M', 'B', 'BS', 'S', 'SS', 'NG', 'J', 'C', 'K', 'T',
51 'P', 'H'
52 )
53
54 def process_range(start, end, outfile, name):
55 '''Writes a range of code points into the CHARMAP section of the
56 output file
57
58 '''
59 if 'Hangul Syllable' in name:
60 # from glibc/localedata/ChangeLog:
61 #
62 # 2000-09-24 Bruno Haible <haible@clisp.cons.org>
63 # * charmaps/UTF-8: Expand <Hangul Syllable> and <Private Use> ranges,
64 # so they become printable and carry a width. Comment out surrogate
65 # ranges. Add a WIDTH table
66 #
67 # So we expand the Hangul Syllables here:
68 for i in range(int(start, 16), int(end, 16)+1 ):
69 index2, index3 = divmod(i - 0xaC00, 28)
70 index1, index2 = divmod(index2, 21)
71 hangul_syllable_name = 'HANGUL SYLLABLE ' \
72 + JAMO_INITIAL_SHORT_NAME[index1] \
73 + JAMO_MEDIAL_SHORT_NAME[index2] \
74 + JAMO_FINAL_SHORT_NAME[index3]
75 outfile.write('{:<11s} {:<12s} {:s}\n'.format(
76 unicode_utils.ucs_symbol(i), convert_to_hex(i),
77 hangul_syllable_name))
78 return
79 # UnicodeData.txt file has contains code point ranges like this:
80 #
81 # 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
82 # 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
83 #
84 # The glibc UTF-8 file splits ranges like these into shorter
85 # ranges of 64 code points each:
86 #
87 # <U3400>..<U343F> /xe3/x90/x80 <CJK Ideograph Extension A>
88 # …
89 # <U4D80>..<U4DB5> /xe4/xb6/x80 <CJK Ideograph Extension A>
90 for i in range(int(start, 16), int(end, 16), 64 ):
91 if i > (int(end, 16)-64):
92 outfile.write('{:s}..{:s} {:<12s} {:s}\n'.format(
93 unicode_utils.ucs_symbol(i),
94 unicode_utils.ucs_symbol(int(end,16)),
95 convert_to_hex(i),
96 name))
97 break
98 outfile.write('{:s}..{:s} {:<12s} {:s}\n'.format(
99 unicode_utils.ucs_symbol(i),
100 unicode_utils.ucs_symbol(i+63),
101 convert_to_hex(i),
102 name))
103
104 def process_charmap(flines, outfile):
105 '''This function takes an array which contains *all* lines of
106 of UnicodeData.txt and write lines to outfile as used in the
107
108 CHARMAP
109 …
110 END CHARMAP
111
112 section of the UTF-8 file in glibc/localedata/charmaps/UTF-8.
113
114 Samples for input lines:
115
116 0010;<control>;Cc;0;BN;;;;;N;DATA LINK ESCAPE;;;;
117 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
118 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
119 D800;<Non Private Use High Surrogate, First>;Cs;0;L;;;;;N;;;;;
120 DB7F;<Non Private Use High Surrogate, Last>;Cs;0;L;;;;;N;;;;;
121 100000;<Plane 16 Private Use, First>;Co;0;L;;;;;N;;;;;
122 10FFFD;<Plane 16 Private Use, Last>;Co;0;L;;;;;N;;;;;
123
124 Samples for output lines (Unicode-Value UTF-8-HEX Unicode-Char-Name):
125
126 <U0010> /x10 DATA LINK ESCAPE
127 <U3400>..<U343F> /xe3/x90/x80 <CJK Ideograph Extension A>
128 %<UD800> /xed/xa0/x80 <Non Private Use High Surrogate, First>
129 %<UDB7F> /xed/xad/xbf <Non Private Use High Surrogate, Last>
130 <U0010FFC0>..<U0010FFFD> /xf4/x8f/xbf/x80 <Plane 16 Private Use>
131
132 '''
133 fields_start = []
134 for line in flines:
135 fields = line.split(";")
136 # Some characters have “<control>” as their name. We try to
137 # use the “Unicode 1.0 Name” (10th field in
138 # UnicodeData.txt) for them.
139 #
140 # The Characters U+0080, U+0081, U+0084 and U+0099 have
141 # “<control>” as their name but do not even have aa
142 # ”Unicode 1.0 Name”. We could write code to take their
143 # alternate names from NameAliases.txt.
144 if fields[1] == "<control>" and fields[10]:
145 fields[1] = fields[10]
146 # Handling code point ranges like:
147 #
148 # 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
149 # 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
150 if fields[1].endswith(', First>') and not 'Surrogate,' in fields[1]:
151 fields_start = fields
152 continue
153 if fields[1].endswith(', Last>') and not 'Surrogate,' in fields[1]:
154 process_range(fields_start[0], fields[0],
155 outfile, fields[1][:-7]+'>')
156 fields_start = []
157 continue
158 fields_start = []
159 if 'Surrogate,' in fields[1]:
160 # Comment out the surrogates in the UTF-8 file.
161 # One could of course skip them completely but
162 # the original UTF-8 file in glibc had them as
163 # comments, so we keep these comment lines.
164 outfile.write('%')
165 outfile.write('{:<11s} {:<12s} {:s}\n'.format(
166 unicode_utils.ucs_symbol(int(fields[0], 16)),
167 convert_to_hex(int(fields[0], 16)),
168 fields[1]))
169
170 def convert_to_hex(code_point):
171 '''Converts a code point to a hexadecimal UTF-8 representation
172 like /x**/x**/x**.'''
173 # Getting UTF8 of Unicode characters.
174 # In Python3, .encode('UTF-8') does not work for
175 # surrogates. Therefore, we use this conversion table
176 surrogates = {
177 0xD800: '/xed/xa0/x80',
178 0xDB7F: '/xed/xad/xbf',
179 0xDB80: '/xed/xae/x80',
180 0xDBFF: '/xed/xaf/xbf',
181 0xDC00: '/xed/xb0/x80',
182 0xDFFF: '/xed/xbf/xbf',
183 }
184 if code_point in surrogates:
185 return surrogates[code_point]
186 return ''.join([
187 '/x{:02x}'.format(c) for c in chr(code_point).encode('UTF-8')
188 ])
189
190 def write_header_charmap(outfile):
191 '''Write the header on top of the CHARMAP section to the output file'''
192 outfile.write("<code_set_name> UTF-8\n")
193 outfile.write("<comment_char> %\n")
194 outfile.write("<escape_char> /\n")
195 outfile.write("<mb_cur_min> 1\n")
196 outfile.write("<mb_cur_max> 6\n\n")
197 outfile.write("% CHARMAP generated using utf8_gen.py\n")
198 outfile.write("% alias ISO-10646/UTF-8\n")
199 outfile.write("CHARMAP\n")
200
201 def write_header_width(outfile, unicode_version):
202 '''Writes the header on top of the WIDTH section to the output file'''
203 outfile.write('% Character width according to Unicode '
204 + '{:s}.\n'.format(unicode_version))
205 outfile.write('% - Default width is 1.\n')
206 outfile.write('% - Double-width characters have width 2; generated from\n')
207 outfile.write('% "grep \'^[^;]*;[WF]\' EastAsianWidth.txt"\n')
208 outfile.write('% - Non-spacing characters have width 0; '
209 + 'generated from PropList.txt or\n')
210 outfile.write('% "grep \'^[^;]*;[^;]*;[^;]*;[^;]*;NSM;\' '
211 + 'UnicodeData.txt"\n')
212 outfile.write('% - Format control characters have width 0; '
213 + 'generated from\n')
214 outfile.write("% \"grep '^[^;]*;[^;]*;Cf;' UnicodeData.txt\"\n")
215 # Not needed covered by Cf
216 # outfile.write("% - Zero width characters have width 0; generated from\n")
217 # outfile.write("% \"grep '^[^;]*;ZERO WIDTH ' UnicodeData.txt\"\n")
218 outfile.write("WIDTH\n")
219
220 def process_width(outfile, ulines, elines, plines):
221 '''ulines are lines from UnicodeData.txt, elines are lines from
222 EastAsianWidth.txt containing characters with width “W” or “F”,
223 plines are lines from PropList.txt which contain characters
224 with the property “Prepended_Concatenation_Mark”.
225
226 '''
227 width_dict = {}
228 for line in elines:
229 fields = line.split(";")
230 if not '..' in fields[0]:
231 code_points = (fields[0], fields[0])
232 else:
233 code_points = fields[0].split("..")
234 for key in range(int(code_points[0], 16),
235 int(code_points[1], 16)+1):
236 width_dict[key] = 2
237
238 for line in ulines:
239 fields = line.split(";")
240 if fields[4] == "NSM" or fields[2] in ("Cf", "Me", "Mn"):
241 width_dict[int(fields[0], 16)] = 0
242
243 for line in plines:
244 # Characters with the property “Prepended_Concatenation_Mark”
245 # should have the width 1:
246 fields = line.split(";")
247 if not '..' in fields[0]:
248 code_points = (fields[0], fields[0])
249 else:
250 code_points = fields[0].split("..")
251 for key in range(int(code_points[0], 16),
252 int(code_points[1], 16)+1):
253 del width_dict[key] # default width is 1
254
255 # handle special cases for compatibility
256 for key in list((0x00AD,)):
257 # https://www.cs.tut.fi/~jkorpela/shy.html
258 if key in width_dict:
259 del width_dict[key] # default width is 1
260 for key in list(range(0x1160, 0x1200)):
261 width_dict[key] = 0
262 for key in list(range(0x3248, 0x3250)):
263 # These are “A” which means we can decide whether to treat them
264 # as “W” or “N” based on context:
265 # http://www.unicode.org/mail-arch/unicode-ml/y2017-m08/0023.html
266 # For us, “W” seems better.
267 width_dict[key] = 2
268 for key in list(range(0x4DC0, 0x4E00)):
269 width_dict[key] = 2
270
271 same_width_lists = []
272 current_width_list = []
273 for key in sorted(width_dict):
274 if not current_width_list:
275 current_width_list = [key]
276 elif (key == current_width_list[-1] + 1
277 and width_dict[key] == width_dict[current_width_list[0]]):
278 current_width_list.append(key)
279 else:
280 same_width_lists.append(current_width_list)
281 current_width_list = [key]
282 if current_width_list:
283 same_width_lists.append(current_width_list)
284
285 for same_width_list in same_width_lists:
286 if len(same_width_list) == 1:
287 outfile.write('{:s}\t{:d}\n'.format(
288 unicode_utils.ucs_symbol(same_width_list[0]),
289 width_dict[same_width_list[0]]))
290 else:
291 outfile.write('{:s}...{:s}\t{:d}\n'.format(
292 unicode_utils.ucs_symbol(same_width_list[0]),
293 unicode_utils.ucs_symbol(same_width_list[-1]),
294 width_dict[same_width_list[0]]))
295
296 if __name__ == "__main__":
297 PARSER = argparse.ArgumentParser(
298 description='''
299 Generate a UTF-8 file from UnicodeData.txt, EastAsianWidth.txt, and PropList.txt.
300 ''')
301 PARSER.add_argument(
302 '-u', '--unicode_data_file',
303 nargs='?',
304 type=str,
305 default='UnicodeData.txt',
306 help=('The UnicodeData.txt file to read, '
307 + 'default: %(default)s'))
308 PARSER.add_argument(
309 '-e', '--east_asian_with_file',
310 nargs='?',
311 type=str,
312 default='EastAsianWidth.txt',
313 help=('The EastAsianWidth.txt file to read, '
314 + 'default: %(default)s'))
315 PARSER.add_argument(
316 '-p', '--prop_list_file',
317 nargs='?',
318 type=str,
319 default='PropList.txt',
320 help=('The PropList.txt file to read, '
321 + 'default: %(default)s'))
322 PARSER.add_argument(
323 '--unicode_version',
324 nargs='?',
325 required=True,
326 type=str,
327 help='The Unicode version of the input files used.')
328 ARGS = PARSER.parse_args()
329
330 with open(ARGS.unicode_data_file, mode='r') as UNIDATA_FILE:
331 UNICODE_DATA_LINES = UNIDATA_FILE.readlines()
332 with open(ARGS.east_asian_with_file, mode='r') as EAST_ASIAN_WIDTH_FILE:
333 EAST_ASIAN_WIDTH_LINES = []
334 for LINE in EAST_ASIAN_WIDTH_FILE:
335 # If characters from EastAasianWidth.txt which are from
336 # from reserved ranges (i.e. not yet assigned code points)
337 # are added to the WIDTH section of the UTF-8 file, then
338 # “make check” produces “Unknown Character” errors for
339 # these code points because such unassigned code points
340 # are not in the CHARMAP section of the UTF-8 file.
341 #
342 # Therefore, we skip all reserved code points when reading
343 # the EastAsianWidth.txt file.
344 if re.match(r'.*<reserved-.+>\.\.<reserved-.+>.*', LINE):
345 continue
346 if re.match(r'^[^;]*;[WF]', LINE):
347 EAST_ASIAN_WIDTH_LINES.append(LINE.strip())
348 with open(ARGS.prop_list_file, mode='r') as PROP_LIST_FILE:
349 PROP_LIST_LINES = []
350 for LINE in PROP_LIST_FILE:
351 if re.match(r'^[^;]*;[\s]*Prepended_Concatenation_Mark', LINE):
352 PROP_LIST_LINES.append(LINE.strip())
353 with open('UTF-8', mode='w') as OUTFILE:
354 # Processing UnicodeData.txt and write CHARMAP to UTF-8 file
355 write_header_charmap(OUTFILE)
356 process_charmap(UNICODE_DATA_LINES, OUTFILE)
357 OUTFILE.write("END CHARMAP\n\n")
358 # Processing EastAsianWidth.txt and write WIDTH to UTF-8 file
359 write_header_width(OUTFILE, ARGS.unicode_version)
360 process_width(OUTFILE,
361 UNICODE_DATA_LINES,
362 EAST_ASIAN_WIDTH_LINES,
363 PROP_LIST_LINES)
364 OUTFILE.write("END WIDTH\n")