111
|
1 #! /usr/bin/python2
|
|
2 import os.path
|
|
3 import sys
|
|
4 import shlex
|
|
5 import re
|
|
6 import tempfile
|
|
7 import copy
|
|
8
|
|
9 from headerutils import *
|
|
10
|
|
11 requires = { }
|
|
12 provides = { }
|
|
13
|
|
14 no_remove = [ "system.h", "coretypes.h", "config.h" , "bconfig.h", "backend.h" ]
|
|
15
|
|
16 # These targets are the ones which provide "coverage". Typically, if any
|
|
17 # target is going to fail compilation, it's one of these. This was determined
|
|
18 # during the initial runs of reduce-headers... On a full set of target builds,
|
|
19 # every failure which occured was triggered by one of these.
|
|
20 # This list is used during target-list construction simply to put any of these
|
|
21 # *first* in the candidate list, increasing the probability that a failure is
|
|
22 # found quickly.
|
|
23 target_priority = [
|
|
24 "aarch64-linux-gnu",
|
|
25 "arm-netbsdelf",
|
|
26 "c6x-elf",
|
|
27 "epiphany-elf",
|
|
28 "hppa2.0-hpux10.1",
|
|
29 "i686-mingw32crt",
|
|
30 "i686-pc-msdosdjgpp",
|
|
31 "mipsel-elf",
|
|
32 "powerpc-eabisimaltivec",
|
|
33 "rs6000-ibm-aix5.1.0",
|
|
34 "sh-superh-elf",
|
|
35 "sparc64-elf",
|
|
36 "spu-elf"
|
|
37 ]
|
|
38
|
|
39
|
|
40 target_dir = ""
|
|
41 build_dir = ""
|
|
42 ignore_list = list()
|
|
43 target_builds = list()
|
|
44
|
|
45 target_dict = { }
|
|
46 header_dict = { }
|
|
47 search_path = [ ".", "../include", "../libcpp/include" ]
|
|
48
|
|
49 remove_count = { }
|
|
50
|
|
51
|
|
52 # Given a header name, normalize it. ie. cp/cp-tree.h could be in gcc, while
|
|
53 # the same header could be referenced from within the cp subdirectory as
|
|
54 # just cp-tree.h
|
|
55 # for now, just assume basenames are unique
|
|
56
|
|
57 def normalize_header (header):
|
|
58 return os.path.basename (header)
|
|
59
|
|
60
|
|
61 # Adds a header file and its sub-includes to the global dictionary if they
|
|
62 # aren't already there. Specify s_path since different build directories may
|
|
63 # append themselves on demand to the global list.
|
|
64 # return entry for the specified header, knowing all sub entries are completed
|
|
65
|
|
66 def get_header_info (header, s_path):
|
|
67 global header_dict
|
|
68 global empty_iinfo
|
|
69 process_list = list ()
|
|
70 location = ""
|
|
71 bname = ""
|
|
72 bname_iinfo = empty_iinfo
|
|
73 for path in s_path:
|
|
74 if os.path.exists (path + "/" + header):
|
|
75 location = path + "/" + header
|
|
76 break
|
|
77
|
|
78 if location:
|
|
79 bname = normalize_header (location)
|
|
80 if header_dict.get (bname):
|
|
81 bname_iinfo = header_dict[bname]
|
|
82 loc2 = ii_path (bname_iinfo)+ "/" + bname
|
|
83 if loc2[:2] == "./":
|
|
84 loc2 = loc2[2:]
|
|
85 if location[:2] == "./":
|
|
86 location = location[2:]
|
|
87 if loc2 != location:
|
|
88 # Don't use the cache if it isnt the right one.
|
|
89 bname_iinfo = process_ii_macro (location)
|
|
90 return bname_iinfo
|
|
91
|
|
92 bname_iinfo = process_ii_macro (location)
|
|
93 header_dict[bname] = bname_iinfo
|
|
94 # now decend into the include tree
|
|
95 for i in ii_include_list (bname_iinfo):
|
|
96 get_header_info (i, s_path)
|
|
97 else:
|
|
98 # if the file isnt in the source directories, look in the build and target
|
|
99 # directories. If it is here, then aggregate all the versions.
|
|
100 location = build_dir + "/gcc/" + header
|
|
101 build_inc = target_inc = False
|
|
102 if os.path.exists (location):
|
|
103 build_inc = True
|
|
104 for x in target_dict:
|
|
105 location = target_dict[x] + "/gcc/" + header
|
|
106 if os.path.exists (location):
|
|
107 target_inc = True
|
|
108 break
|
|
109
|
|
110 if (build_inc or target_inc):
|
|
111 bname = normalize_header(header)
|
|
112 defines = set()
|
|
113 consumes = set()
|
|
114 incl = set()
|
|
115 if build_inc:
|
|
116 iinfo = process_ii_macro (build_dir + "/gcc/" + header)
|
|
117 defines = set (ii_macro_define (iinfo))
|
|
118 consumes = set (ii_macro_consume (iinfo))
|
|
119 incl = set (ii_include_list (iinfo))
|
|
120
|
|
121 if (target_inc):
|
|
122 for x in target_dict:
|
|
123 location = target_dict[x] + "/gcc/" + header
|
|
124 if os.path.exists (location):
|
|
125 iinfo = process_ii_macro (location)
|
|
126 defines.update (ii_macro_define (iinfo))
|
|
127 consumes.update (ii_macro_consume (iinfo))
|
|
128 incl.update (ii_include_list (iinfo))
|
|
129
|
|
130 bname_iinfo = (header, "build", list(incl), list(), list(consumes), list(defines), list(), list())
|
|
131
|
|
132 header_dict[bname] = bname_iinfo
|
|
133 for i in incl:
|
|
134 get_header_info (i, s_path)
|
|
135
|
|
136 return bname_iinfo
|
|
137
|
|
138
|
|
139 # return a list of all headers brought in by this header
|
|
140 def all_headers (fname):
|
|
141 global header_dict
|
|
142 headers_stack = list()
|
|
143 headers_list = list()
|
|
144 if header_dict.get (fname) == None:
|
|
145 return list ()
|
|
146 for y in ii_include_list (header_dict[fname]):
|
|
147 headers_stack.append (y)
|
|
148
|
|
149 while headers_stack:
|
|
150 h = headers_stack.pop ()
|
|
151 hn = normalize_header (h)
|
|
152 if hn not in headers_list:
|
|
153 headers_list.append (hn)
|
|
154 if header_dict.get(hn):
|
|
155 for y in ii_include_list (header_dict[hn]):
|
|
156 if normalize_header (y) not in headers_list:
|
|
157 headers_stack.append (y)
|
|
158
|
|
159 return headers_list
|
|
160
|
|
161
|
|
162
|
|
163
|
|
164 # Search bld_dir for all target tuples, confirm that they have a build path with
|
|
165 # bld_dir/target-tuple/gcc, and build a dictionary of build paths indexed by
|
|
166 # target tuple..
|
|
167
|
|
168 def build_target_dict (bld_dir, just_these):
|
|
169 global target_dict
|
|
170 target_doct = { }
|
|
171 error = False
|
|
172 if os.path.exists (bld_dir):
|
|
173 if just_these:
|
|
174 ls = just_these
|
|
175 else:
|
|
176 ls = os.listdir(bld_dir)
|
|
177 for t in ls:
|
|
178 if t.find("-") != -1:
|
|
179 target = t.strip()
|
|
180 tpath = bld_dir + "/" + target
|
|
181 if not os.path.exists (tpath + "/gcc"):
|
|
182 print "Error: gcc build directory for target " + t + " Does not exist: " + tpath + "/gcc"
|
|
183 error = True
|
|
184 else:
|
|
185 target_dict[target] = tpath
|
|
186
|
|
187 if error:
|
|
188 target_dict = { }
|
|
189
|
|
190 def get_obj_name (src_file):
|
|
191 if src_file[-2:] == ".c":
|
|
192 return src_file.replace (".c", ".o")
|
|
193 elif src_file[-3:] == ".cc":
|
|
194 return src_file.replace (".cc", ".o")
|
|
195 return ""
|
|
196
|
|
197 def target_obj_exists (target, obj_name):
|
|
198 global target_dict
|
|
199 # look in a subdir if src has a subdir, then check gcc base directory.
|
|
200 if target_dict.get(target):
|
|
201 obj = target_dict[target] + "/gcc/" + obj_name
|
|
202 if not os.path.exists (obj):
|
|
203 obj = target_dict[target] + "/gcc/" + os.path.basename(obj_name)
|
|
204 if os.path.exists (obj):
|
|
205 return True
|
|
206 return False
|
|
207
|
|
208 # Given a src file, return a list of targets which may build this file.
|
|
209 def find_targets (src_file):
|
|
210 global target_dict
|
|
211 targ_list = list()
|
|
212 obj_name = get_obj_name (src_file)
|
|
213 if not obj_name:
|
|
214 print "Error: " + src_file + " - Cannot determine object name."
|
|
215 return list()
|
|
216
|
|
217 # Put the high priority targets which tend to trigger failures first
|
|
218 for target in target_priority:
|
|
219 if target_obj_exists (target, obj_name):
|
|
220 targ_list.append ((target, target_dict[target]))
|
|
221
|
|
222 for target in target_dict:
|
|
223 if target not in target_priority and target_obj_exists (target, obj_name):
|
|
224 targ_list.append ((target, target_dict[target]))
|
|
225
|
|
226 return targ_list
|
|
227
|
|
228
|
|
229 def try_to_remove (src_file, h_list, verbose):
|
|
230 global target_dict
|
|
231 global header_dict
|
|
232 global build_dir
|
|
233
|
|
234 # build from scratch each time
|
|
235 header_dict = { }
|
|
236 summary = ""
|
|
237 rmcount = 0
|
|
238
|
|
239 because = { }
|
|
240 src_info = process_ii_macro_src (src_file)
|
|
241 src_data = ii_src (src_info)
|
|
242 if src_data:
|
|
243 inclist = ii_include_list_non_cond (src_info)
|
|
244 # work is done if there are no includes to check
|
|
245 if not inclist:
|
|
246 return src_file + ": No include files to attempt to remove"
|
|
247
|
|
248 # work on the include list in reverse.
|
|
249 inclist.reverse()
|
|
250
|
|
251 # Get the target list
|
|
252 targ_list = list()
|
|
253 targ_list = find_targets (src_file)
|
|
254
|
|
255 spath = search_path
|
|
256 if os.path.dirname (src_file):
|
|
257 spath.append (os.path.dirname (src_file))
|
|
258
|
|
259 hostbuild = True
|
|
260 if src_file.find("config/") != -1:
|
|
261 # config files dont usually build on the host
|
|
262 hostbuild = False
|
|
263 obn = get_obj_name (os.path.basename (src_file))
|
|
264 if obn and os.path.exists (build_dir + "/gcc/" + obn):
|
|
265 hostbuild = True
|
|
266 if not target_dict:
|
|
267 summary = src_file + ": Target builds are required for config files. None found."
|
|
268 print summary
|
|
269 return summary
|
|
270 if not targ_list:
|
|
271 summary =src_file + ": Cannot find any targets which build this file."
|
|
272 print summary
|
|
273 return summary
|
|
274
|
|
275 if hostbuild:
|
|
276 # confirm it actually builds before we do anything
|
|
277 print "Confirming source file builds"
|
|
278 res = get_make_output (build_dir + "/gcc", "all")
|
|
279 if res[0] != 0:
|
|
280 message = "Error: " + src_file + " does not build currently."
|
|
281 summary = src_file + " does not build on host."
|
|
282 print message
|
|
283 print res[1]
|
|
284 if verbose:
|
|
285 verbose.write (message + "\n")
|
|
286 verbose.write (res[1]+ "\n")
|
|
287 return summary
|
|
288
|
|
289 src_requires = set (ii_macro_consume (src_info))
|
|
290 for macro in src_requires:
|
|
291 because[macro] = src_file
|
|
292 header_seen = list ()
|
|
293
|
|
294 os.rename (src_file, src_file + ".bak")
|
|
295 src_orig = copy.deepcopy (src_data)
|
|
296 src_tmp = copy.deepcopy (src_data)
|
|
297
|
|
298 try:
|
|
299 # process the includes from bottom to top. This is because we know that
|
|
300 # later includes have are known to be needed, so any dependency from this
|
|
301 # header is a true dependency
|
|
302 for inc_file in inclist:
|
|
303 inc_file_norm = normalize_header (inc_file)
|
|
304
|
|
305 if inc_file in no_remove:
|
|
306 continue
|
|
307 if len (h_list) != 0 and inc_file_norm not in h_list:
|
|
308 continue
|
|
309 if inc_file_norm[0:3] == "gt-":
|
|
310 continue
|
|
311 if inc_file_norm[0:6] == "gtype-":
|
|
312 continue
|
|
313 if inc_file_norm.replace(".h",".c") == os.path.basename(src_file):
|
|
314 continue
|
|
315
|
|
316 lookfor = ii_src_line(src_info)[inc_file]
|
|
317 src_tmp.remove (lookfor)
|
|
318 message = "Trying " + src_file + " without " + inc_file
|
|
319 print message
|
|
320 if verbose:
|
|
321 verbose.write (message + "\n")
|
|
322 out = open(src_file, "w")
|
|
323 for line in src_tmp:
|
|
324 out.write (line)
|
|
325 out.close()
|
|
326
|
|
327 keep = False
|
|
328 if hostbuild:
|
|
329 res = get_make_output (build_dir + "/gcc", "all")
|
|
330 else:
|
|
331 res = (0, "")
|
|
332
|
|
333 rc = res[0]
|
|
334 message = "Passed Host build"
|
|
335 if (rc != 0):
|
|
336 # host build failed
|
|
337 message = "Compilation failed:\n";
|
|
338 keep = True
|
|
339 else:
|
|
340 if targ_list:
|
|
341 objfile = get_obj_name (src_file)
|
|
342 t1 = targ_list[0]
|
|
343 if objfile and os.path.exists(t1[1] +"/gcc/"+objfile):
|
|
344 res = get_make_output_parallel (targ_list, objfile, 0)
|
|
345 else:
|
|
346 res = get_make_output_parallel (targ_list, "all-gcc", 0)
|
|
347 rc = res[0]
|
|
348 if rc != 0:
|
|
349 message = "Compilation failed on TARGET : " + res[2]
|
|
350 keep = True
|
|
351 else:
|
|
352 message = "Passed host and target builds"
|
|
353
|
|
354 if keep:
|
|
355 print message + "\n"
|
|
356
|
|
357 if (rc != 0):
|
|
358 if verbose:
|
|
359 verbose.write (message + "\n");
|
|
360 verbose.write (res[1])
|
|
361 verbose.write ("\n");
|
|
362 if os.path.exists (inc_file):
|
|
363 ilog = open(inc_file+".log","a")
|
|
364 ilog.write (message + " for " + src_file + ":\n\n");
|
|
365 ilog.write ("============================================\n");
|
|
366 ilog.write (res[1])
|
|
367 ilog.write ("\n");
|
|
368 ilog.close()
|
|
369 if os.path.exists (src_file):
|
|
370 ilog = open(src_file+".log","a")
|
|
371 ilog.write (message + " for " +inc_file + ":\n\n");
|
|
372 ilog.write ("============================================\n");
|
|
373 ilog.write (res[1])
|
|
374 ilog.write ("\n");
|
|
375 ilog.close()
|
|
376
|
|
377 # Given a sequence where :
|
|
378 # #include "tm.h"
|
|
379 # #include "target.h" // includes tm.h
|
|
380
|
|
381 # target.h was required, and when attempting to remove tm.h we'd see that
|
|
382 # all the macro defintions are "required" since they all look like:
|
|
383 # #ifndef HAVE_blah
|
|
384 # #define HAVE_blah
|
|
385 # endif
|
|
386
|
|
387 # when target.h was found to be required, tm.h will be tagged as included.
|
|
388 # so when we get this far, we know we dont have to check the macros for
|
|
389 # tm.h since we know it is already been included.
|
|
390
|
|
391 if inc_file_norm not in header_seen:
|
|
392 iinfo = get_header_info (inc_file, spath)
|
|
393 newlist = all_headers (inc_file_norm)
|
|
394 if ii_path(iinfo) == "build" and not target_dict:
|
|
395 keep = True
|
|
396 text = message + " : Will not remove a build file without some targets."
|
|
397 print text
|
|
398 ilog = open(src_file+".log","a")
|
|
399 ilog.write (text +"\n")
|
|
400 ilog.write ("============================================\n");
|
|
401 ilog.close()
|
|
402 ilog = open("reduce-headers-kept.log","a")
|
|
403 ilog.write (src_file + " " + text +"\n")
|
|
404 ilog.close()
|
|
405 else:
|
|
406 newlist = list()
|
|
407 if not keep and inc_file_norm not in header_seen:
|
|
408 # now look for any macro requirements.
|
|
409 for h in newlist:
|
|
410 if not h in header_seen:
|
|
411 if header_dict.get(h):
|
|
412 defined = ii_macro_define (header_dict[h])
|
|
413 for dep in defined:
|
|
414 if dep in src_requires and dep not in ignore_list:
|
|
415 keep = True;
|
|
416 text = message + ", but must keep " + inc_file + " because it provides " + dep
|
|
417 if because.get(dep) != None:
|
|
418 text = text + " Possibly required by " + because[dep]
|
|
419 print text
|
|
420 ilog = open(inc_file+".log","a")
|
|
421 ilog.write (because[dep]+": Requires [dep] in "+src_file+"\n")
|
|
422 ilog.write ("============================================\n");
|
|
423 ilog.close()
|
|
424 ilog = open(src_file+".log","a")
|
|
425 ilog.write (text +"\n")
|
|
426 ilog.write ("============================================\n");
|
|
427 ilog.close()
|
|
428 ilog = open("reduce-headers-kept.log","a")
|
|
429 ilog.write (src_file + " " + text +"\n")
|
|
430 ilog.close()
|
|
431 if verbose:
|
|
432 verbose.write (text + "\n")
|
|
433
|
|
434 if keep:
|
|
435 # add all headers 'consumes' to src_requires list, and mark as seen
|
|
436 for h in newlist:
|
|
437 if not h in header_seen:
|
|
438 header_seen.append (h)
|
|
439 if header_dict.get(h):
|
|
440 consume = ii_macro_consume (header_dict[h])
|
|
441 for dep in consume:
|
|
442 if dep not in src_requires:
|
|
443 src_requires.add (dep)
|
|
444 if because.get(dep) == None:
|
|
445 because[dep] = inc_file
|
|
446
|
|
447 src_tmp = copy.deepcopy (src_data)
|
|
448 else:
|
|
449 print message + " --> removing " + inc_file + "\n"
|
|
450 rmcount += 1
|
|
451 if verbose:
|
|
452 verbose.write (message + " --> removing " + inc_file + "\n")
|
|
453 if remove_count.get(inc_file) == None:
|
|
454 remove_count[inc_file] = 1
|
|
455 else:
|
|
456 remove_count[inc_file] += 1
|
|
457 src_data = copy.deepcopy (src_tmp)
|
|
458 except:
|
|
459 print "Interuption: restoring original file"
|
|
460 out = open(src_file, "w")
|
|
461 for line in src_orig:
|
|
462 out.write (line)
|
|
463 out.close()
|
|
464 raise
|
|
465
|
|
466 # copy current version, since it is the "right" one now.
|
|
467 out = open(src_file, "w")
|
|
468 for line in src_data:
|
|
469 out.write (line)
|
|
470 out.close()
|
|
471
|
|
472 # Try a final host bootstrap build to make sure everything is kosher.
|
|
473 if hostbuild:
|
|
474 res = get_make_output (build_dir, "all")
|
|
475 rc = res[0]
|
|
476 if (rc != 0):
|
|
477 # host build failed! return to original version
|
|
478 print "Error: " + src_file + " Failed to bootstrap at end!!! restoring."
|
|
479 print " Bad version at " + src_file + ".bad"
|
|
480 os.rename (src_file, src_file + ".bad")
|
|
481 out = open(src_file, "w")
|
|
482 for line in src_orig:
|
|
483 out.write (line)
|
|
484 out.close()
|
|
485 return src_file + ": failed to build after reduction. Restored original"
|
|
486
|
|
487 if src_data == src_orig:
|
|
488 summary = src_file + ": No change."
|
|
489 else:
|
|
490 summary = src_file + ": Reduction performed, "+str(rmcount)+" includes removed."
|
|
491 print summary
|
|
492 return summary
|
|
493
|
|
494 only_h = list ()
|
|
495 ignore_cond = False
|
|
496
|
|
497 usage = False
|
|
498 src = list()
|
|
499 only_targs = list ()
|
|
500 for x in sys.argv[1:]:
|
|
501 if x[0:2] == "-b":
|
|
502 build_dir = x[2:]
|
|
503 elif x[0:2] == "-f":
|
|
504 fn = normalize_header (x[2:])
|
|
505 if fn not in only_h:
|
|
506 only_h.append (fn)
|
|
507 elif x[0:2] == "-h":
|
|
508 usage = True
|
|
509 elif x[0:2] == "-d":
|
|
510 ignore_cond = True
|
|
511 elif x[0:2] == "-D":
|
|
512 ignore_list.append(x[2:])
|
|
513 elif x[0:2] == "-T":
|
|
514 only_targs.append(x[2:])
|
|
515 elif x[0:2] == "-t":
|
|
516 target_dir = x[2:]
|
|
517 elif x[0] == "-":
|
|
518 print "Error: Unrecognized option " + x
|
|
519 usgae = True
|
|
520 else:
|
|
521 if not os.path.exists (x):
|
|
522 print "Error: specified file " + x + " does not exist."
|
|
523 usage = True
|
|
524 else:
|
|
525 src.append (x)
|
|
526
|
|
527 if target_dir:
|
|
528 build_target_dict (target_dir, only_targs)
|
|
529
|
|
530 if build_dir == "" and target_dir == "":
|
|
531 print "Error: Must specify a build directory, and/or a target directory."
|
|
532 usage = True
|
|
533
|
|
534 if build_dir and not os.path.exists (build_dir):
|
|
535 print "Error: specified build directory does not exist : " + build_dir
|
|
536 usage = True
|
|
537
|
|
538 if target_dir and not os.path.exists (target_dir):
|
|
539 print "Error: specified target directory does not exist : " + target_dir
|
|
540 usage = True
|
|
541
|
|
542 if usage:
|
|
543 print "Attempts to remove extraneous include files from source files."
|
|
544 print " "
|
|
545 print "Should be run from the main gcc source directory, and works on a target"
|
|
546 print "directory, as we attempt to make the 'all' target."
|
|
547 print " "
|
|
548 print "By default, gcc-reorder-includes is run on each file before attempting"
|
|
549 print "to remove includes. this removes duplicates and puts some headers in a"
|
|
550 print "canonical ordering"
|
|
551 print " "
|
|
552 print "The build directory should be ready to compile via make. Time is saved"
|
|
553 print "if the build is already complete, so that only changes need to be built."
|
|
554 print " "
|
|
555 print "Usage: [options] file1.c [file2.c] ... [filen.c]"
|
|
556 print " -bdir : the root build directory to attempt buiding .o files."
|
|
557 print " -tdir : the target build directory"
|
|
558 print " -d : Ignore conditional macro dependencies."
|
|
559 print " "
|
|
560 print " -Dmacro : Ignore a specific macro for dependencies"
|
|
561 print " -Ttarget : Only consider target in target directory."
|
|
562 print " -fheader : Specifies a specific .h file to be considered."
|
|
563 print " "
|
|
564 print " -D, -T, and -f can be specified mulitple times and are aggregated."
|
|
565 print " "
|
|
566 print " The original file will be in filen.bak"
|
|
567 print " "
|
|
568 sys.exit (0)
|
|
569
|
|
570 if only_h:
|
|
571 print "Attempting to remove only these files:"
|
|
572 for x in only_h:
|
|
573 print x
|
|
574 print " "
|
|
575
|
|
576 logfile = open("reduce-headers.log","w")
|
|
577
|
|
578 for x in src:
|
|
579 msg = try_to_remove (x, only_h, logfile)
|
|
580 ilog = open("reduce-headers.sum","a")
|
|
581 ilog.write (msg + "\n")
|
|
582 ilog.close()
|
|
583
|
|
584 ilog = open("reduce-headers.sum","a")
|
|
585 ilog.write ("===============================================================\n")
|
|
586 for x in remove_count:
|
|
587 msg = x + ": Removed " + str(remove_count[x]) + " times."
|
|
588 print msg
|
|
589 logfile.write (msg + "\n")
|
|
590 ilog.write (msg + "\n")
|
|
591
|
|
592
|
|
593
|
|
594
|
|
595
|