1 """SCons.Subst
2
3 SCons string substitution.
4
5 """
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29 __revision__ = "src/engine/SCons/Subst.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog"
30
31 import collections
32 import re
33
34 import SCons.Errors
35
36 from SCons.Util import is_String, is_Sequence
37
38
39 _strconv = [SCons.Util.to_String_for_subst,
40 SCons.Util.to_String_for_subst,
41 SCons.Util.to_String_for_signature]
42
43
44
45 AllowableExceptions = (IndexError, NameError)
46
50
58
59
60
62 """A wrapper for a string. If you use this object wrapped
63 around a string, then it will be interpreted as literal.
64 When passed to the command interpreter, all special
65 characters will be escaped."""
68
71
72 - def escape(self, escape_func):
73 return escape_func(self.lstr)
74
77
80
82 if not isinstance(other, Literal):
83 return False
84 return self.lstr == other.lstr
85
87 return not self.__eq__(other)
88
90 """This is a wrapper for what we call a 'Node special attribute.'
91 This is any of the attributes of a Node that we can reference from
92 Environment variable substitution, such as $TARGET.abspath or
93 $SOURCES[1].filebase. We implement the same methods as Literal
94 so we can handle special characters, plus a for_signature method,
95 such that we can return some canonical string during signature
96 calculation to avoid unnecessary rebuilds."""
97
98 - def __init__(self, lstr, for_signature=None):
99 """The for_signature parameter, if supplied, will be the
100 canonical string we return from for_signature(). Else
101 we will simply return lstr."""
102 self.lstr = lstr
103 if for_signature:
104 self.forsig = for_signature
105 else:
106 self.forsig = lstr
107
110
111 - def escape(self, escape_func):
112 return escape_func(self.lstr)
113
116
119
121 """Generic function for putting double quotes around any string that
122 has white space in it."""
123 if ' ' in arg or '\t' in arg:
124 return '"%s"' % arg
125 else:
126 return str(arg)
127
129 """This is a special class used to hold strings generated by
130 scons_subst() and scons_subst_list(). It defines a special method
131 escape(). When passed a function with an escape algorithm for a
132 particular platform, it will return the contained string with the
133 proper escape sequences inserted.
134 """
136 collections.UserString.__init__(self, cmd)
137 self.literal = literal
138
141
142 - def escape(self, escape_func, quote_func=quote_spaces):
143 """Escape the string with the supplied function. The
144 function is expected to take an arbitrary string, then
145 return it with all special characters escaped and ready
146 for passing to the command interpreter.
147
148 After calling this function, the next call to str() will
149 return the escaped string.
150 """
151
152 if self.is_literal():
153 return escape_func(self.data)
154 elif ' ' in self.data or '\t' in self.data:
155 return quote_func(self.data)
156 else:
157 return self.data
158
160 """Escape a list of arguments by running the specified escape_func
161 on every object in the list that has an escape() method."""
162 def escape(obj, escape_func=escape_func):
163 try:
164 e = obj.escape
165 except AttributeError:
166 return obj
167 else:
168 return e(escape_func)
169 return list(map(escape, mylist))
170
172 """A wrapper class that delays turning a list of sources or targets
173 into a NodeList until it's needed. The specified function supplied
174 when the object is initialized is responsible for turning raw nodes
175 into proxies that implement the special attributes like .abspath,
176 .source, etc. This way, we avoid creating those proxies just
177 "in case" someone is going to use $TARGET or the like, and only
178 go through the trouble if we really have to.
179
180 In practice, this might be a wash performance-wise, but it's a little
181 cleaner conceptually...
182 """
183
185 self.list = list
186 self.func = func
190 mylist = self.list
191 if mylist is None:
192 mylist = []
193 elif not is_Sequence(mylist):
194 mylist = [mylist]
195
196
197 self.nodelist = SCons.Util.NodeList(list(map(self.func, mylist)))
198 self._create_nodelist = self._return_nodelist
199 return self.nodelist
200 _create_nodelist = _gen_nodelist
201
202
204 """A class that implements $TARGETS or $SOURCES expansions by in turn
205 wrapping a NLWrapper. This class handles the different methods used
206 to access the list, calling the NLWrapper to create proxies on demand.
207
208 Note that we subclass collections.UserList purely so that the
209 is_Sequence() function will identify an object of this class as
210 a list during variable expansion. We're not really using any
211 collections.UserList methods in practice.
212 """
216 nl = self.nl._create_nodelist()
217 return getattr(nl, attr)
219 nl = self.nl._create_nodelist()
220 return nl[i]
222 nl = self.nl._create_nodelist()
223 i = max(i, 0); j = max(j, 0)
224 return nl[i:j]
226 nl = self.nl._create_nodelist()
227 return str(nl)
229 nl = self.nl._create_nodelist()
230 return repr(nl)
231
233 """A class that implements $TARGET or $SOURCE expansions by in turn
234 wrapping a NLWrapper. This class handles the different methods used
235 to access an individual proxy Node, calling the NLWrapper to create
236 a proxy on demand.
237 """
241 nl = self.nl._create_nodelist()
242 try:
243 nl0 = nl[0]
244 except IndexError:
245
246
247 raise AttributeError("NodeList has no attribute: %s" % attr)
248 return getattr(nl0, attr)
250 nl = self.nl._create_nodelist()
251 if nl:
252 return str(nl[0])
253 return ''
255 nl = self.nl._create_nodelist()
256 if nl:
257 return repr(nl[0])
258 return ''
259
261 - def __call__(self, *args, **kwargs): return ''
263
264 NullNodesList = NullNodeList()
265
267 """Create a dictionary for substitution of special
268 construction variables.
269
270 This translates the following special arguments:
271
272 target - the target (object or array of objects),
273 used to generate the TARGET and TARGETS
274 construction variables
275
276 source - the source (object or array of objects),
277 used to generate the SOURCES and SOURCE
278 construction variables
279 """
280 dict = {}
281
282 if target:
283 def get_tgt_subst_proxy(thing):
284 try:
285 subst_proxy = thing.get_subst_proxy()
286 except AttributeError:
287 subst_proxy = thing
288 return subst_proxy
289 tnl = NLWrapper(target, get_tgt_subst_proxy)
290 dict['TARGETS'] = Targets_or_Sources(tnl)
291 dict['TARGET'] = Target_or_Source(tnl)
292
293
294
295
296
297 dict['CHANGED_TARGETS'] = '$TARGETS'
298 dict['UNCHANGED_TARGETS'] = '$TARGETS'
299 else:
300 dict['TARGETS'] = NullNodesList
301 dict['TARGET'] = NullNodesList
302
303 if source:
304 def get_src_subst_proxy(node):
305 try:
306 rfile = node.rfile
307 except AttributeError:
308 pass
309 else:
310 node = rfile()
311 try:
312 return node.get_subst_proxy()
313 except AttributeError:
314 return node
315 snl = NLWrapper(source, get_src_subst_proxy)
316 dict['SOURCES'] = Targets_or_Sources(snl)
317 dict['SOURCE'] = Target_or_Source(snl)
318
319
320
321
322
323 dict['CHANGED_SOURCES'] = '$SOURCES'
324 dict['UNCHANGED_SOURCES'] = '$SOURCES'
325 else:
326 dict['SOURCES'] = NullNodesList
327 dict['SOURCE'] = NullNodesList
328
329 return dict
330
331
332
333
334
335
336 SUBST_CMD = 0
337 SUBST_RAW = 1
338 SUBST_SIG = 2
339
340 _rm = re.compile(r'\$[()]')
341 _rm_split = re.compile(r'(\$[()])')
342
343
344 _regex_remove = [ _rm, None, _rm_split ]
345
347 return [l for l in list if not l in ('$(', '$)')]
348
350 result = []
351 depth = 0
352 for l in list:
353 if l == '$(':
354 depth += 1
355 elif l == '$)':
356 depth -= 1
357 if depth < 0:
358 break
359 elif depth == 0:
360 result.append(l)
361 if depth != 0:
362 return None
363 return result
364
365
366 _list_remove = [ _rm_list, None, _remove_list ]
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389 _dollar_exps_str = r'\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}'
390 _dollar_exps = re.compile(r'(%s)' % _dollar_exps_str)
391 _separate_args = re.compile(r'(%s|\s+|[^\s\$]+|\$)' % _dollar_exps_str)
392
393
394
395 _space_sep = re.compile(r'[\t ]+(?![^{]*})')
396
397 -def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None):
398 """Expand a string or list containing construction variable
399 substitutions.
400
401 This is the work-horse function for substitutions in file names
402 and the like. The companion scons_subst_list() function (below)
403 handles separating command lines into lists of arguments, so see
404 that function if that's what you're looking for.
405 """
406 if isinstance(strSubst, str) and strSubst.find('$') < 0:
407 return strSubst
408
409 class StringSubber(object):
410 """A class to construct the results of a scons_subst() call.
411
412 This binds a specific construction environment, mode, target and
413 source with two methods (substitute() and expand()) that handle
414 the expansion.
415 """
416 def __init__(self, env, mode, conv, gvars):
417 self.env = env
418 self.mode = mode
419 self.conv = conv
420 self.gvars = gvars
421
422 def expand(self, s, lvars):
423 """Expand a single "token" as necessary, returning an
424 appropriate string containing the expansion.
425
426 This handles expanding different types of things (strings,
427 lists, callables) appropriately. It calls the wrapper
428 substitute() method to re-expand things as necessary, so that
429 the results of expansions of side-by-side strings still get
430 re-evaluated separately, not smushed together.
431 """
432 if is_String(s):
433 try:
434 s0, s1 = s[:2]
435 except (IndexError, ValueError):
436 return s
437 if s0 != '$':
438 return s
439 if s1 == '$':
440 return '$'
441 elif s1 in '()':
442 return s
443 else:
444 key = s[1:]
445 if key[0] == '{' or '.' in key:
446 if key[0] == '{':
447 key = key[1:-1]
448 try:
449 s = eval(key, self.gvars, lvars)
450 except KeyboardInterrupt:
451 raise
452 except Exception as e:
453 if e.__class__ in AllowableExceptions:
454 return ''
455 raise_exception(e, lvars['TARGETS'], s)
456 else:
457 if key in lvars:
458 s = lvars[key]
459 elif key in self.gvars:
460 s = self.gvars[key]
461 elif not NameError in AllowableExceptions:
462 raise_exception(NameError(key), lvars['TARGETS'], s)
463 else:
464 return ''
465
466
467
468
469
470
471
472
473
474
475
476
477
478 lv = lvars.copy()
479 var = key.split('.')[0]
480 lv[var] = ''
481 return self.substitute(s, lv)
482 elif is_Sequence(s):
483 def func(l, conv=self.conv, substitute=self.substitute, lvars=lvars):
484 return conv(substitute(l, lvars))
485 return list(map(func, s))
486 elif callable(s):
487 try:
488 s = s(target=lvars['TARGETS'],
489 source=lvars['SOURCES'],
490 env=self.env,
491 for_signature=(self.mode != SUBST_CMD))
492 except TypeError:
493
494
495
496 if self.mode == SUBST_RAW:
497 return s
498 s = self.conv(s)
499 return self.substitute(s, lvars)
500 elif s is None:
501 return ''
502 else:
503 return s
504
505 def substitute(self, args, lvars):
506 """Substitute expansions in an argument or list of arguments.
507
508 This serves as a wrapper for splitting up a string into
509 separate tokens.
510 """
511 if is_String(args) and not isinstance(args, CmdStringHolder):
512 args = str(args)
513 try:
514 def sub_match(match):
515 return self.conv(self.expand(match.group(1), lvars))
516 result = _dollar_exps.sub(sub_match, args)
517 except TypeError:
518
519
520
521
522
523 args = _separate_args.findall(args)
524 result = []
525 for a in args:
526 result.append(self.conv(self.expand(a, lvars)))
527 if len(result) == 1:
528 result = result[0]
529 else:
530 result = ''.join(map(str, result))
531 return result
532 else:
533 return self.expand(args, lvars)
534
535 if conv is None:
536 conv = _strconv[mode]
537
538
539
540
541
542
543
544
545
546
547 if 'TARGET' not in lvars:
548 d = subst_dict(target, source)
549 if d:
550 lvars = lvars.copy()
551 lvars.update(d)
552
553
554
555
556
557
558
559 gvars['__builtins__'] = __builtins__
560
561 ss = StringSubber(env, mode, conv, gvars)
562 result = ss.substitute(strSubst, lvars)
563
564 try:
565 del gvars['__builtins__']
566 except KeyError:
567 pass
568
569 res = result
570 if is_String(result):
571
572
573 remove = _regex_remove[mode]
574 if remove:
575 if mode == SUBST_SIG:
576 result = _list_remove[mode](remove.split(result))
577 if result is None:
578 raise SCons.Errors.UserError("Unbalanced $(/$) in: " + res)
579 result = ' '.join(result)
580 else:
581 result = remove.sub('', result)
582 if mode != SUBST_RAW:
583
584
585 result = _space_sep.sub(' ', result).strip()
586 elif is_Sequence(result):
587 remove = _list_remove[mode]
588 if remove:
589 result = remove(result)
590 if result is None:
591 raise SCons.Errors.UserError("Unbalanced $(/$) in: " + str(res))
592
593 return result
594
596 """Substitute construction variables in a string (or list or other
597 object) and separate the arguments into a command list.
598
599 The companion scons_subst() function (above) handles basic
600 substitutions within strings, so see that function instead
601 if that's what you're looking for.
602 """
603 class ListSubber(collections.UserList):
604 """A class to construct the results of a scons_subst_list() call.
605
606 Like StringSubber, this class binds a specific construction
607 environment, mode, target and source with two methods
608 (substitute() and expand()) that handle the expansion.
609
610 In addition, however, this class is used to track the state of
611 the result(s) we're gathering so we can do the appropriate thing
612 whenever we have to append another word to the result--start a new
613 line, start a new word, append to the current word, etc. We do
614 this by setting the "append" attribute to the right method so
615 that our wrapper methods only need ever call ListSubber.append(),
616 and the rest of the object takes care of doing the right thing
617 internally.
618 """
619 def __init__(self, env, mode, conv, gvars):
620 collections.UserList.__init__(self, [])
621 self.env = env
622 self.mode = mode
623 self.conv = conv
624 self.gvars = gvars
625
626 if self.mode == SUBST_RAW:
627 self.add_strip = lambda x: self.append(x)
628 else:
629 self.add_strip = lambda x: None
630 self.in_strip = None
631 self.next_line()
632
633 def expand(self, s, lvars, within_list):
634 """Expand a single "token" as necessary, appending the
635 expansion to the current result.
636
637 This handles expanding different types of things (strings,
638 lists, callables) appropriately. It calls the wrapper
639 substitute() method to re-expand things as necessary, so that
640 the results of expansions of side-by-side strings still get
641 re-evaluated separately, not smushed together.
642 """
643
644 if is_String(s):
645 try:
646 s0, s1 = s[:2]
647 except (IndexError, ValueError):
648 self.append(s)
649 return
650 if s0 != '$':
651 self.append(s)
652 return
653 if s1 == '$':
654 self.append('$')
655 elif s1 == '(':
656 self.open_strip('$(')
657 elif s1 == ')':
658 self.close_strip('$)')
659 else:
660 key = s[1:]
661 if key[0] == '{' or key.find('.') >= 0:
662 if key[0] == '{':
663 key = key[1:-1]
664 try:
665 s = eval(key, self.gvars, lvars)
666 except KeyboardInterrupt:
667 raise
668 except Exception as e:
669 if e.__class__ in AllowableExceptions:
670 return
671 raise_exception(e, lvars['TARGETS'], s)
672 else:
673 if key in lvars:
674 s = lvars[key]
675 elif key in self.gvars:
676 s = self.gvars[key]
677 elif not NameError in AllowableExceptions:
678 raise_exception(NameError(), lvars['TARGETS'], s)
679 else:
680 return
681
682
683
684
685
686
687 lv = lvars.copy()
688 var = key.split('.')[0]
689 lv[var] = ''
690 self.substitute(s, lv, 0)
691 self.this_word()
692 elif is_Sequence(s):
693 for a in s:
694 self.substitute(a, lvars, 1)
695 self.next_word()
696 elif callable(s):
697 try:
698 s = s(target=lvars['TARGETS'],
699 source=lvars['SOURCES'],
700 env=self.env,
701 for_signature=(self.mode != SUBST_CMD))
702 except TypeError:
703
704
705
706 if self.mode == SUBST_RAW:
707 self.append(s)
708 return
709 s = self.conv(s)
710 self.substitute(s, lvars, within_list)
711 elif s is None:
712 self.this_word()
713 else:
714 self.append(s)
715
716 def substitute(self, args, lvars, within_list):
717 """Substitute expansions in an argument or list of arguments.
718
719 This serves as a wrapper for splitting up a string into
720 separate tokens.
721 """
722
723 if is_String(args) and not isinstance(args, CmdStringHolder):
724 args = str(args)
725 args = _separate_args.findall(args)
726 for a in args:
727 if a[0] in ' \t\n\r\f\v':
728 if '\n' in a:
729 self.next_line()
730 elif within_list:
731 self.append(a)
732 else:
733 self.next_word()
734 else:
735 self.expand(a, lvars, within_list)
736 else:
737 self.expand(args, lvars, within_list)
738
739 def next_line(self):
740 """Arrange for the next word to start a new line. This
741 is like starting a new word, except that we have to append
742 another line to the result."""
743 collections.UserList.append(self, [])
744 self.next_word()
745
746 def this_word(self):
747 """Arrange for the next word to append to the end of the
748 current last word in the result."""
749 self.append = self.add_to_current_word
750
751 def next_word(self):
752 """Arrange for the next word to start a new word."""
753 self.append = self.add_new_word
754
755 def add_to_current_word(self, x):
756 """Append the string x to the end of the current last word
757 in the result. If that is not possible, then just add
758 it as a new word. Make sure the entire concatenated string
759 inherits the object attributes of x (in particular, the
760 escape function) by wrapping it as CmdStringHolder."""
761
762 if not self.in_strip or self.mode != SUBST_SIG:
763 try:
764 current_word = self[-1][-1]
765 except IndexError:
766 self.add_new_word(x)
767 else:
768
769
770
771
772
773
774
775
776
777
778 try:
779 last_char = str(current_word)[-1]
780 except IndexError:
781 last_char = '\0'
782 if last_char in '<>|':
783 self.add_new_word(x)
784 else:
785 y = current_word + x
786
787
788
789
790
791
792
793
794
795
796
797 y = self.conv(y)
798 if is_String(y):
799
800 y = CmdStringHolder(y, None)
801 self[-1][-1] = y
802
803 def add_new_word(self, x):
804 if not self.in_strip or self.mode != SUBST_SIG:
805 literal = self.literal(x)
806 x = self.conv(x)
807 if is_String(x):
808 x = CmdStringHolder(x, literal)
809 self[-1].append(x)
810 self.append = self.add_to_current_word
811
812 def literal(self, x):
813 try:
814 l = x.is_literal
815 except AttributeError:
816 return None
817 else:
818 return l()
819
820 def open_strip(self, x):
821 """Handle the "open strip" $( token."""
822 self.add_strip(x)
823 self.in_strip = 1
824
825 def close_strip(self, x):
826 """Handle the "close strip" $) token."""
827 self.add_strip(x)
828 self.in_strip = None
829
830 if conv is None:
831 conv = _strconv[mode]
832
833
834
835
836
837
838
839
840
841
842 if 'TARGET' not in lvars:
843 d = subst_dict(target, source)
844 if d:
845 lvars = lvars.copy()
846 lvars.update(d)
847
848
849
850
851
852
853
854 gvars['__builtins__'] = __builtins__
855
856 ls = ListSubber(env, mode, conv, gvars)
857 ls.substitute(strSubst, lvars, 0)
858
859 try:
860 del gvars['__builtins__']
861 except KeyError:
862 pass
863
864 return ls.data
865
867 """Perform single (non-recursive) substitution of a single
868 construction variable keyword.
869
870 This is used when setting a variable when copying or overriding values
871 in an Environment. We want to capture (expand) the old value before
872 we override it, so people can do things like:
873
874 env2 = env.Clone(CCFLAGS = '$CCFLAGS -g')
875
876 We do this with some straightforward, brute-force code here...
877 """
878 if isinstance(strSubst, str) and strSubst.find('$') < 0:
879 return strSubst
880
881 matchlist = ['$' + key, '${' + key + '}']
882 val = env.get(key, '')
883 def sub_match(match, val=val, matchlist=matchlist):
884 a = match.group(1)
885 if a in matchlist:
886 a = val
887 if is_Sequence(a):
888 return ' '.join(map(str, a))
889 else:
890 return str(a)
891
892 if is_Sequence(strSubst):
893 result = []
894 for arg in strSubst:
895 if is_String(arg):
896 if arg in matchlist:
897 arg = val
898 if is_Sequence(arg):
899 result.extend(arg)
900 else:
901 result.append(arg)
902 else:
903 result.append(_dollar_exps.sub(sub_match, arg))
904 else:
905 result.append(arg)
906 return result
907 elif is_String(strSubst):
908 return _dollar_exps.sub(sub_match, strSubst)
909 else:
910 return strSubst
911
912
913
914
915
916
917