Please note:The SCons wiki is in read-only mode due to ongoing spam/DoS issues. Also, new account creation is currently disabled. We are looking into alternative wiki hosts.

The generic substitution builder is an improvement upon the SubstInFile2 builder. It provides a more generic way of performing substitutions and can be extended with new methods. This can be used to take *.in files and produce results from them, such as config.h from a config.h.in, an so on. It provides two common substitution methods, SubstFile and SubstHeader. These simply call the SubstGeneric builder with the desired values.

This builder has been updated to allow the SubstHeader substitutions to be quoted if desired.

The code

   1 # File:         subst.py
   2 # Author:       Brian A. Vanderburg II
   3 # Purpose:      A generic SCons file substitution mechanism
   4 # Copyright:    This file is placed in the public domain.
   5 ##############################################################################
   6 
   7 
   8 # Requirements
   9 ##############################################################################
  10 import re
  11 
  12 from SCons.Script import *
  13 import SCons.Errors
  14 
  15 
  16 # Helper/core functions
  17 ##############################################################################
  18 
  19 # Do the substitution
  20 def _subst_file(target, source, env, pattern, replace):
  21     # Read file
  22     f = open(source, "rU")
  23     try:
  24         contents = f.read()
  25     finally:
  26         f.close()
  27 
  28     # Substitute, make sure result is a string
  29     def subfn(mo):
  30         value = replace(env, mo)
  31         if not SCons.Util.is_String(value):
  32             raise SCons.Errors.UserError("Substitution must be a string.")
  33         return value
  34 
  35     contents = re.sub(pattern, subfn, contents)
  36 
  37     # Write file
  38     f = open(target, "wt")
  39     try:
  40         f.write(contents)
  41     finally:
  42         f.close()
  43 
  44 # Determine which keys are used
  45 def _subst_keys(source, pattern):
  46     # Read file
  47     f = open(source, "rU")
  48     try:
  49         contents = f.read()
  50     finally:
  51         f.close()
  52 
  53     # Determine keys
  54     keys = []
  55     def subfn(mo):
  56         key = mo.group("key")
  57         if key:
  58             keys.append(key)
  59         return ""
  60 
  61     re.sub(pattern, subfn, contents)
  62 
  63     return keys
  64 
  65 # Get the value of a key as a string, or None if it is not in the environment
  66 def _subst_value(env, key):
  67     # Why does "if key in env" result in "KeyError: 0:"?
  68     try:
  69         env[key]
  70     except KeyError:
  71         return None
  72 
  73     # Do a raw substitution so it will not replace tabs/whitespaces with a
  74     # single space.  This will return a string even if the result really
  75     # isn't, such as env['HAVE_STRDUP'] = 0
  76     return env.subst("${%s}" % key, 1)
  77 
  78 
  79 # Builder related functions
  80 ##############################################################################
  81 
  82 # Builder action
  83 def _subst_action(target, source, env):
  84     # Substitute in the files
  85     pattern = env["SUBST_PATTERN"]
  86     replace = env["SUBST_REPLACE"]
  87 
  88     for (t, s) in zip(target, source):
  89         _subst_file(str(t), str(s), env, pattern, replace)
  90 
  91     return 0
  92 
  93 # Builder message
  94 def _subst_message(target, source, env):
  95     items = ["Substituting vars from %s to %s" % (str(s), str(t))
  96              for (t, s) in zip(target, source)]
  97 
  98     return "\n".join(items)
  99 
 100 # Builder dependency emitter
 101 def _subst_emitter(target, source, env):
 102     pattern = env["SUBST_PATTERN"]
 103     for (t, s) in zip(target, source):
 104         # When building, if a variant directory is used and source files
 105         # are being duplicated, the source file will not be duplicated yet
 106         # when this is called, so the real source must be used instead of
 107         # the duplicated source
 108         path = s.srcnode().abspath
 109 
 110         # Get keys used
 111         keys = _subst_keys(path, pattern)
 112 
 113         d = dict()
 114         for key in keys:
 115             value = _subst_value(env, key)
 116             if not value is None:
 117                 d[key] = value
 118 
 119         # Only the current target depends on this dictionary
 120         Depends(t, SCons.Node.Python.Value(d))
 121 
 122     return target, source
 123 
 124 
 125 # Replace @key@ with the value of that key, and @@ with a single @
 126 ##############################################################################
 127 
 128 _SubstFile_pattern = "@(?P<key>\w*?)@"
 129 def _SubstFile_replace(env, mo):
 130     key = mo.group("key")
 131     if not key:
 132         return "@"
 133 
 134     value = _subst_value(env, key)
 135     if value is None:
 136         raise SCons.Errors.UserError("Error: key %s does not exist" % key)
 137     return value
 138     
 139 def SubstFile(env, target, source):
 140     return env.SubstGeneric(target,
 141                             source,
 142                             SUBST_PATTERN=_SubstFile_pattern,
 143                             SUBST_REPLACE=_SubstFile_replace)
 144 
 145 
 146 # A substitutor similar to config.h header substitution
 147 # Supported patterns are:
 148 #
 149 # Pattern: #define @key@
 150 # Found:   #define key value
 151 # Missing: /* #define key */
 152 #
 153 # Pattern: #define @key@ default
 154 # Found:   #define key value
 155 # Missing: #define key default
 156 #
 157 # Pattern: #undef @key@
 158 # Found:   #define key value
 159 # Missing: #undef key
 160 #
 161 # The "@" is used so that these defines can be used in addition to
 162 # other defines that you do not desire to be replaced.  Also, each
 163 # key can specify a format to apply some formatting to the returned
 164 # value if used:
 165 #
 166 # str: The returned value will be enclosed in double quotes and escaped
 167 # chr: The returned value will be enclosed in single quotes and escaped
 168 #
 169 # Example:
 170 #
 171 # #define @key:str@ "Default"
 172 #
 173 ##############################################################################
 174 
 175 # Escape function
 176 _SubstHeader_escape_map = { "\n": "\\n",
 177                             "\r": "\\r",
 178                             "\t": "\\t",
 179                             "\\": "\\\\",
 180                             "\0": "\\0",
 181                             "\"": "\\\"",
 182                             "\'": "\\\'" }
 183 def _SubstHeader_escape(value):
 184     # TODO: support replacement on all characters that need it
 185     result = []
 186     for i in value:
 187         if i in _SubstHeader_escape_map:
 188             result.append(_SubstHeader_escape_map[i])
 189         else:
 190             result.append(i)
 191 
 192     return "".join(result)
 193 
 194 # Format functions
 195 def _SubstHeader_format_chr(value):
 196     escaped = _SubstHeader_escape(value)
 197 
 198     return "\'%s\'" % escaped[0]
 199 
 200 def _SubstHeader_format_str(value):
 201     escaped = _SubstHeader_escape(value)
 202 
 203     return "\"%s\"" % escaped
 204 
 205 _SubstHeader_formats = { "chr": _SubstHeader_format_chr,
 206                          "str": _SubstHeader_format_str }
 207 
 208 
 209 # Actual substitution
 210 _SubstHeader_pattern = "(?m)^(?P<space>\\s*?)(?P<type>#define|#undef)\\s+?@(?P<key>\w+?)(:(?P<fmt>\w+?))?@(?P<ending>.*?)$"
 211 def _SubstHeader_replace(env, mo):
 212     space = mo.group("space")
 213     type = mo.group("type")
 214     key = mo.group("key")
 215     ending = mo.group("ending")
 216     fmt = mo.group("fmt")
 217 
 218     value = _subst_value(env, key)
 219     if not value is None:
 220         if fmt in _SubstHeader_formats:
 221             value = _SubstHeader_formats[fmt](value)
 222 
 223         # If found it is always #define key value
 224         return "%s#define %s %s" % (space, key, value)
 225         
 226     # Not found
 227     if type == "#define":
 228         defval = ending.strip()
 229         if defval:
 230             # There is a default value
 231             return "%s#define %s %s" % (space, key, defval)
 232         else:
 233             # There is no default value
 234             return "%s/* #define %s */" % (space, key)
 235 
 236     # It was #undef
 237     return "%s#undef %s" % (space, key)
 238         
 239 def SubstHeader(env, target, source):
 240     return env.SubstGeneric(target,
 241                             source,
 242                             SUBST_PATTERN=_SubstHeader_pattern,
 243                             SUBST_REPLACE=_SubstHeader_replace)
 244 
 245 
 246 # Create builders
 247 ##############################################################################
 248 def TOOL_SUBST(env):
 249     # The generic builder
 250     subst = SCons.Action.Action(_subst_action, _subst_message)
 251     env["BUILDERS"]["SubstGeneric"] = Builder(action=subst,
 252                                               emitter=_subst_emitter)
 253 
 254     # Additional ones
 255     env.AddMethod(SubstFile, "SubstFile")
 256     env.AddMethod(SubstHeader, "SubstHeader")

SubstFile

The SubstFile builder works just like SubstInFile2, replacing any pattern of "@name@" with the value of that environment variable, and any "@@" with a single "@". If the environment variable does not exist it will result in an error instead of silently replacing it with an empty string.

SConstruct:

import subst

env = Environment()
TOOL_SUBST(env)

env["DISPLAY_NAME"] = "My Application 2009"
env["PREFIX"] = "/usr/local"
env["DESCRIPTION"] = "An application to do something."

env.SubstFile("myapp.desktop", "desktop.in")

desktop.in:

[Desktop Entry]
Encoding=UTF-8
Name=@DISPLAY_NAME@
Type=Application
Comment=@DESCRIPTION@
TryExec=@PREFIX@/bin/myapp
Exec=@PREFIX@/bin/myapp
Icon=@PREFIX@/share/myapp/pixmaps/mainicon.svg

myapp.desktop:

[Desktop Entry]
Encoding=UTF-8
Name=My Application 2009
Type=Application
Comment=An application to do something.
TryExec=/usr/local/bin/myapp
Exec=/usr/local/bin/myapp
Icon=/usr/local/share/myapp/pixmaps/mainicon.svg

SubstHeader

This works a little bit different than autotools, but the basic idea is the same. Certain matches as described in the source file will be replaced with the value from the environment if available. If not available, then it will be replaced according to the type of define statement. In addition, the names in the define statement are still surrounded with "@" to control which statements may be substituted or not. A define statement without "@" around the name will not be substituted.

SConstruct:

import subst

env = Environment()
TOOL_SUBST(env)

env["HAVE_STRDUP"] = 1
env["WITH_WXWIDGETS"] = 1
env["HAVE_MATH_H"] = 1

env.SubstHeader("config.h", "config.h.in")

config.h.in:

// If you do not use @NAME@ then it will not be a candidate for substitution.  This
// allows regular defines to be placed in a config.h.in type file as well without
// needing to make sure they are not environment variables that would be substituted.
#define COPYRIGHT "Copyright (C) 2009 Foobar"

// This provides a default value if the key is not in the environment.
#define @HAVE_STRDUP@ 0
#define @HAVE_STRCAT@ 0

// If there is no defualt value and it is not in the environment, it will be commented out
#define @WITH_WXWIDGETS@
#define @WITH_OPENGL@

// If #undef is used and the value is not in the environment, it will emit an #undef statement
#undef @HAVE_MATH_H@
#undef @HAVE_STRING_H@

config.h.in:

// If you do not use @NAME@ then it will not be a candidate for substitution.  This
// allows regular defines to be placed in a config.h.in type file as well without
// needing to make sure they are not environment variables that would be substituted.
#define COPYRIGHT "Copyright (C) 2009 Foobar"

// This provides a default value if the key is not in the environment.
#define HAVE_STRDUP 1
#define HAVE_STRCAT 0

// If there is no defualt value and it is not in the environment, it will be commented out
#define WITH_WXWIDGETS
/* #define WITH_OPENGL */

// If #undef is used and the value is not in the environment, it will emit an #undef statement
#define HAVE_MATH_H 1
#undef HAVE_STRING_H

Custom Substitution

To create a custom substitution, all that is needed is a pattern to match for and a function that will be called to return the substituted value. The pattern must have a named parameter called "key" which will be used for dependency tracking. The function will take the environment and the regular expression match object as parameters. If an there is an error such as a missing variable if it is required, the function should raise a SCons.Errors.UserError

   1 pattern = "#(P<key>\w*?)#"
   2 def replace(env, mo):
   3     key = mo.group("key")
   4     if not key:
   5         return "#"
   6 
   7     # "if key in env: ..." causes a KeyError for some reason instead of "key in env" being False
   8     # So test like this instead
   9     try:
  10         env[key] 
  11     except KeyError:
  12         raise SCons.Errors.UserError("Key not found: %s" % key)
  13 
  14     return env.subst("${%s}" % key); 
  15         
  16 env.SubstGeneric("output", "input.h", SUBST_PATTERN=pattern, SUBST_REPLACE=replace)

GenericSubstBuilder (last edited 2012-05-09 15:08:38 by BrianVanderburgII)