http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/feature.jam ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/boost-build/build/feature.jam b/ext/kenlm/jam-files/boost-build/build/feature.jam deleted file mode 100644 index ee6abc5..0000000 --- a/ext/kenlm/jam-files/boost-build/build/feature.jam +++ /dev/null @@ -1,1350 +0,0 @@ -# Copyright 2001, 2002, 2003 Dave Abrahams -# Copyright 2002, 2006 Rene Rivera -# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import assert : * ; -import "class" : * ; -import indirect ; -import modules ; -import regex ; -import sequence ; -import set ; -import utility ; - - -local rule setup ( ) -{ - .all-attributes = - implicit - composite - optional - symmetric - free - incidental - path - dependency - propagated - link-incompatible - subfeature - order-sensitive - ; - - .all-features = ; - .all-subfeatures = ; - .all-top-features = ; # non-subfeatures - .all-implicit-values = ; -} -setup ; - - -# Prepare a fresh space to test in by moving all global variable settings into -# the given temporary module and erasing them here. -# -rule prepare-test ( temp-module ) -{ - DELETE_MODULE $(temp-module) ; - - # Transfer globals to temp-module. - for local v in [ VARNAMES feature ] - { - if [ MATCH (\\.) : $(v) ] - { - modules.poke $(temp-module) : $(v) : $($(v)) ; - $(v) = ; - } - } - setup ; -} - - -# Clear out all global variables and recover all variables from the given -# temporary module. -# -rule finish-test ( temp-module ) -{ - # Clear globals. - for local v in [ VARNAMES feature ] - { - if [ MATCH (\\.) : $(v) ] - { - $(v) = ; - } - } - - for local v in [ VARNAMES $(temp-module) ] - { - $(v) = [ modules.peek $(temp-module) : $(v) ] ; - } - DELETE_MODULE $(temp-module) ; -} - - -# Transform features by bracketing any elements which are not already bracketed -# by "<>". -# -local rule grist ( features * ) -{ - local empty = "" ; - return $(empty:G=$(features)) ; -} - - -# Declare a new feature with the given name, values, and attributes. -# -rule feature ( - name # Feature name. - : values * # Allowable values - may be extended later using feature.extend. - : attributes * # Feature attributes (e.g. implicit, free, propagated...). -) -{ - name = [ grist $(name) ] ; - - local error ; - - # Check for any unknown attributes. - if ! ( $(attributes) in $(.all-attributes) ) - { - error = unknown attributes: - [ set.difference $(attributes) : $(.all-attributes) ] ; - } - else if $(name) in $(.all-features) - { - error = feature already defined: ; - } - else if implicit in $(attributes) && free in $(attributes) - { - error = free features cannot also be implicit ; - } - else if free in $(attributes) && propagated in $(attributes) - { - error = free features cannot be propagated ; - } - else - { - local m = [ MATCH (.*=.*) : $(values) ] ; - if $(m[1]) - { - error = "feature value may not contain '='" ; - } - } - - if $(error) - { - import errors ; - errors.error $(error) - : "in" feature declaration: - : feature [ errors.lol->list $(1) : $(2) : $(3) ] ; - } - - $(name).values ?= ; - $(name).attributes = $(attributes) ; - $(name).subfeatures ?= ; - $(attributes).features += $(name) ; - - .all-features += $(name) ; - if subfeature in $(attributes) - { - .all-subfeatures += $(name) ; - } - else - { - .all-top-features += $(name) ; - } - extend $(name) : $(values) ; -} - - -# Sets the default value of the given feature, overriding any previous default. -# -rule set-default ( feature : value ) -{ - local f = [ grist $(feature) ] ; - local a = $($(f).attributes) ; - local bad-attribute = ; - if free in $(a) - { - bad-attribute = free ; - } - else if optional in $(a) - { - bad-attribute = optional ; - } - if $(bad-attribute) - { - import errors ; - errors.error $(bad-attribute) property $(f) cannot have a default. ; - } - if ! $(value) in $($(f).values) - { - import errors ; - errors.error The specified default value, '$(value)' is invalid : - allowed values are: $($(f).values) ; - } - $(f).default = $(value) ; -} - - -# Returns the default property values for the given features. -# -rule defaults ( features * ) -{ - local result ; - for local f in $(features) - { - local gf = $(:E=:G=$(f)) ; - local a = $($(gf).attributes) ; - if ( free in $(a) ) || ( optional in $(a) ) - { - } - else - { - result += $(gf)$($(gf).default) ; - } - } - return $(result) ; -} - - -# Returns true iff all 'names' elements are valid features. -# -rule valid ( names + ) -{ - if $(names) in $(.all-features) - { - return true ; - } -} - - -# Returns the attibutes of the given feature. -# -rule attributes ( feature ) -{ - return $($(feature).attributes) ; -} - - -# Returns the values of the given feature. -# -rule values ( feature ) -{ - return $($(:E=:G=$(feature)).values) ; -} - - -# Returns true iff 'value-string' is a value-string of an implicit feature. -# -rule is-implicit-value ( value-string ) -{ - local v = [ regex.split $(value-string) - ] ; - local failed ; - if ! $(v[1]) in $(.all-implicit-values) - { - failed = true ; - } - else - { - local feature = $($(v[1]).implicit-feature) ; - for local subvalue in $(v[2-]) - { - if ! [ find-implied-subfeature $(feature) $(subvalue) : $(v[1]) ] - { - failed = true ; - } - } - } - - if ! $(failed) - { - return true ; - } -} - - -# Returns the implicit feature associated with the given implicit value. -# -rule implied-feature ( implicit-value ) -{ - local components = [ regex.split $(implicit-value) "-" ] ; - local feature = $($(components[1]).implicit-feature) ; - if ! $(feature) - { - import errors ; - errors.error \"$(implicit-value)\" is not an implicit feature value ; - feature = "" ; # Keep testing happy; it expects a result. - } - return $(feature) ; -} - - -local rule find-implied-subfeature ( feature subvalue : value-string ? ) -{ - # Feature should be of the form <feature-name>. - if $(feature) != $(feature:G) - { - import errors ; - errors.error invalid feature $(feature) ; - } - return $($(feature)$(value-string:E="")<>$(subvalue).subfeature) ; -} - - -# Given a feature and a value of one of its subfeatures, find the name of the -# subfeature. If value-string is supplied, looks for implied subfeatures that -# are specific to that value of feature -# -rule implied-subfeature ( - feature # The main feature name. - subvalue # The value of one of its subfeatures. - : value-string ? # The value of the main feature. -) -{ - local subfeature = [ find-implied-subfeature $(feature) $(subvalue) - : $(value-string) ] ; - if ! $(subfeature) - { - value-string ?= "" ; - import errors ; - errors.error \"$(subvalue)\" is not a known subfeature value of - $(feature)$(value-string) ; - } - return $(subfeature) ; -} - - -# Generate an error if the feature is unknown. -# -local rule validate-feature ( feature ) -{ - if ! $(feature) in $(.all-features) - { - import errors ; - errors.error unknown feature \"$(feature)\" ; - } -} - - -# Given a feature and its value or just a value corresponding to an implicit -# feature, returns a property set consisting of all component subfeatures and -# their values. For example all the following calls: -# -# expand-subfeatures-aux <toolset>gcc-2.95.2-linux-x86 -# expand-subfeatures-aux gcc-2.95.2-linux-x86 -# -# return: -# -# <toolset>gcc <toolset-version>2.95.2 <toolset-os>linux <toolset-cpu>x86 -# -local rule expand-subfeatures-aux ( - feature ? # Feature name or empty if value corresponds to an - # implicit property. - : value # Feature value. - : dont-validate ? # If set, no value string validation will be done. -) -{ - if $(feature) - { - feature = $(feature) ; - } - - if ! $(feature) - { - feature = [ implied-feature $(value) ] ; - } - else - { - validate-feature $(feature) ; - } - if ! $(dont-validate) - { - validate-value-string $(feature) $(value) ; - } - - local components = [ regex.split $(value) "-" ] ; - - # Get the top-level feature's value. - local value = $(components[1]:G=) ; - - local result = $(components[1]:G=$(feature)) ; - - local subvalues = $(components[2-]) ; - while $(subvalues) - { - local subvalue = $(subvalues[1]) ; # Pop the head off of subvalues. - subvalues = $(subvalues[2-]) ; - - local subfeature = [ find-implied-subfeature $(feature) $(subvalue) : - $(value) ] ; - - # If no subfeature was found reconstitute the value string and use that. - if ! $(subfeature) - { - result = $(components:J=-) ; - result = $(result:G=$(feature)) ; - subvalues = ; # Stop looping. - } - else - { - local f = [ MATCH ^<(.*)>$ : $(feature) ] ; - result += $(subvalue:G=$(f)-$(subfeature)) ; - } - } - - return $(result) ; -} - - -# Make all elements of properties corresponding to implicit features explicit, -# and express all subfeature values as separate properties in their own right. -# For example, all of the following properties -# -# gcc-2.95.2-linux-x86 -# <toolset>gcc-2.95.2-linux-x86 -# -# might expand to -# -# <toolset>gcc <toolset-version>2.95.2 <toolset-os>linux <toolset-cpu>x86 -# -rule expand-subfeatures ( - properties * # Property set with elements of the form - # <feature>value-string or just value-string in the case - # of implicit features. - : dont-validate ? -) -{ - local result ; - for local p in $(properties) - { - # Don't expand subfeatures in subfeatures - if ! [ MATCH "(:)" : $(p:G) ] - { - result += [ expand-subfeatures-aux $(p:G) : $(p:G=) : $(dont-validate) ] ; - } - else - { - result += $(p) ; - } - } - return $(result) ; -} - - -# Helper for extend, below. Handles the feature case. -# -local rule extend-feature ( feature : values * ) -{ - feature = [ grist $(feature) ] ; - validate-feature $(feature) ; - if implicit in $($(feature).attributes) - { - for local v in $(values) - { - if $($(v).implicit-feature) - { - import errors ; - errors.error $(v) is already associated with the - \"$($(v).implicit-feature)\" feature ; - } - $(v).implicit-feature = $(feature) ; - } - - .all-implicit-values += $(values) ; - } - if ! $($(feature).values) - { - # This is the first value specified for this feature so make it be the - # default. - $(feature).default = $(values[1]) ; - } - $(feature).values += $(values) ; -} - - -# Checks that value-string is a valid value-string for the given feature. -# -rule validate-value-string ( feature value-string ) -{ - if ! ( - free in $($(feature).attributes) - || ( $(value-string) in $(feature).values ) - ) - { - local values = $(value-string) ; - - if $($(feature).subfeatures) - { - if ! $(value-string) in $($(feature).values) - $($(feature).subfeatures) - { - values = [ regex.split $(value-string) - ] ; - } - } - - if ! ( $(values[1]) in $($(feature).values) ) && - - # An empty value is allowed for optional features. - ( $(values[1]) || ! ( optional in $($(feature).attributes) ) ) - { - import errors ; - errors.error \"$(values[1])\" is not a known value of feature - $(feature) : legal values: \"$($(feature).values)\" ; - } - - for local v in $(values[2-]) - { - # This will validate any subfeature values in value-string. - implied-subfeature $(feature) $(v) : $(values[1]) ; - } - } -} - - -# A helper that computes: -# * name(s) of module-local variable(s) used to record the correspondence -# between subvalue(s) and a subfeature -# * value of that variable when such a subfeature/subvalue has been defined and -# returns a list consisting of the latter followed by the former. -# -local rule subvalue-var ( - feature # Main feature name. - value-string ? # If supplied, specifies a specific value of the main - # feature for which the subfeature values are valid. - : subfeature # Subfeature name. - : subvalues * # Subfeature values. -) -{ - feature = [ grist $(feature) ] ; - validate-feature $(feature) ; - if $(value-string) - { - validate-value-string $(feature) $(value-string) ; - } - - local subfeature-name = [ get-subfeature-name $(subfeature) $(value-string) ] ; - - return $(subfeature-name) - $(feature)$(value-string:E="")<>$(subvalues).subfeature ; -} - - -# Extends the given subfeature with the subvalues. If the optional value-string -# is provided, the subvalues are only valid for the given value of the feature. -# Thus, you could say that <target-platform>mingw is specific to -# <toolset>gcc-2.95.2 as follows: -# -# extend-subfeature toolset gcc-2.95.2 : target-platform : mingw ; -# -rule extend-subfeature ( - feature # The feature whose subfeature is being extended. - - value-string ? # If supplied, specifies a specific value of the main - # feature for which the new subfeature values are valid. - - : subfeature # Subfeature name. - : subvalues * # Additional subfeature values. -) -{ - local subfeature-vars = [ subvalue-var $(feature) $(value-string) - : $(subfeature) : $(subvalues) ] ; - - local f = [ utility.ungrist [ grist $(feature) ] ] ; - extend $(f)-$(subfeature-vars[1]) : $(subvalues) ; - - # Provide a way to get from the given feature or property and subfeature - # value to the subfeature name. - $(subfeature-vars[2-]) = $(subfeature-vars[1]) ; -} - - -# Returns true iff the subvalues are valid for the feature. When the optional -# value-string is provided, returns true iff the subvalues are valid for the -# given value of the feature. -# -rule is-subvalue ( feature : value-string ? : subfeature : subvalue ) -{ - local subfeature-vars = [ subvalue-var $(feature) $(value-string) - : $(subfeature) : $(subvalue) ] ; - - if $($(subfeature-vars[2])) = $(subfeature-vars[1]) - { - return true ; - } -} - - -# Can be called three ways: -# -# 1. extend feature : values * -# 2. extend <feature> subfeature : values * -# 3. extend <feature>value-string subfeature : values * -# -# * Form 1 adds the given values to the given feature. -# * Forms 2 and 3 add subfeature values to the given feature. -# * Form 3 adds the subfeature values as specific to the given property -# value-string. -# -rule extend ( feature-or-property subfeature ? : values * ) -{ - local feature ; # If a property was specified this is its feature. - local value-string ; # E.g., the gcc-2.95-2 part of <toolset>gcc-2.95.2. - - # If a property was specified. - if $(feature-or-property:G) && $(feature-or-property:G=) - { - # Extract the feature and value-string, if any. - feature = $(feature-or-property:G) ; - value-string = $(feature-or-property:G=) ; - } - else - { - feature = [ grist $(feature-or-property) ] ; - } - - # Dispatch to the appropriate handler. - if $(subfeature) - { - extend-subfeature $(feature) $(value-string) : $(subfeature) - : $(values) ; - } - else - { - # If no subfeature was specified, we do not expect to see a - # value-string. - if $(value-string) - { - import errors ; - errors.error can only specify a property as the first argument when - extending a subfeature - : usage: - : " extend" feature ":" values... - : " | extend" <feature>value-string subfeature ":" values... ; - } - - extend-feature $(feature) : $(values) ; - } -} - - -local rule get-subfeature-name ( subfeature value-string ? ) -{ - local prefix = $(value-string): ; - return $(prefix:E="")$(subfeature) ; -} - - -# Declares a subfeature. -# -rule subfeature ( - feature # Root feature that is not a subfeature. - value-string ? # A value-string specifying which feature or subfeature - # values this subfeature is specific to, if any. - : subfeature # The name of the subfeature being declared. - : subvalues * # The allowed values of this subfeature. - : attributes * # The attributes of the subfeature. -) -{ - feature = [ grist $(feature) ] ; - validate-feature $(feature) ; - - # Add grist to the subfeature name if a value-string was supplied. - local subfeature-name = [ get-subfeature-name $(subfeature) $(value-string) ] ; - - if $(subfeature-name) in $($(feature).subfeatures) - { - import errors ; - errors.error \"$(subfeature)\" already declared as a subfeature of - \"$(feature)\" "specific to "$(value-string) ; - } - $(feature).subfeatures += $(subfeature-name) ; - - # First declare the subfeature as a feature in its own right. - local f = [ utility.ungrist $(feature) ] ; - feature $(f)-$(subfeature-name) : $(subvalues) : $(attributes) subfeature ; - - # Now make sure the subfeature values are known. - extend-subfeature $(feature) $(value-string) : $(subfeature) : $(subvalues) ; -} - - -# Set components of the given composite property. -# -rule compose ( composite-property : component-properties * ) -{ - local feature = $(composite-property:G) ; - if ! ( composite in [ attributes $(feature) ] ) - { - import errors ; - errors.error "$(feature)" is not a composite feature ; - } - - $(composite-property).components ?= ; - if $($(composite-property).components) - { - import errors ; - errors.error components of "$(composite-property)" already set: - $($(composite-property).components) ; - } - - if $(composite-property) in $(component-properties) - { - import errors ; - errors.error composite property "$(composite-property)" cannot have itself as a component ; - } - $(composite-property).components = $(component-properties) ; -} - - -local rule expand-composite ( property ) -{ - return $(property) - [ sequence.transform expand-composite : $($(property).components) ] ; -} - - -# Return all values of the given feature specified by the given property set. -# -rule get-values ( feature : properties * ) -{ - local result ; - - feature = $(:E=:G=$(feature)) ; # Add <> if necessary. - for local p in $(properties) - { - if $(p:G) = $(feature) - { - # Use MATCH instead of :G= to get the value, in order to preserve - # the value intact instead of having bjam treat it as a decomposable - # path. - result += [ MATCH ">(.*)" : $(p) ] ; - } - } - return $(result) ; -} - - -rule free-features ( ) -{ - return $(free.features) ; -} - - -# Expand all composite properties in the set so that all components are -# explicitly expressed. -# -rule expand-composites ( properties * ) -{ - local explicit-features = $(properties:G) ; - local result ; - - # Now expand composite features. - for local p in $(properties) - { - local expanded = [ expand-composite $(p) ] ; - - for local x in $(expanded) - { - if ! $(x) in $(result) - { - local f = $(x:G) ; - - if $(f) in $(free.features) - { - result += $(x) ; - } - else if ! $(x) in $(properties) # x is the result of expansion - { - if ! $(f) in $(explicit-features) # not explicitly-specified - { - if $(f) in $(result:G) - { - import errors ; - errors.error expansions of composite features result - in conflicting values for $(f) - : values: [ get-values $(f) : $(result) ] $(x:G=) - : one contributing composite property was $(p) ; - } - else - { - result += $(x) ; - } - } - } - else if $(f) in $(result:G) - { - import errors ; - errors.error explicitly-specified values of non-free feature - $(f) conflict : - "existing values:" [ get-values $(f) : $(properties) ] : - "value from expanding " $(p) ":" $(x:G=) ; - } - else - { - result += $(x) ; - } - } - } - } - return $(result) ; -} - - -# Return true iff f is an ordinary subfeature of the parent-property's feature, -# or if f is a subfeature of the parent-property's feature specific to the -# parent-property's value. -# -local rule is-subfeature-of ( parent-property f ) -{ - if subfeature in $($(f).attributes) - { - local specific-subfeature = [ MATCH <(.*):(.*)> : $(f) ] ; - if $(specific-subfeature) - { - # The feature has the form <topfeature-topvalue:subfeature>, e.g. - # <toolset-msvc:version>. - local feature-value = [ split-top-feature $(specific-subfeature[1]) - ] ; - if <$(feature-value[1])>$(feature-value[2]) = $(parent-property) - { - return true ; - } - } - else - { - # The feature has the form <topfeature-subfeature>, e.g. - # <toolset-version> - local top-sub = [ split-top-feature [ utility.ungrist $(f) ] ] ; - if $(top-sub[2]) && <$(top-sub[1])> = $(parent-property:G) - { - return true ; - } - } - } -} - - -# As for is-subfeature-of but for subproperties. -# -local rule is-subproperty-of ( parent-property p ) -{ - return [ is-subfeature-of $(parent-property) $(p:G) ] ; -} - - -# Given a property, return the subset of features consisting of all ordinary -# subfeatures of the property's feature, and all specific subfeatures of the -# property's feature which are conditional on the property's value. -# -local rule select-subfeatures ( parent-property : features * ) -{ - return [ sequence.filter is-subfeature-of $(parent-property) : $(features) ] ; -} - - -# As for select-subfeatures but for subproperties. -# -local rule select-subproperties ( parent-property : properties * ) -{ - return [ sequence.filter is-subproperty-of $(parent-property) : $(properties) ] ; -} - - -# Given a property set which may consist of composite and implicit properties -# and combined subfeature values, returns an expanded, normalized property set -# with all implicit features expressed explicitly, all subfeature values -# individually expressed, and all components of composite properties expanded. -# Non-free features directly expressed in the input properties cause any values -# of those features due to composite feature expansion to be dropped. If two -# values of a given non-free feature are directly expressed in the input, an -# error is issued. -# -rule expand ( properties * ) -{ - local expanded = [ expand-subfeatures $(properties) ] ; - return [ expand-composites $(expanded) ] ; -} - - -# Helper rule for minimize. Returns true iff property's feature is present in -# the contents of the variable named by feature-set-var. -# -local rule in-features ( feature-set-var property ) -{ - if $(property:G) in $($(feature-set-var)) - { - return true ; - } -} - - -# Helper rule for minimize. Returns the list with the same properties, but with -# all subfeatures moved to the end of the list. -# -local rule move-subfeatures-to-the-end ( properties * ) -{ - local x1 ; - local x2 ; - for local p in $(properties) - { - if subfeature in $($(p:G).attributes) - { - x2 += $(p) ; - } - else - { - x1 += $(p) ; - } - } - return $(x1) $(x2) ; -} - - -# Given an expanded property set, eliminate all redundancy: properties that are -# elements of other (composite) properties in the set will be eliminated. -# Non-symmetric properties equal to default values will be eliminated unless -# they override a value from some composite property. Implicit properties will -# be expressed without feature grist, and sub-property values will be expressed -# as elements joined to the corresponding main property. -# -rule minimize ( properties * ) -{ - # Precondition checking - local implicits = [ set.intersection $(p:G=) : $(p:G) ] ; - if $(implicits) - { - import errors ; - errors.error minimize requires an expanded property set, but - \"$(implicits[1])\" appears to be the value of an un-expanded - implicit feature ; - } - - # Remove properties implied by composite features. - local components = $($(properties).components) ; - local x = [ set.difference $(properties) : $(components) ] ; - - # Handle subfeatures and implicit features. - x = [ move-subfeatures-to-the-end $(x) ] ; - local result ; - while $(x) - { - local p fullp = $(x[1]) ; - local f = $(p:G) ; - local v = $(p:G=) ; - - # Eliminate features in implicit properties. - if implicit in [ attributes $(f) ] - { - p = $(v) ; - } - - # Locate all subproperties of $(x[1]) in the property set. - local subproperties = [ select-subproperties $(fullp) : $(x) ] ; - if $(subproperties) - { - # Reconstitute the joined property name. - local sorted = [ sequence.insertion-sort $(subproperties) ] ; - result += $(p)-$(sorted:G="":J=-) ; - - x = [ set.difference $(x[2-]) : $(subproperties) ] ; - } - else - { - # Eliminate properties whose value is equal to feature's default, - # which are not symmetric and which do not contradict values implied - # by composite properties. - - # Since all component properties of composites in the set have been - # eliminated, any remaining property whose feature is the same as a - # component of a composite in the set must have a non-redundant - # value. - if $(fullp) != [ defaults $(f) ] - || symmetric in [ attributes $(f) ] - || $(fullp:G) in $(components:G) - { - result += $(p) ; - } - - x = $(x[2-]) ; - } - } - return $(result) ; -} - - -# Combine all subproperties into their parent properties -# -# Requires: for every subproperty, there is a parent property. All features are -# explicitly expressed. -# -# This rule probably should not be needed, but build-request.expand-no-defaults -# is being abused for unintended purposes and it needs help. -# -rule compress-subproperties ( properties * ) -{ - local all-subs ; - local matched-subs ; - local result ; - - for local p in $(properties) - { - if ! $(p:G) - { - # Expecting fully-gristed properties. - assert.variable-not-empty p:G ; - } - - if ! subfeature in $($(p:G).attributes) - { - local subs = [ sequence.insertion-sort - [ sequence.filter is-subproperty-of $(p) : $(properties) ] ] ; - - matched-subs += $(subs) ; - - local subvalues = -$(subs:G=:J=-) ; - subvalues ?= "" ; - result += $(p)$(subvalues) ; - } - else - { - all-subs += $(p) ; - } - } - assert.result true : set.equal $(all-subs) : $(matched-subs) ; - return $(result) ; -} - - -# Given an ungristed string, finds the longest prefix which is a top-level -# feature name followed by a dash, and return a pair consisting of the parts -# before and after that dash. More interesting than a simple split because -# feature names may contain dashes. -# -local rule split-top-feature ( feature-plus ) -{ - local e = [ regex.split $(feature-plus) - ] ; - local f = $(e[1]) ; - local v ; - while $(e) - { - if <$(f)> in $(.all-top-features) - { - v = $(f) $(e[2-]:J=-) ; - } - e = $(e[2-]) ; - f = $(f)-$(e[1]) ; - } - return $(v) ; -} - - -# Given a set of properties, add default values for features not represented in -# the set. -# -# Note: if there's an ordinary feature F1 and a composite feature F2 which -# includes some value for F1 and both feature have default values then the -# default value of F1 will be added (as opposed to the value in F2). This might -# not be the right idea, e.g. consider: -# -# feature variant : debug ... ; -# <variant>debug : .... <runtime-debugging>on -# feature <runtime-debugging> : off on ; -# -# Here, when adding default for an empty property set, we'll get -# -# <variant>debug <runtime_debugging>off -# -# and that's kind of strange. -# -rule add-defaults ( properties * ) -{ - for local v in $(properties:G=) - { - if $(v) in $(properties) - { - import errors ; - errors.error add-defaults requires explicitly specified features, - but \"$(v)\" appears to be the value of an un-expanded implicit - feature ; - } - } - # We don't add default for elements with ":" inside. This catches: - # 1. Conditional properties --- we don't want <variant>debug:<define>DEBUG - # to be takes as specified value for <variant> - # 2. Free properties with ":" in values. We don't care, since free - # properties don't have defaults. - local xproperties = [ MATCH "^([^:]+)$" : $(properties) ] ; - local missing-top = [ set.difference $(.all-top-features) : $(xproperties:G) ] ; - local more = [ defaults $(missing-top) ] ; - properties += $(more) ; - xproperties += $(more) ; - - # Add defaults for subfeatures of features which are present. - for local p in $(xproperties) - { - local s = $($(p:G).subfeatures) ; - local f = [ utility.ungrist $(p:G) ] ; - local missing-subs = [ set.difference <$(f)-$(s)> : $(properties:G) ] ; - properties += [ defaults [ select-subfeatures $(p) : $(missing-subs) ] ] ; - } - - return $(properties) ; -} - - -# Given a property-set of the form -# v1/v2/...vN-1/<fN>vN/<fN+1>vN+1/...<fM>vM -# -# Returns -# v1 v2 ... vN-1 <fN>vN <fN+1>vN+1 ... <fM>vM -# -# Note that vN...vM may contain slashes. This needs to be resilient to the -# substitution of backslashes for slashes, since Jam, unbidden, sometimes swaps -# slash direction on NT. -# -rule split ( property-set ) -{ - local pieces = [ regex.split $(property-set) [\\/] ] ; - local result ; - - for local x in $(pieces) - { - if ( ! $(x:G) ) && $(result[-1]:G) - { - result = $(result[1--2]) $(result[-1])/$(x) ; - } - else - { - result += $(x) ; - } - } - - return $(result) ; -} - - -# Tests of module feature. -# -rule __test__ ( ) -{ - # Use a fresh copy of the feature module. - prepare-test feature-test-temp ; - - import assert ; - import errors : try catch ; - - # These are local rules and so must be explicitly reimported into the - # testing module. - import feature : extend-feature validate-feature select-subfeatures ; - - feature toolset : gcc : implicit ; - feature define : : free ; - feature runtime-link : dynamic static : symmetric ; - feature optimization : on off ; - feature variant : debug release profile : implicit composite symmetric ; - feature stdlib : native stlport ; - feature magic : : free ; - - compose <variant>debug : <define>_DEBUG <optimization>off ; - compose <variant>release : <define>NDEBUG <optimization>on ; - - assert.result dynamic static : values <runtime-link> ; - assert.result dynamic static : values runtime-link ; - - try ; - { - compose <variant>profile : <variant>profile ; - } - catch composite property <variant>profile cannot have itself as a component ; - - extend-feature toolset : msvc metrowerks ; - subfeature toolset gcc : version : 2.95.2 2.95.3 2.95.4 3.0 3.0.1 3.0.2 ; - - assert.true is-subvalue toolset : gcc : version : 2.95.3 ; - assert.false is-subvalue toolset : gcc : version : 1.1 ; - - assert.false is-subvalue toolset : msvc : version : 2.95.3 ; - assert.false is-subvalue toolset : : version : yabba ; - - feature yabba ; - subfeature yabba : version : dabba ; - assert.true is-subvalue yabba : : version : dabba ; - - subfeature toolset gcc : platform : linux cygwin : optional ; - - assert.result <toolset-gcc:version> - : select-subfeatures <toolset>gcc - : <toolset-gcc:version> - <toolset-msvc:version> - <toolset-version> - <stdlib> ; - - subfeature stdlib : version : 3 4 : optional ; - - assert.result <stdlib-version> - : select-subfeatures <stdlib>native - : <toolset-gcc:version> - <toolset-msvc:version> - <toolset-version> - <stdlib-version> ; - - assert.result <toolset>gcc <toolset-gcc:version>3.0.1 - : expand-subfeatures <toolset>gcc-3.0.1 ; - - assert.result <toolset>gcc <toolset-gcc:version>3.0.1 <toolset-gcc:platform>linux - : expand-subfeatures <toolset>gcc-3.0.1-linux ; - - assert.result <toolset>gcc <toolset-gcc:version>3.0.1 - : expand <toolset>gcc <toolset-gcc:version>3.0.1 ; - - assert.result <define>foo=x-y - : expand-subfeatures <define>foo=x-y ; - - assert.result <toolset>gcc <toolset-gcc:version>3.0.1 - : expand-subfeatures gcc-3.0.1 ; - - assert.result a c e - : get-values <x> : <x>a <y>b <x>c <y>d <x>e ; - - assert.result <toolset>gcc <toolset-gcc:version>3.0.1 - <variant>debug <define>_DEBUG <optimization>on - : expand gcc-3.0.1 debug <optimization>on ; - - assert.result <variant>debug <define>_DEBUG <optimization>on - : expand debug <optimization>on ; - - assert.result <optimization>on <variant>debug <define>_DEBUG - : expand <optimization>on debug ; - - assert.result <runtime-link>dynamic <optimization>on - : defaults <runtime-link> <define> <optimization> ; - - # Make sure defaults is resilient to missing grist. - assert.result <runtime-link>dynamic <optimization>on - : defaults runtime-link define optimization ; - - feature dummy : dummy1 dummy2 ; - subfeature dummy : subdummy : x y z : optional ; - - feature fu : fu1 fu2 : optional ; - subfeature fu : subfu : x y z : optional ; - subfeature fu : subfu2 : q r s ; - - assert.result optional : attributes <fu> ; - - assert.result <runtime-link>static <define>foobar <optimization>on - <toolset>gcc:<define>FOO <toolset>gcc <variant>debug <stdlib>native - <dummy>dummy1 <toolset-gcc:version>2.95.2 - : add-defaults <runtime-link>static <define>foobar <optimization>on - <toolset>gcc:<define>FOO ; - - assert.result <runtime-link>static <define>foobar <optimization>on - <toolset>gcc:<define>FOO <fu>fu1 <toolset>gcc <variant>debug - <stdlib>native <dummy>dummy1 <fu-subfu2>q <toolset-gcc:version>2.95.2 - : add-defaults <runtime-link>static <define>foobar <optimization>on - <toolset>gcc:<define>FOO <fu>fu1 ; - - set-default <runtime-link> : static ; - assert.result <runtime-link>static : defaults <runtime-link> ; - - assert.result gcc-3.0.1 debug <optimization>on - : minimize [ expand gcc-3.0.1 debug <optimization>on <stdlib>native ] ; - - assert.result gcc-3.0.1 debug <runtime-link>dynamic - : minimize - [ expand gcc-3.0.1 debug <optimization>off <runtime-link>dynamic ] ; - - assert.result gcc-3.0.1 debug - : minimize [ expand gcc-3.0.1 debug <optimization>off ] ; - - assert.result debug <optimization>on - : minimize [ expand debug <optimization>on ] ; - - assert.result gcc-3.0 - : minimize <toolset>gcc <toolset-gcc:version>3.0 ; - - assert.result gcc-3.0 - : minimize <toolset-gcc:version>3.0 <toolset>gcc ; - - assert.result <x>y/z <a>b/c <d>e/f - : split <x>y/z/<a>b/c/<d>e/f ; - - assert.result <x>y/z <a>b/c <d>e/f - : split <x>y\\z\\<a>b\\c\\<d>e\\f ; - - assert.result a b c <d>e/f/g <h>i/j/k - : split a/b/c/<d>e/f/g/<h>i/j/k ; - - assert.result a b c <d>e/f/g <h>i/j/k - : split a\\b\\c\\<d>e\\f\\g\\<h>i\\j\\k ; - - # Test error checking. - - try ; - { - expand release <optimization>off <optimization>on ; - } - catch explicitly-specified values of non-free feature <optimization> conflict ; - - try ; - { - validate-feature <foobar> ; - } - catch unknown feature ; - - validate-value-string <toolset> gcc ; - validate-value-string <toolset> gcc-3.0.1 ; - - try ; - { - validate-value-string <toolset> digital_mars ; - } - catch \"digital_mars\" is not a known value of <toolset> ; - - try ; - { - feature foobar : : baz ; - } - catch unknown attributes: baz ; - - feature feature1 ; - try ; - { - feature feature1 ; - } - catch feature already defined: ; - - try ; - { - feature feature2 : : free implicit ; - } - catch free features cannot also be implicit ; - - try ; - { - feature feature3 : : free propagated ; - } - catch free features cannot be propagated ; - - try ; - { - implied-feature lackluster ; - } - catch \"lackluster\" is not an implicit feature value ; - - try ; - { - implied-subfeature <toolset> 3.0.1 ; - } - catch \"3.0.1\" is not a known subfeature value of <toolset> ; - - try ; - { - implied-subfeature <toolset> not-a-version : gcc ; - } - catch \"not-a-version\" is not a known subfeature value of <toolset>gcc ; - - # Leave a clean copy of the features module behind. - finish-test feature-test-temp ; -}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/generators.jam ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/boost-build/build/generators.jam b/ext/kenlm/jam-files/boost-build/build/generators.jam deleted file mode 100644 index ec7183a..0000000 --- a/ext/kenlm/jam-files/boost-build/build/generators.jam +++ /dev/null @@ -1,1420 +0,0 @@ -# Copyright 2002. Vladimir Prus -# Copyright 2006. Rene Rivera -# -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -# Manages 'generators' --- objects which can do transformation between different -# target types and contain algorithm for finding transformation from sources to -# targets. -# -# The main entry point to this module is generators.construct rule. It is given -# a list of source targets, desired target type and a set of properties. It -# starts by selecting 'viable generators', which have any chances of producing -# the desired target type with the required properties. Generators are ranked -# and a set of the most specific ones is selected. -# -# The most specific generators have their 'run' methods called, with the -# properties and list of sources. Each one selects a target which can be -# directly consumed, and tries to convert the remaining ones to the types it can -# consume. This is done by recursively calling 'construct' with all consumable -# types. -# -# If the generator has collected all the targets it needs, it creates targets -# corresponding to result, and returns it. When all generators have been run, -# results of one of them are selected and returned as a result. -# -# It is quite possible for 'construct' to return more targets that it was asked -# for. For example, if it were asked to generate a target of type EXE, but the -# only found generator produces both EXE and TDS (file with debug) information. -# The extra target will be returned. -# -# Likewise, when generator tries to convert sources to consumable types, it can -# get more targets that it was asked for. The question is what to do with extra -# targets. Boost.Build attempts to convert them to requested types, and attempts -# that as early as possible. Specifically, this is done after invoking each -# generator. TODO: An example is needed to document the rationale for trying -# extra target conversion at that point. -# -# In order for the system to be able to use a specific generator instance 'when -# needed', the instance needs to be registered with the system using -# generators.register() or one of its related rules. Unregistered generators may -# only be run explicitly and will not be considered by Boost.Build when when -# converting between given target types. - -import "class" : new ; -import property-set ; -import sequence ; -import set ; -import type ; -import utility ; -import virtual-target ; - - -if "--debug-generators" in [ modules.peek : ARGV ] -{ - .debug = true ; -} - - -# Updated cached viable source target type information as needed after a new -# target type gets defined. This is needed because if a target type is a viable -# source target type for some generator then all of the target type's derived -# target types should automatically be considered as viable source target types -# for the same generator as well. Does nothing if a non-derived target type is -# passed to it. -# -rule update-cached-information-with-a-new-type ( type ) -{ - local base-type = [ type.base $(type) ] ; - if $(base-type) - { - for local g in $(.vstg-cached-generators) - { - if $(base-type) in $(.vstg.$(g)) - { - .vstg.$(g) += $(type) ; - } - } - - for local t in $(.vst-cached-types) - { - if $(base-type) in $(.vst.$(t)) - { - .vst.$(t) += $(type) ; - } - } - } -} - - -# Clears cached viable source target type information except for target types -# and generators with all source types listed as viable. Should be called when -# something invalidates those cached values by possibly causing some new source -# types to become viable. -# -local rule invalidate-extendable-viable-source-target-type-cache ( ) -{ - local generators-with-cached-source-types = $(.vstg-cached-generators) ; - .vstg-cached-generators = ; - for local g in $(generators-with-cached-source-types) - { - if $(.vstg.$(g)) = * - { - .vstg-cached-generators += $(g) ; - } - else - { - .vstg.$(g) = ; - } - } - - local types-with-cached-source-types = $(.vst-cached-types) ; - .vst-cached-types = ; - for local t in $(types-with-cached-source-types) - { - if $(.vst.$(t)) = * - { - .vst-cached-types += $(t) ; - } - else - { - .vst.$(t) = ; - } - } -} - - -# Outputs a debug message if generators debugging is on. Each element of -# 'message' is checked to see if it is a class instance. If so, instead of the -# value, the result of 'str' call is output. -# -local rule generators.dout ( message * ) -{ - if $(.debug) - { - ECHO [ sequence.transform utility.str : $(message) ] ; - } -} - - -local rule indent ( ) -{ - return $(.indent:J="") ; -} - - -local rule increase-indent ( ) -{ - .indent += " " ; -} - - -local rule decrease-indent ( ) -{ - .indent = $(.indent[2-]) ; -} - - -# Models a generator. -# -class generator -{ - import "class" : new ; - import feature ; - import generators : indent increase-indent decrease-indent generators.dout ; - import utility ; - import path ; - import property ; - import sequence ; - import set ; - import type ; - import virtual-target ; - - EXPORT class@generator : indent increase-indent decrease-indent - generators.dout ; - - rule __init__ ( - id # Identifies the generator - should be name - # of the rule which sets up the build - # actions. - - composing ? # Whether generator processes each source - # target in turn, converting it to required - # types. Ordinary generators pass all - # sources together to the recursive - # generators.construct-types call. - - : source-types * # Types that this generator can handle. If - # empty, the generator can consume anything. - - : target-types-and-names + # Types the generator will create and, - # optionally, names for created targets. - # Each element should have the form - # type["(" name-pattern ")"], for example, - # obj(%_x). Generated target name will be - # found by replacing % with the name of - # source, provided an explicit name was not - # specified. - - : requirements * - ) - { - self.id = $(id) ; - self.rule-name = $(id) ; - self.composing = $(composing) ; - self.source-types = $(source-types) ; - self.target-types-and-names = $(target-types-and-names) ; - self.requirements = $(requirements) ; - - for local e in $(target-types-and-names) - { - # Create three parallel lists: one with the list of target types, - # and two other with prefixes and postfixes to be added to target - # name. We use parallel lists for prefix and postfix (as opposed to - # mapping), because given target type might occur several times, for - # example "H H(%_symbols)". - local m = [ MATCH ([^\\(]*)(\\((.*)%(.*)\\))? : $(e) ] ; - self.target-types += $(m[1]) ; - self.name-prefix += $(m[3]:E="") ; - self.name-postfix += $(m[4]:E="") ; - } - - for local r in [ requirements ] - { - if $(r:G=) - { - self.property-requirements += $(r) ; - } - else - { - self.feature-requirements += $(r) ; - } - } - - # Note that 'transform' here, is the same as 'for_each'. - sequence.transform type.validate : $(self.source-types) ; - sequence.transform type.validate : $(self.target-types) ; - } - - ################# End of constructor ################# - - rule id ( ) - { - return $(self.id) ; - } - - # Returns the list of target type the generator accepts. - # - rule source-types ( ) - { - return $(self.source-types) ; - } - - # Returns the list of target types that this generator produces. It is - # assumed to be always the same -- i.e. it can not change depending on some - # provided list of sources. - # - rule target-types ( ) - { - return $(self.target-types) ; - } - - # Returns the required properties for this generator. Properties in returned - # set must be present in build properties if this generator is to be used. - # If result has grist-only element, that build properties must include some - # value of that feature. - # - # XXX: remove this method? - # - rule requirements ( ) - { - return $(self.requirements) ; - } - - rule set-rule-name ( rule-name ) - { - self.rule-name = $(rule-name) ; - } - - rule rule-name ( ) - { - return $(self.rule-name) ; - } - - # Returns a true value if the generator can be run with the specified - # properties. - # - rule match-rank ( property-set-to-match ) - { - # See if generator requirements are satisfied by 'properties'. Treat a - # feature name in requirements (i.e. grist-only element), as matching - # any value of the feature. - - if [ $(property-set-to-match).contains-raw $(self.property-requirements) ] && - [ $(property-set-to-match).contains-features $(self.feature-requirements) ] - { - return true ; - } - else - { - return ; - } - } - - # Returns another generator which differs from $(self) in - # - id - # - value to <toolset> feature in properties - # - rule clone ( new-id : new-toolset-properties + ) - { - local g = [ new $(__class__) $(new-id) $(self.composing) : - $(self.source-types) : $(self.target-types-and-names) : - # Note: this does not remove any subfeatures of <toolset> which - # might cause problems. - [ property.change $(self.requirements) : <toolset> ] - $(new-toolset-properties) ] ; - return $(g) ; - } - - # Creates another generator that is the same as $(self), except that if - # 'base' is in target types of $(self), 'type' will in target types of the - # new generator. - # - rule clone-and-change-target-type ( base : type ) - { - local target-types ; - for local t in $(self.target-types-and-names) - { - local m = [ MATCH ([^\\(]*)(\\(.*\\))? : $(t) ] ; - if $(m) = $(base) - { - target-types += $(type)$(m[2]:E="") ; - } - else - { - target-types += $(t) ; - } - } - - local g = [ new $(__class__) $(self.id) $(self.composing) : - $(self.source-types) : $(target-types) : $(self.requirements) ] ; - if $(self.rule-name) - { - $(g).set-rule-name $(self.rule-name) ; - } - return $(g) ; - } - - # Tries to invoke this generator on the given sources. Returns a list of - # generated targets (instances of 'virtual-target') and optionally a set of - # properties to be added to the usage-requirements for all the generated - # targets. Returning nothing from run indicates that the generator was - # unable to create the target. - # - rule run - ( - project # Project for which the targets are generated. - name ? # Used when determining the 'name' attribute for all - # generated targets. See the 'generated-targets' method. - : property-set # Desired properties for generated targets. - : sources + # Source targets. - ) - { - generators.dout [ indent ] " ** generator" $(self.id) ; - generators.dout [ indent ] " composing:" $(self.composing) ; - - if ! $(self.composing) && $(sources[2]) && $(self.source-types[2]) - { - import errors : error : errors.error ; - errors.error "Unsupported source/source-type combination" ; - } - - # We do not run composing generators if no name is specified. The reason - # is that composing generator combines several targets, which can have - # different names, and it cannot decide which name to give for produced - # target. Therefore, the name must be passed. - # - # This in effect, means that composing generators are runnable only at - # the top-level of a transformation graph, or if their name is passed - # explicitly. Thus, we dissallow composing generators in the middle. For - # example, the transformation CPP -> OBJ -> STATIC_LIB -> RSP -> EXE - # will not be allowed as the OBJ -> STATIC_LIB generator is composing. - if ! $(self.composing) || $(name) - { - run-really $(project) $(name) : $(property-set) : $(sources) ; - } - } - - rule run-really ( project name ? : property-set : sources + ) - { - # Targets that this generator will consume directly. - local consumed = ; - # Targets that can not be consumed and will be returned as-is. - local bypassed = ; - - if $(self.composing) - { - consumed = [ convert-multiple-sources-to-consumable-types $(project) - : $(property-set) : $(sources) ] ; - } - else - { - consumed = [ convert-to-consumable-types $(project) $(name) - : $(property-set) : $(sources) ] ; - } - - local result ; - if $(consumed) - { - result = [ construct-result $(consumed) : $(project) $(name) : - $(property-set) ] ; - } - - if $(result) - { - generators.dout [ indent ] " SUCCESS: " $(result) ; - } - else - { - generators.dout [ indent ] " FAILURE" ; - } - generators.dout ; - return $(result) ; - } - - # Constructs the dependency graph to be returned by this generator. - # - rule construct-result - ( - consumed + # Already prepared list of consumable targets. - # Composing generators may receive multiple sources - # all of which will have types matching those in - # $(self.source-types). Non-composing generators with - # multiple $(self.source-types) will receive exactly - # len $(self.source-types) sources with types matching - # those in $(self.source-types). And non-composing - # generators with only a single source type may - # receive multiple sources with all of them of the - # type listed in $(self.source-types). - : project name ? - : property-set # Properties to be used for all actions created here. - ) - { - local result ; - # If this is a 1->1 transformation, apply it to all consumed targets in - # order. - if ! $(self.source-types[2]) && ! $(self.composing) - { - for local r in $(consumed) - { - result += [ generated-targets $(r) : $(property-set) : - $(project) $(name) ] ; - } - } - else if $(consumed) - { - result += [ generated-targets $(consumed) : $(property-set) : - $(project) $(name) ] ; - } - return $(result) ; - } - - # Determine target name from fullname (maybe including path components) - # Place optional prefix and postfix around basename - # - rule determine-target-name ( fullname : prefix ? : postfix ? ) - { - # See if we need to add directory to the target name. - local dir = $(fullname:D) ; - local name = $(fullname:B) ; - - name = $(prefix:E=)$(name) ; - name = $(name)$(postfix:E=) ; - - if $(dir) - # Never append '..' to target path. - && ! [ MATCH .*(\\.\\.).* : $(dir) ] - && ! [ path.is-rooted $(dir) ] - { - # Relative path is always relative to the source directory. Retain - # it, so that users can have files with the same name in two - # different subdirectories. - name = $(dir)/$(name) ; - } - return $(name) ; - } - - # Determine the name of the produced target from the names of the sources. - # - rule determine-output-name ( sources + ) - { - # The simple case if when a name of source has single dot. Then, we take - # the part before dot. Several dots can be caused by: - # - using source file like a.host.cpp, or - # - a type whose suffix has a dot. Say, we can type 'host_cpp' with - # extension 'host.cpp'. - # In the first case, we want to take the part up to the last dot. In the - # second case -- not sure, but for now take the part up to the last dot - # too. - name = [ utility.basename [ $(sources[1]).name ] ] ; - for local s in $(sources[2-]) - { - if [ utility.basename [ $(s).name ] ] != $(name) - { - import errors : error : errors.error ; - errors.error "$(self.id): source targets have different names: cannot determine target name" ; - } - } - return [ determine-target-name [ $(sources[1]).name ] ] ; - } - - # Constructs targets that are created after consuming 'sources'. The result - # will be the list of virtual-target, which has the same length as the - # 'target-types' attribute and with corresponding types. - # - # When 'name' is empty, all source targets must have the same 'name' - # attribute value, which will be used instead of the 'name' argument. - # - # The 'name' attribute value for each generated target will be equal to the - # 'name' parameter if there is no name pattern for this type. Otherwise, the - # '%' symbol in the name pattern will be replaced with the 'name' parameter - # to obtain the 'name' attribute. - # - # For example, if targets types are T1 and T2 (with name pattern "%_x"), - # suffixes for T1 and T2 are .t1 and .t2, and source is foo.z, then created - # files would be "foo.t1" and "foo_x.t2". The 'name' attribute actually - # determines the basename of a file. - # - # Note that this pattern mechanism has nothing to do with implicit patterns - # in make. It is a way to produce a target whose name is different than the - # name of its source. - # - rule generated-targets ( sources + : property-set : project name ? ) - { - if ! $(name) - { - name = [ determine-output-name $(sources) ] ; - } - - # Assign an action for each target. - local action = [ action-class ] ; - local a = [ class.new $(action) $(sources) : $(self.rule-name) : - $(property-set) ] ; - - # Create generated target for each target type. - local targets ; - local pre = $(self.name-prefix) ; - local post = $(self.name-postfix) ; - for local t in $(self.target-types) - { - local generated-name = $(pre[1])$(name:BS)$(post[1]) ; - generated-name = $(generated-name:R=$(name:D)) ; - pre = $(pre[2-]) ; - post = $(post[2-]) ; - - targets += [ class.new file-target $(generated-name) : $(t) : - $(project) : $(a) ] ; - } - - return [ sequence.transform virtual-target.register : $(targets) ] ; - } - - # Attempts to convert 'sources' to targets of types that this generator can - # handle. The intention is to produce the set of targets that can be used - # when the generator is run. - # - rule convert-to-consumable-types - ( - project name ? - : property-set - : sources + - : only-one ? # Convert 'source' to only one of the source types. If - # there is more that one possibility, report an error. - ) - { - local _consumed ; - local missing-types ; - - if $(sources[2]) - { - # Do not know how to handle several sources yet. Just try to pass - # the request to other generator. - missing-types = $(self.source-types) ; - } - else - { - local temp = [ consume-directly $(sources) ] ; - if $(temp[1]) - { - _consumed = $(temp[1]) ; - } - missing-types = $(temp[2-]) ; - } - - # No need to search for transformation if some source type has consumed - # source and no more source types are needed. - if $(only-one) && $(_consumed) - { - missing-types = ; - } - - # TODO: we should check that only one source type is created if - # 'only-one' is true. - - if $(missing-types) - { - local transformed = [ generators.construct-types $(project) $(name) - : $(missing-types) : $(property-set) : $(sources) ] ; - - # Add targets of right type to 'consumed'. Add others to 'bypassed'. - # The 'generators.construct' rule has done its best to convert - # everything to the required type. There is no need to rerun it on - # targets of different types. - - # NOTE: ignoring usage requirements. - for local t in $(transformed[2-]) - { - if [ $(t).type ] in $(missing-types) - { - _consumed += $(t) ; - } - } - } - - return [ sequence.unique $(_consumed) ] ; - } - - # Converts several files to consumable types. Called for composing - # generators only. - # - rule convert-multiple-sources-to-consumable-types ( project : property-set : - sources * ) - { - local result ; - # We process each source one-by-one, trying to convert it to a usable - # type. - if ! $(self.source-types) - { - # Anything is acceptible - return $(sources) ; - } - else - { - local acceptible-types = [ sequence.unique - [ sequence.transform type.all-derived : $(self.source-types) ] ] ; - for local source in $(sources) - { - if ! [ $(source).type ] in $(acceptible-types) - { - local transformed = [ generators.construct-types $(project) - : $(self.source-types) : $(property-set) : $(source) ] ; - for local t in $(transformed[2-]) - { - if [ $(t).type ] in $(self.source-types) - { - result += $(t) ; - } - } - if ! $(transformed) - { - generators.dout [ indent ] " failed to convert " $(source) ; - } - } - else - { - result += $(source) ; - } - } - return [ sequence.unique $(result) ] ; - } - } - - rule consume-directly ( source ) - { - local real-source-type = [ $(source).type ] ; - - # If there are no source types, we can consume anything. - local source-types = $(self.source-types) ; - source-types ?= $(real-source-type) ; - - local result = "" ; - local missing-types ; - - for local st in $(source-types) - { - # The 'source' if of the right type already. - if $(real-source-type) = $(st) || [ type.is-derived - $(real-source-type) $(st) ] - { - result = $(source) ; - } - else - { - missing-types += $(st) ; - } - } - return $(result) $(missing-types) ; - } - - # Returns the class to be used to actions. Default implementation returns - # "action". - # - rule action-class ( ) - { - return "action" ; - } -} - - -# Registers a new generator instance 'g'. -# -rule register ( g ) -{ - .all-generators += $(g) ; - - # A generator can produce several targets of the same type. We want unique - # occurrence of that generator in .generators.$(t) in that case, otherwise, - # it will be tried twice and we will get a false ambiguity. - for local t in [ sequence.unique [ $(g).target-types ] ] - { - .generators.$(t) += $(g) ; - } - - # Update the set of generators for toolset. - - # TODO: should we check that generator with this id is not already - # registered. For example, the fop.jam module intentionally declared two - # generators with the same id, so such check will break it. - local id = [ $(g).id ] ; - - # Some generators have multiple periods in their name, so a simple $(id:S=) - # will not generate the right toolset name. E.g. if id = gcc.compile.c++, - # then .generators-for-toolset.$(id:S=) will append to - # .generators-for-toolset.gcc.compile, which is a separate value from - # .generators-for-toolset.gcc. Correcting this makes generator inheritance - # work properly. See also inherit-generators in the toolset module. - local base = $(id) ; - while $(base:S) - { - base = $(base:B) ; - } - .generators-for-toolset.$(base) += $(g) ; - - - # After adding a new generator that can construct new target types, we need - # to clear the related cached viable source target type information for - # constructing a specific target type or using a specific generator. Cached - # viable source target type lists affected by this are those containing any - # of the target types constructed by the new generator or any of their base - # target types. - # - # A more advanced alternative to clearing that cached viable source target - # type information would be to expand it with additional source types or - # even better - mark it as needing to be expanded on next use. - # - # Also see the http://thread.gmane.org/gmane.comp.lib.boost.build/19077 - # mailing list thread for an even more advanced idea of how we could convert - # Boost Build's Jamfile processing, target selection and generator selection - # into separate steps which would prevent these caches from ever being - # invalidated. - # - # For now we just clear all the cached viable source target type information - # that does not simply state 'all types' and may implement a more detailed - # algorithm later on if it becomes needed. - - invalidate-extendable-viable-source-target-type-cache ; -} - - -# Creates a new non-composing 'generator' class instance and registers it. -# Returns the created instance. Rationale: the instance is returned so that it -# is possible to first register a generator and then call its 'run' method, -# bypassing the whole generator selection process. -# -rule register-standard ( id : source-types * : target-types + : requirements * ) -{ - local g = [ new generator $(id) : $(source-types) : $(target-types) : - $(requirements) ] ; - register $(g) ; - return $(g) ; -} - - -# Creates a new composing 'generator' class instance and registers it. -# -rule register-composing ( id : source-types * : target-types + : requirements * - ) -{ - local g = [ new generator $(id) true : $(source-types) : $(target-types) : - $(requirements) ] ; - register $(g) ; - return $(g) ; -} - - -# Returns all generators belonging to the given 'toolset', i.e. whose ids are -# '$(toolset).<something>'. -# -rule generators-for-toolset ( toolset ) -{ - return $(.generators-for-toolset.$(toolset)) ; -} - - -# Make generator 'overrider-id' be preferred to 'overridee-id'. If, when -# searching for generators that could produce a target of a certain type, both -# those generators are among viable generators, the overridden generator is -# immediately discarded. -# -# The overridden generators are discarded immediately after computing the list -# of viable generators but before running any of them. -# -rule override ( overrider-id : overridee-id ) -{ - .override.$(overrider-id) += $(overridee-id) ; -} - - -# Returns a list of source type which can possibly be converted to 'target-type' -# by some chain of generator invocation. -# -# More formally, takes all generators for 'target-type' and returns a union of -# source types for those generators and result of calling itself recursively on -# source types. -# -# Returns '*' in case any type should be considered a viable source type for the -# given type. -# -local rule viable-source-types-real ( target-type ) -{ - local result ; - - # 't0' is the initial list of target types we need to process to get a list - # of their viable source target types. New target types will not be added to - # this list. - local t0 = [ type.all-bases $(target-type) ] ; - - # 't' is the list of target types which have not yet been processed to get a - # list of their viable source target types. This list will get expanded as - # we locate more target types to process. - local t = $(t0) ; - - while $(t) - { - # Find all generators for the current type. Unlike - # 'find-viable-generators' we do not care about the property-set. - local generators = $(.generators.$(t[1])) ; - t = $(t[2-]) ; - - while $(generators) - { - local g = $(generators[1]) ; - generators = $(generators[2-]) ; - - if ! [ $(g).source-types ] - { - # Empty source types -- everything can be accepted. - result = * ; - # This will terminate this loop. - generators = ; - # This will terminate the outer loop. - t = ; - } - - for local source-type in [ $(g).source-types ] - { - if ! $(source-type) in $(result) - { - # If a generator accepts a 'source-type' it will also - # happily accept any type derived from it. - for local n in [ type.all-derived $(source-type) ] - { - if ! $(n) in $(result) - { - # Here there is no point in adding target types to - # the list of types to process in case they are or - # have already been on that list. We optimize this - # check by realizing that we only need to avoid the - # original target type's base types. Other target - # types that are or have been on the list of target - # types to process have been added to the 'result' - # list as well and have thus already been eliminated - # by the previous if. - if ! $(n) in $(t0) - { - t += $(n) ; - } - result += $(n) ; - } - } - } - } - } - } - - return $(result) ; -} - - -# Helper rule, caches the result of 'viable-source-types-real'. -# -rule viable-source-types ( target-type ) -{ - local key = .vst.$(target-type) ; - if ! $($(key)) - { - .vst-cached-types += $(target-type) ; - local v = [ viable-source-types-real $(target-type) ] ; - if ! $(v) - { - v = none ; - } - $(key) = $(v) ; - } - - if $($(key)) != none - { - return $($(key)) ; - } -} - - -# Returns the list of source types, which, when passed to 'run' method of -# 'generator', has some change of being eventually used (probably after -# conversion by other generators). -# -# Returns '*' in case any type should be considered a viable source type for the -# given generator. -# -rule viable-source-types-for-generator-real ( generator ) -{ - local source-types = [ $(generator).source-types ] ; - if ! $(source-types) - { - # If generator does not specify any source types, it might be a special - # generator like builtin.lib-generator which just relays to other - # generators. Return '*' to indicate that any source type is possibly - # OK, since we do not know for sure. - return * ; - } - else - { - local result ; - while $(source-types) - { - local s = $(source-types[1]) ; - source-types = $(source-types[2-]) ; - local viable-sources = [ generators.viable-source-types $(s) ] ; - if $(viable-sources) = * - { - result = * ; - source-types = ; # Terminate the loop. - } - else - { - result += [ type.all-derived $(s) ] $(viable-sources) ; - } - } - return [ sequence.unique $(result) ] ; - } -} - - -# Helper rule, caches the result of 'viable-source-types-for-generator'. -# -local rule viable-source-types-for-generator ( generator ) -{ - local key = .vstg.$(generator) ; - if ! $($(key)) - { - .vstg-cached-generators += $(generator) ; - local v = [ viable-source-types-for-generator-real $(generator) ] ; - if ! $(v) - { - v = none ; - } - $(key) = $(v) ; - } - - if $($(key)) != none - { - return $($(key)) ; - } -} - - -# Returns usage requirements + list of created targets. -# -local rule try-one-generator-really ( project name ? : generator : target-type - : property-set : sources * ) -{ - local targets = - [ $(generator).run $(project) $(name) : $(property-set) : $(sources) ] ; - - local usage-requirements ; - local success ; - - generators.dout [ indent ] returned $(targets) ; - - if $(targets) - { - success = true ; - - if [ class.is-a $(targets[1]) : property-set ] - { - usage-requirements = $(targets[1]) ; - targets = $(targets[2-]) ; - } - else - { - usage-requirements = [ property-set.empty ] ; - } - } - - generators.dout [ indent ] " generator" [ $(generator).id ] " spawned " ; - generators.dout [ indent ] " " $(targets) ; - if $(usage-requirements) - { - generators.dout [ indent ] " with usage requirements:" $(x) ; - } - - if $(success) - { - return $(usage-requirements) $(targets) ; - } -} - - -# Checks if generator invocation can be pruned, because it is guaranteed to -# fail. If so, quickly returns an empty list. Otherwise, calls -# try-one-generator-really. -# -local rule try-one-generator ( project name ? : generator : target-type - : property-set : sources * ) -{ - local source-types ; - for local s in $(sources) - { - source-types += [ $(s).type ] ; - } - local viable-source-types = [ viable-source-types-for-generator $(generator) - ] ; - - if $(source-types) && $(viable-source-types) != * && - ! [ set.intersection $(source-types) : $(viable-source-types) ] - { - local id = [ $(generator).id ] ; - generators.dout [ indent ] " ** generator '$(id)' pruned" ; - #generators.dout [ indent ] "source-types" '$(source-types)' ; - #generators.dout [ indent ] "viable-source-types" '$(viable-source-types)' ; - } - else - { - return [ try-one-generator-really $(project) $(name) : $(generator) : - $(target-type) : $(property-set) : $(sources) ] ; - } -} - - -rule construct-types ( project name ? : target-types + : property-set - : sources + ) -{ - local result ; - local usage-requirements = [ property-set.empty ] ; - for local t in $(target-types) - { - local r = [ construct $(project) $(name) : $(t) : $(property-set) : - $(sources) ] ; - if $(r) - { - usage-requirements = [ $(usage-requirements).add $(r[1]) ] ; - result += $(r[2-]) ; - } - } - # TODO: have to introduce parameter controlling if several types can be - # matched and add appropriate checks. - - # TODO: need to review the documentation for 'construct' to see if it should - # return $(source) even if nothing can be done with it. Currents docs seem - # to imply that, contrary to the behaviour. - if $(result) - { - return $(usage-requirements) $(result) ; - } - else - { - return $(usage-requirements) $(sources) ; - } -} - - -# Ensures all 'targets' have their type. If this is not so, exists with error. -# -local rule ensure-type ( targets * ) -{ - for local t in $(targets) - { - if ! [ $(t).type ] - { - import errors ; - errors.error "target" [ $(t).str ] "has no type" ; - } - } -} - - -# Returns generators which can be used to construct target of specified type -# with specified properties. Uses the following algorithm: -# - iterates over requested target-type and all its bases (in the order returned -# by type.all-bases). -# - for each type find all generators that generate that type and whose -# requirements are satisfied by properties. -# - if the set of generators is not empty, returns that set. -# -# Note: this algorithm explicitly ignores generators for base classes if there -# is at least one generator for the requested target-type. -# -local rule find-viable-generators-aux ( target-type : property-set ) -{ - # Select generators that can create the required target type. - local viable-generators = ; - - import type ; - local t = $(target-type) ; - - if $(.debug) - { - generators.dout [ indent ] find-viable-generators target-type= $(target-type) - property-set= [ $(property-set).as-path ] ; - generators.dout [ indent ] "trying type" $(target-type) ; - } - - local generators = $(.generators.$(target-type)) ; - if $(generators) - { - if $(.debug) - { - generators.dout [ indent ] "there are generators for this type" ; - } - } - else - { - local t = [ type.base $(target-type) ] ; - - # Get the list of generators for the requested type. If no generator is - # registered, try base type, and so on. - while $(t) - { - if $(.debug) - { - generators.dout [ indent ] "trying type" $(t) ; - } - if $(.generators.$(t)) - { - generators.dout [ indent ] "there are generators for this type" ; - generators = $(.generators.$(t)) ; - - # We are here because there were no generators found for - # target-type but there are some generators for its base type. - # We will try to use them, but they will produce targets of - # base type, not of 'target-type'. So, we clone the generators - # and modify the list of target types. - local generators2 ; - for local g in $(generators) - { - # generators.register adds a generator to the list of - # generators for toolsets, which is a bit strange, but - # should work. That list is only used when inheriting a - # toolset, which should have been done before running - # generators. - generators2 += [ $(g).clone-and-change-target-type $(t) : - $(target-type) ] ; - generators.register $(generators2[-1]) ; - } - generators = $(generators2) ; - t = ; - } - else - { - t = [ type.base $(t) ] ; - } - } - } - - for local g in $(generators) - { - if $(.debug) - { - generators.dout [ indent ] "trying generator" [ $(g).id ] "(" [ $(g).source-types ] -> [ $(g).target-types ] ")" ; - } - - if [ $(g).match-rank $(property-set) ] - { - if $(.debug) - { - generators.dout [ indent ] " is viable" ; - } - viable-generators += $(g) ; - } - } - - return $(viable-generators) ; -} - - -rule find-viable-generators ( target-type : property-set ) -{ - local key = $(target-type).$(property-set) ; - local l = $(.fv.$(key)) ; - if ! $(l) - { - l = [ find-viable-generators-aux $(target-type) : $(property-set) ] ; - if ! $(l) - { - l = none ; - } - .fv.$(key) = $(l) ; - } - - if $(l) = none - { - l = ; - } - - local viable-generators ; - for local g in $(l) - { - # Avoid trying the same generator twice on different levels. - if ! $(g) in $(.active-generators) - { - viable-generators += $(g) ; - } - else - { - generators.dout [ indent ] " generator " [ $(g).id ] "is active, discaring" ; - } - } - - # Generators which override 'all'. - local all-overrides ; - # Generators which are overriden. - local overriden-ids ; - for local g in $(viable-generators) - { - local id = [ $(g).id ] ; - local this-overrides = $(.override.$(id)) ; - overriden-ids += $(this-overrides) ; - if all in $(this-overrides) - { - all-overrides += $(g) ; - } - } - if $(all-overrides) - { - viable-generators = $(all-overrides) ; - } - local result ; - for local g in $(viable-generators) - { - if ! [ $(g).id ] in $(overriden-ids) - { - result += $(g) ; - } - } - - return $(result) ; -} - - -.construct-stack = ; - - -# Attempts to construct a target by finding viable generators, running them and -# selecting the dependency graph. -# -local rule construct-really ( project name ? : target-type : property-set : - sources * ) -{ - viable-generators = [ find-viable-generators $(target-type) : - $(property-set) ] ; - - generators.dout [ indent ] "*** " [ sequence.length $(viable-generators) ] - " viable generators" ; - - local result ; - local generators-that-succeeded ; - for local g in $(viable-generators) - { - # This variable will be restored on exit from this scope. - local .active-generators = $(g) $(.active-generators) ; - - local r = [ try-one-generator $(project) $(name) : $(g) : $(target-type) - : $(property-set) : $(sources) ] ; - - if $(r) - { - generators-that-succeeded += $(g) ; - if $(result) - { - ECHO "Error: ambiguity found when searching for best transformation" ; - ECHO "Trying to produce type '$(target-type)' from: " ; - for local s in $(sources) - { - ECHO " - " [ $(s).str ] ; - } - ECHO "Generators that succeeded:" ; - for local g in $(generators-that-succeeded) - { - ECHO " - " [ $(g).id ] ; - } - ECHO "First generator produced: " ; - for local t in $(result[2-]) - { - ECHO " - " [ $(t).str ] ; - } - ECHO "Second generator produced: " ; - for local t in $(r[2-]) - { - ECHO " - " [ $(t).str ] ; - } - EXIT ; - } - else - { - result = $(r) ; - } - } - } - - return $(result) ; -} - - -# Attempts to create a target of 'target-type' with 'properties' from 'sources'. -# The 'sources' are treated as a collection of *possible* ingridients, i.e. -# there is no obligation to consume them all. -# -# Returns a list of targets. When this invocation is first instance of -# 'construct' in stack, returns only targets of requested 'target-type', -# otherwise, returns also unused sources and additionally generated targets. -# -# If 'top-level' is set, does not suppress generators that are already -# used in the stack. This may be useful in cases where a generator -# has to build a metatargets -- for example a target corresponding to -# built tool. -# -rule construct ( project name ? : target-type : property-set * : sources * : top-level ? ) -{ - local saved-stack ; - if $(top-level) - { - saved-active = $(.active-generators) ; - .active-generators = ; - } - - if (.construct-stack) - { - ensure-type $(sources) ; - } - - .construct-stack += 1 ; - - increase-indent ; - - if $(.debug) - { - generators.dout [ indent ] "*** construct" $(target-type) ; - - for local s in $(sources) - { - generators.dout [ indent ] " from" $(s) ; - } - generators.dout [ indent ] " properties:" [ $(property-set).raw ] ; - } - - local result = [ construct-really $(project) $(name) : $(target-type) : - $(property-set) : $(sources) ] ; - - decrease-indent ; - - .construct-stack = $(.construct-stack[2-]) ; - - if $(top-level) - { - .active-generators = $(saved-active) ; - } - - return $(result) ; -} - -# Given 'result', obtained from some generator or generators.construct, adds -# 'raw-properties' as usage requirements to it. If result already contains usage -# requirements -- that is the first element of result of an instance of the -# property-set class, the existing usage requirements and 'raw-properties' are -# combined. -# -rule add-usage-requirements ( result * : raw-properties * ) -{ - if $(result) - { - if [ class.is-a $(result[1]) : property-set ] - { - return [ $(result[1]).add-raw $(raw-properties) ] $(result[2-]) ; - } - else - { - return [ property-set.create $(raw-properties) ] $(result) ; - } - } -} - -rule dump ( ) -{ - for local g in $(.all-generators) - { - ECHO [ $(g).id ] ":" [ $(g).source-types ] -> [ $(g).target-types ] ; - } -} -
