This is an automated email from the ASF dual-hosted git repository. andrzej-kaczmarek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mynewt-newt.git
commit dce260d9c4a9a47ff55f6f3f6b48e97dd67fd297 Author: Andrzej Kaczmarek <[email protected]> AuthorDate: Sun Oct 18 19:37:07 2026 +0200 newt: Add support for local.project.yml Allow overriding project.yml settings locally, without modifying the file itself, e.g. to use a different branch or fork of a repository. local.project.yml is read from the project base directory after project.yml and is intended to be kept out of version control. The file uses the same syntax as project.yml and its settings are merged into the ones from project.yml: maps are merged per key, sequences are appended to and other values are replaced. A setting name can have one of the following suffixes to change this: <setting>.OVERWRITE overwrite the value instead of merging, the same way as in other YAML files <setting>.REMOVE remove listed items from a sequence <setting>.DELETE: true delete the setting This allows e.g. redefining or deleting a repository and adding or removing single entries of project.repositories.allowed and project.repositories.ignored. Assisted-by: Claude:claude-opus-5-5 --- docs/index.rst | 63 +++++++++++++++++ newt/config/config.go | 180 ++++++++++++++++++++++++++++++++++++++++++++++++ newt/project/project.go | 11 +++ newt/ycfg/ycfg.go | 61 ++++++++++++++++ 4 files changed, 315 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index a979b123..773d3fb0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -204,6 +204,69 @@ the build system to look in the base directory of repos/apache-mynewt-core for t - "@apache-mynewt-core/hw/hal" - "@apache-mynewt-core/sys/console/full" +Local Project Overrides +~~~~~~~~~~~~~~~~~~~~~~~ + +Settings from project.yml can be overridden locally, without modifying the file itself, by creating a +``local.project.yml`` file next to it. The file is intended to be kept out of version control (e.g. added to +``.gitignore``) and is always loaded after project.yml. When it is present, newt prints +``Using local project overrides from local.project.yml``. + +The file uses the same syntax as project.yml. By default, each setting is merged into the corresponding setting from +project.yml: + +* maps (e.g. ``repository.<name>``) are merged per key, so only the specified fields are added or replaced, +* sequences (e.g. ``project.repositories.allowed``) are appended to, +* other values are replaced. + +.. note:: + The setting name suffixes described below are intended only for local.project.yml. They are used to modify + settings from project.yml and are not meant to be used in project.yml itself. + +A setting that does not exist in project.yml is simply added. To change this behavior, one of the following suffixes +can be appended to a setting name: + +* ``<setting>.OVERWRITE`` - replaces the whole value instead of merging into it, the same way as in other YAML + files, +* ``<setting>.REMOVE`` - removes the listed items from a sequence; items are compared as plain strings, not as + regular expressions, +* ``<setting>.DELETE: true`` - deletes the setting. + +``REMOVE`` and a plain setting can be used together to remove items from a sequence and add new ones; items are +always removed first. ``OVERWRITE`` and ``DELETE`` cannot be combined with any other operation on the same setting. +Removing an item or deleting a setting that does not exist results in a warning. ``REMOVE`` and ``DELETE`` are only +supported in local.project.yml itself, not in files it imports with ``$import``. + +.. code-block:: yaml + + # Use a feature branch of apache-mynewt-core; other fields stay as in project.yml + repository.apache-mynewt-core: + vers: 0.0.0 + branch: my-feature + + # Fetch mcuboot from a different location; no fields are inherited from project.yml + repository.mcuboot.OVERWRITE: + type: git + vers: 0.0.0 + url: https://example.com/mcuboot.git + + # Do not use apache-mynewt-nffs + repository.apache-mynewt-nffs.DELETE: true + + # Replace apache-mynewt-nimble with a private repository on the allowed list + project.repositories.allowed.REMOVE: + - apache-mynewt-nimble + project.repositories.allowed: + - my-private-repo + + repository.my-private-repo: + type: git + vers: 0.0.0 + url: [email protected]:me/private.git + +Note that deleting a repository only removes its definition from project.yml. If another repository depends on it, +it is still added to the project; add it to ``project.repositories.ignored`` to exclude it completely. + Newt has the ability to autocomplete within ``bash``. The following instructions allow MAC users to enable autocomplete within ``bash``. diff --git a/newt/config/config.go b/newt/config/config.go index 93eeb055..1fe59ba2 100644 --- a/newt/config/config.go +++ b/newt/config/config.go @@ -234,3 +234,183 @@ func ReadFile(path string) (ycfg.YCfg, error) { return yc, nil } + +// Operations that can be appended to a key in a local overrides file, e.g. +// "repository.foo.DELETE". OVERWRITE is supported in all configuration +// files; REMOVE and DELETE only in local overrides files. +const ( + OP_OVERWRITE = "OVERWRITE" + OP_REMOVE = "REMOVE" + OP_DELETE = "DELETE" +) + +// localOps holds all operations specified for a single key in a local +// overrides file. +type localOps struct { + merge interface{} + overwrite interface{} + remove interface{} + delete interface{} + ops []string +} + +// splitOpKey splits a key into the base key and an operation suffix. The +// returned operation is empty if the key has no suffix. +func splitOpKey(key string) (string, string) { + idx := strings.LastIndex(key, ".") + if idx < 0 { + return key, "" + } + + switch op := key[idx+1:]; op { + case OP_OVERWRITE, OP_REMOVE, OP_DELETE: + return key[:idx], op + default: + return key, "" + } +} + +func applyLocalOps(yc *ycfg.YCfg, key string, lo *localOps, + fileInfo *util.FileInfo) error { + + path := newtutil.ProjRelPath(fileInfo.Path) + + if len(lo.ops) > 1 { + sort.Strings(lo.ops) + for _, op := range lo.ops { + if op == OP_OVERWRITE || op == OP_DELETE { + return util.FmtNewtError( + "%s: \"%s.%s\" can't be combined with other operations "+ + "on the same key", path, key, op) + } + } + } + + if lo.delete != nil { + del, err := cast.ToBoolE(lo.delete) + if err != nil { + return util.FmtNewtError("%s: invalid value of \"%s.%s\"; "+ + "must be a boolean", path, key, OP_DELETE) + } + if del && !yc.DeleteValue(key) { + util.OneTimeWarning("%s: can't delete \"%s\"; key not found", + path, key) + } + } + + if lo.overwrite != nil { + // Overwrite the value the same way as in other configuration files. + if err := yc.ReplaceFromFile(key+"."+OP_OVERWRITE, lo.overwrite, + fileInfo); err != nil { + + return err + } + } + + // Removal is done before merge so that items can be removed from and + // added to the same list. + if lo.remove != nil { + vals, ok := lo.remove.([]interface{}) + if !ok { + return util.FmtNewtError("%s: invalid value of \"%s.%s\"; "+ + "must be a sequence", path, key, OP_REMOVE) + } + notFound, err := yc.RemoveFromSlice(key, vals) + if err != nil { + return util.FmtNewtError("%s: %s", path, err.Error()) + } + for _, v := range notFound { + util.OneTimeWarning("%s: can't remove \"%v\" from \"%s\"; "+ + "item not found", path, v, key) + } + } + + if lo.merge != nil { + switch lo.merge.(type) { + case map[interface{}]interface{}, []interface{}: + return yc.MergeFromFile(key, lo.merge, fileInfo) + default: + return yc.ReplaceFromFile(key, lo.merge, fileInfo) + } + } + + return nil +} + +// ApplyLocalFile reads a local overrides file (e.g., `local.project.yml`) and +// applies it on top of an already populated YCfg tree. Files imported by the +// local file are merged as by ReadFile. Keys in the local file itself can +// have one of the following operation suffixes: +// +// <key>.OVERWRITE Overwrite the value instead of merging into it. +// <key>.REMOVE Remove the listed items from a sequence. +// <key>.DELETE Delete the value if set to true. +// +// A key without a suffix is merged into the existing value: maps are merged +// per key, sequences are appended to and other values are replaced. +func ApplyLocalFile(yc *ycfg.YCfg, path string) error { + entries, err := readLineage(path) + if err != nil { + return err + } + + // The local file itself is processed last, after its imports. + local := entries[len(entries)-1] + + for _, e := range entries[:len(entries)-1] { + for k, v := range e.Settings { + if _, op := splitOpKey(k); op == OP_REMOVE || op == OP_DELETE { + return e.FileInfo.ErrTree(util.FmtNewtError( + "\"%s\": %s and %s are only supported in %s", + k, OP_REMOVE, OP_DELETE, + newtutil.ProjRelPath(local.FileInfo.Path))) + } + if err := yc.MergeFromFile(k, v, e.FileInfo); err != nil { + return e.FileInfo.Parent.ErrTree(err) + } + } + } + + opsMap := map[string]*localOps{} + for k, v := range local.Settings { + key, op := splitOpKey(k) + + lo := opsMap[key] + if lo == nil { + lo = &localOps{} + opsMap[key] = lo + } + lo.ops = append(lo.ops, op) + + // The null value gets interpreted as an empty string during YAML + // parsing. + if s, ok := v.(string); ok && s == "" { + v = nil + } + + switch op { + case "": + lo.merge = v + case OP_OVERWRITE: + lo.overwrite = v + case OP_REMOVE: + lo.remove = v + case OP_DELETE: + lo.delete = v + } + } + + keys := make([]string, 0, len(opsMap)) + for k, _ := range opsMap { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + if err := applyLocalOps(yc, k, opsMap[k], local.FileInfo); err != nil { + return err + } + } + + return nil +} diff --git a/newt/project/project.go b/newt/project/project.go index e782edd3..2a04e63f 100644 --- a/newt/project/project.go +++ b/newt/project/project.go @@ -45,6 +45,7 @@ import ( var globalProject *Project = nil const PROJECT_FILE_NAME = "project.yml" +const PROJECT_LOCAL_FILE_NAME = "local.project.yml" const PATCHES_DIR = "patches" var ignoreSearchDirs []string = []string{ @@ -796,6 +797,16 @@ func (proj *Project) loadConfig(download bool) error { if err != nil { return util.NewNewtError(err.Error()) } + + localPath := proj.BasePath + "/" + PROJECT_LOCAL_FILE_NAME + if util.NodeExist(localPath) { + util.ErrorMessage(util.VERBOSITY_DEFAULT, + "Using local project overrides from %s\n", PROJECT_LOCAL_FILE_NAME) + if err := config.ApplyLocalFile(&yc, localPath); err != nil { + return util.NewNewtError(err.Error()) + } + } + // Store configuration object for access to future values, // this avoids keeping every string around as a project variable when // we need to process it later. diff --git a/newt/ycfg/ycfg.go b/newt/ycfg/ycfg.go index 50a7c147..53571175 100644 --- a/newt/ycfg/ycfg.go +++ b/newt/ycfg/ycfg.go @@ -672,6 +672,67 @@ func (yc *YCfg) Delete(key string) { delete(yc.tree, key) } +// DeleteValue clears the value of the node with the specified key. Child +// nodes are left intact; the node itself (and any ancestors left without a +// value or children) is removed from the tree. The returned bool is false if +// the node has no value. +func (yc *YCfg) DeleteValue(key string) bool { + node := yc.find(key) + if node == nil || node.Value == nil { + return false + } + + node.Value = nil + for node != nil && node.Value == nil && len(node.Children) == 0 { + if node.Parent == nil { + delete(yc.tree, node.Name) + } else { + delete(node.Parent.Children, node.Name) + } + node = node.Parent + } + + return true +} + +// RemoveFromSlice removes all items equal to any of the specified values from +// the slice stored in the node with the specified key. Items are compared by +// their string representation. The returned slice contains the values that +// were not found. +func (yc *YCfg) RemoveFromSlice(key string, vals []interface{}) ( + []interface{}, error) { + + node := yc.find(key) + if node == nil || node.Value == nil { + return vals, nil + } + + nodeVal, ok := node.Value.([]interface{}) + if !ok { + return nil, util.FmtNewtError( + "can't remove items from cfg node \"%s\"; node type: %T", + key, node.Value) + } + + notFound := []interface{}{} + for _, v := range vals { + s := cast.ToString(v) + filtered := make([]interface{}, 0, len(nodeVal)) + for _, item := range nodeVal { + if cast.ToString(item) != s { + filtered = append(filtered, item) + } + } + if len(filtered) == len(nodeVal) { + notFound = append(notFound, v) + } + nodeVal = filtered + } + node.Value = nodeVal + + return notFound, nil +} + // Clear removes all entries from the YCfg. func (yc *YCfg) Clear() { yc.tree = YCfgTree{}
