Repository: incubator-mynewt-newt Updated Branches: refs/heads/master 4e940c181 -> a895dc5cb
MYNEWT-495: updates to sync command - When repo is already cloned, try to update from remote - If -f is passed, create a modifications diff, and try harder to update current local (stash, clean, checkout, pull) Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/0196ba53 Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/0196ba53 Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/0196ba53 Branch: refs/heads/master Commit: 0196ba5338aa5efdd8110eb1012e17104e435616 Parents: 7ecee8b Author: Fabio Utzig <[email protected]> Authored: Fri Mar 3 09:49:52 2017 -0300 Committer: Fabio Utzig <[email protected]> Committed: Mon Mar 6 09:42:46 2017 -0300 ---------------------------------------------------------------------- newt/cli/project_cmds.go | 23 ++- newt/downloader/downloader.go | 110 ++++++++++--- newt/repo/repo.go | 157 +++++++++++++++++-- newt/vendor/mynewt.apache.org/newt/util/util.go | 1 - 4 files changed, 247 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/0196ba53/newt/cli/project_cmds.go ---------------------------------------------------------------------- diff --git a/newt/cli/project_cmds.go b/newt/cli/project_cmds.go index a8d93b2..89a6eae 100644 --- a/newt/cli/project_cmds.go +++ b/newt/cli/project_cmds.go @@ -20,6 +20,7 @@ package cli import ( + "fmt" "os" "sort" "strings" @@ -156,26 +157,32 @@ func syncRunCmd(cmd *cobra.Command, args []string) { NewtUsage(nil, err) } + var failedRepos []string for _, repo := range repos { var exists bool + var updated bool if repo.IsLocal() { continue } vers := ps.GetInstalledVersion(repo.Name()) if vers == nil { util.StatusMessage(util.VERBOSITY_DEFAULT, - "No installed version of %s found, skipping\n", + "No installed version of %s found, skipping\n\n", repo.Name()) } - if err, exists = repo.Sync(vers, newtutil.NewtForce); err != nil { - NewtUsage(nil, err) + exists, updated, err = repo.Sync(vers, newtutil.NewtForce) + if exists && !updated { + failedRepos = append(failedRepos, repo.Name()) } - - if exists && !newtutil.NewtForce { - util.StatusMessage(util.VERBOSITY_DEFAULT, - "Skipping resync of %s because directory exists. To "+ - "force resync, add the -f (force) option.\n", repo.Name()) + } + if len(failedRepos) > 0 { + var forceMsg string + if !newtutil.NewtForce { + forceMsg = " To force resync, add the -f (force) option." } + err = util.NewNewtError(fmt.Sprintf("Failed for repos: %v."+ + forceMsg, failedRepos)) + NewtUsage(nil, err) } } http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/0196ba53/newt/downloader/downloader.go ---------------------------------------------------------------------- diff --git a/newt/downloader/downloader.go b/newt/downloader/downloader.go index 0ab201e..e57ecfa 100644 --- a/newt/downloader/downloader.go +++ b/newt/downloader/downloader.go @@ -27,6 +27,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" log "github.com/Sirupsen/logrus" @@ -39,6 +40,10 @@ type Downloader interface { Branch() string SetBranch(branch string) DownloadRepo(branch string) (string, error) + CurrentBranch(path string) (string, error) + UpdateRepo(path string) error + CleanupRepo(path string, branchName string) error + LocalDiff(path string) ([]byte, error) } type GenericDownloader struct { @@ -66,42 +71,57 @@ type LocalDownloader struct { Path string } -func checkout(repoDir string, commit string) error { - // Retrieve the current directory so that we can get back to where we - // started after the download completes. - pwd, err := os.Getwd() +func executeGitCommand(dir string, cmd []string) ([]byte, error) { + wd, err := os.Getwd() if err != nil { - return util.NewNewtError(err.Error()) + return nil, util.NewNewtError(err.Error()) } gitPath, err := exec.LookPath("git") if err != nil { - return util.NewNewtError(fmt.Sprintf("Can't find git binary: %s\n", + return nil, util.NewNewtError(fmt.Sprintf("Can't find git binary: %s\n", err.Error())) } gitPath = filepath.ToSlash(gitPath) - if err := os.Chdir(repoDir); err != nil { - return util.NewNewtError(err.Error()) + if err := os.Chdir(dir); err != nil { + return nil, util.NewNewtError(err.Error()) } - // Checkout the specified commit. + defer os.Chdir(wd) + + gitCmd := []string{gitPath} + gitCmd = append(gitCmd, cmd...) + output, err := util.ShellCommand(gitCmd, nil) + if err != nil { + return nil, err + } + + return output, nil +} + +func checkout(repoDir string, commit string) error { cmd := []string{ - gitPath, "checkout", commit, } + _, err := executeGitCommand(repoDir, cmd) + return err +} - if o, err := util.ShellCommand(cmd, nil); err != nil { - return util.NewNewtError(string(o)) - } +func pull(repoDir string) error { + _, err := executeGitCommand(repoDir, []string{"pull"}) + return err +} - // Go back to original directory. - if err := os.Chdir(pwd); err != nil { - return util.NewNewtError(err.Error()) - } +func stash(repoDir string) error { + _, err := executeGitCommand(repoDir, []string{"stash"}) + return err +} - return nil +func clean(repoDir string) error { + _, err := executeGitCommand(repoDir, []string{"clean", "-f"}) + return err } func (gd *GenericDownloader) Branch() string { @@ -172,6 +192,39 @@ func (gd *GithubDownloader) FetchFile(name string, dest string) error { return nil } +func (gd *GithubDownloader) CurrentBranch(path string) (string, error) { + cmd := []string{"rev-parse", "--abbrev-ref", "HEAD"} + branch, err := executeGitCommand(path, cmd) + return strings.Trim(string(branch), "\r\n"), err +} + +func (gd *GithubDownloader) UpdateRepo(path string) error { + return pull(path) +} + +func (gd *GithubDownloader) CleanupRepo(path string, branchName string) error { + err := stash(path) + if err != nil { + return err + } + + err = clean(path) + if err != nil { + return err + } + + err = checkout(path, branchName) + if err != nil { + return err + } + + return pull(path) +} + +func (gd *GithubDownloader) LocalDiff(path string) ([]byte, error) { + return executeGitCommand(path, []string{"diff"}) +} + func (gd *GithubDownloader) DownloadRepo(commit string) (string, error) { // Get a temporary directory, and copy the repository into that directory. tmpdir, err := ioutil.TempDir("", "newt-repo") @@ -243,6 +296,27 @@ func (ld *LocalDownloader) FetchFile(name string, dest string) error { return nil } +func (ld *LocalDownloader) CurrentBranch(path string) (string, error) { + cmd := []string{"rev-parse", "--abbrev-ref", "HEAD"} + branch, err := executeGitCommand(path, cmd) + return strings.Trim(string(branch), "\r\n"), err +} + +// NOTE: intentionally always error... +func (ld *LocalDownloader) UpdateRepo(path string) error { + return util.NewNewtError(fmt.Sprintf("Can't pull from a local repo\n")) +} + +func (ld *LocalDownloader) CleanupRepo(path string, branchName string) error { + os.RemoveAll(path) + _, err := ld.DownloadRepo(branchName) + return err +} + +func (ld *LocalDownloader) LocalDiff(path string) ([]byte, error) { + return executeGitCommand(path, []string{"diff"}) +} + func (ld *LocalDownloader) DownloadRepo(commit string) (string, error) { // Get a temporary directory, and copy the repository into that directory. tmpdir, err := ioutil.TempDir("", "newt-repo") http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/0196ba53/newt/repo/repo.go ---------------------------------------------------------------------- diff --git a/newt/repo/repo.go b/newt/repo/repo.go index a2331f7..a4a916f 100644 --- a/newt/repo/repo.go +++ b/newt/repo/repo.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "strings" + "time" log "github.com/Sirupsen/logrus" "github.com/spf13/cast" @@ -347,6 +348,11 @@ func (r *Repo) repoFilePath() string { ".configs/" + r.name + "/" } +func (r *Repo) patchesFilePath() string { + return interfaces.GetProject().Path() + "/" + REPOS_DIR + + "/.patches/" +} + func (r *Repo) downloadRepo(branchName string) error { dl := r.downloader @@ -368,6 +374,66 @@ func (r *Repo) downloadRepo(branchName string) error { return nil } +func (r *Repo) checkExists() bool { + return util.NodeExist(r.Path()) +} + +func (r *Repo) updateRepo() error { + dl := r.downloader + err := dl.UpdateRepo(r.Path()) + if err != nil { + return util.NewNewtError(fmt.Sprintf("\tError updating\n")) + } + return nil +} + +func (r *Repo) cleanupRepo(branchName string) error { + dl := r.downloader + err := dl.CleanupRepo(r.Path(), branchName) + if err != nil { + return util.NewNewtError(fmt.Sprintf("\tError cleaning and updating\n")) + } + return nil +} + +func (r *Repo) saveLocalDiff() (string, error) { + dl := r.downloader + diff, err := dl.LocalDiff(r.Path()) + if err != nil { + return "", util.NewNewtError(fmt.Sprintf( + "Error creating diff for \"%s\" : %s", r.Name(), err.Error())) + } + + // NOTE: date was not a typo: https://golang.org/pkg/time/#Time.Format + timenow := time.Now().Format("20060102_150405") + filename := r.patchesFilePath() + r.Name() + "_" + timenow + ".diff" + + f, err := os.Create(filename) + if err != nil { + return "", util.NewNewtError(fmt.Sprintf( + "Error creating repo diff file \"%s\"", filename)) + } + defer f.Close() + + _, err = f.Write(diff) + if err != nil { + return "", util.NewNewtError(fmt.Sprintf( + "Error writing repo diff file \"%s\"", filename)) + } + + return filename, nil +} + +func (r *Repo) currentBranch() (string, error) { + dl := r.downloader + branch, err := dl.CurrentBranch(r.Path()) + if err != nil { + return "", util.NewNewtError(fmt.Sprintf("Error finding current branch for \"%s\" : %s", + r.Name(), err.Error())) + } + return branch, nil +} + func (r *Repo) checkForceInstall(force bool) (error, bool) { // Copy the git repo into /repos/, error'ing out if the repo already exists if util.NodeExist(r.Path()) { @@ -407,33 +473,82 @@ func (r *Repo) Install(force bool) (*Version, error) { return vers, nil } -func (r *Repo) Sync(vers *Version, force bool) (error, bool) { +func (r *Repo) Sync(vers *Version, force bool) (bool, bool, error) { var exists bool var err error + var currBranch string - if err, exists = r.checkForceInstall(force); err != nil { - return err, exists - } - if exists && !force { - return nil, exists - } + exists = r.checkExists() // Update the repo description if _, updated, err := r.UpdateDesc(); updated != true || err != nil { - return util.NewNewtError("Cannot update repository description."), exists + return exists, false, util.NewNewtError("Cannot update repository description.") } - branchName, vers, found := r.rdesc.MatchVersion(vers) + branchName, _, found := r.rdesc.MatchVersion(vers) if found == false { - return util.NewNewtError(fmt.Sprintf("Branch description for %s not found", - r.Name())), exists + return exists, false, util.NewNewtError(fmt.Sprintf( + "Branch description for %s not found", r.Name())) } - if err := r.downloadRepo(branchName); err != nil { - return err, exists + if exists { + // Here assuming that if the branch was changed by the user, + // the user must know what he's doing... + // but, if -f is passed let's just save the work and re-clone + currBranch, err = r.currentBranch() + if err != nil { + return exists, false, err + } else if currBranch != branchName { + msg := "Unexpected local branch for %s: \"%s\" != \"%s\"" + if force { + util.StatusMessage(util.VERBOSITY_VERBOSE, + msg, r.rdesc.name, currBranch, branchName) + } else { + err = util.NewNewtError( + fmt.Sprintf(msg, r.rdesc.name, currBranch, branchName)) + return exists, false, err + } + } + + // Don't try updating if on an invalid branch... + if currBranch == branchName { + util.StatusMessage(util.VERBOSITY_VERBOSE, "\tTrying to update repository... ") + err = r.updateRepo() + if err == nil { + util.StatusMessage(util.VERBOSITY_VERBOSE, " success!\n") + return exists, true, err + } else { + util.StatusMessage(util.VERBOSITY_VERBOSE, " failed!\n") + if !force { + return exists, false, err + } + } + } + + filename, err := r.saveLocalDiff() + if err != nil { + return exists, false, err + } + wd, _ := os.Getwd() + filename, _ = filepath.Rel(wd, filename) + + util.StatusMessage(util.VERBOSITY_DEFAULT, "Saved local diff: "+ + "\"%s\"\n", filename) + + err = r.cleanupRepo(branchName) + if err != nil { + return exists, false, err + } + + } else { + // fresh or updating was unsuccessfull and force was given... + err = r.downloadRepo(branchName) + if err != nil { + return exists, false, err + } } - return nil, exists + return exists, true, nil } func (r *Repo) UpdateDesc() ([]*Repo, bool, error) { @@ -443,7 +558,7 @@ func (r *Repo) UpdateDesc() ([]*Repo, bool, error) { return nil, false, nil } - util.StatusMessage(util.VERBOSITY_DEFAULT, "%s\n", r.Name()) + util.StatusMessage(util.VERBOSITY_VERBOSE, "[%s]:\n", r.Name()) if err = r.DownloadDesc(); err != nil { return nil, false, err @@ -463,8 +578,8 @@ func (r *Repo) UpdateDesc() ([]*Repo, bool, error) { func (r *Repo) DownloadDesc() error { dl := r.downloader - util.StatusMessage(util.VERBOSITY_VERBOSE, "Downloading "+ - "repository description for %s...\n", r.Name()) + util.StatusMessage(util.VERBOSITY_VERBOSE, "\tDownloading "+ + "repository description... ") // Configuration path cpath := r.repoFilePath() @@ -481,6 +596,14 @@ func (r *Repo) DownloadDesc() error { return err } + // also create a directory to save diffs for sync + cpath = r.patchesFilePath() + if util.NodeNotExist(cpath) { + if err := os.MkdirAll(cpath, REPO_DEFAULT_PERMS); err != nil { + return util.NewNewtError(err.Error()) + } + } + util.StatusMessage(util.VERBOSITY_VERBOSE, " success!\n") return nil http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/0196ba53/newt/vendor/mynewt.apache.org/newt/util/util.go ---------------------------------------------------------------------- diff --git a/newt/vendor/mynewt.apache.org/newt/util/util.go b/newt/vendor/mynewt.apache.org/newt/util/util.go index fa3e60f..083ee9c 100644 --- a/newt/vendor/mynewt.apache.org/newt/util/util.go +++ b/newt/vendor/mynewt.apache.org/newt/util/util.go @@ -300,7 +300,6 @@ func ShellCommandLimitDbgOutput( } o, err := cmd.CombinedOutput() - if maxDbgOutputChrs < 0 || len(o) <= maxDbgOutputChrs { dbgStr := string(o) log.Debugf("o=%s", dbgStr)
