Hello community, here is the log from the commit of package geoipupdate for openSUSE:Factory checked in at 2020-12-17 17:04:42 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/geoipupdate (Old) and /work/SRC/openSUSE:Factory/.geoipupdate.new.5145 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "geoipupdate" Thu Dec 17 17:04:42 2020 rev:15 rq:856477 version:4.6.0 Changes: -------- --- /work/SRC/openSUSE:Factory/geoipupdate/geoipupdate.changes 2020-11-05 21:55:33.080099001 +0100 +++ /work/SRC/openSUSE:Factory/.geoipupdate.new.5145/geoipupdate.changes 2020-12-17 17:08:41.969934795 +0100 @@ -1,0 +2,8 @@ +Wed Dec 16 12:19:29 UTC 2020 - Paolo Stivanin <[email protected]> + +- Update to 4.6.0 + * Show version number in verbose output. + * Retry downloads in more scenarios. Previously we would not retry + failures occurring when reading the response body, but now we do. + +------------------------------------------------------------------- Old: ---- geoipupdate-4.5.0.tar.gz New: ---- geoipupdate-4.6.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ geoipupdate.spec ++++++ --- /var/tmp/diff_new_pack.OIhbCt/_old 2020-12-17 17:08:43.805936617 +0100 +++ /var/tmp/diff_new_pack.OIhbCt/_new 2020-12-17 17:08:43.809936622 +0100 @@ -18,13 +18,14 @@ # Common info Name: geoipupdate -Version: 4.5.0 +Version: 4.6.0 Release: 0 Summary: GeoIP update client code License: Apache-2.0 OR MIT Group: Productivity/Networking/Other URL: https://github.com/maxmind/geoipupdate Source0: %{name}-%{version}.tar.gz +# go mod vendor && tar cf vendor.tar.gz vendor/ Source1: vendor.tar.gz Source2: geoipupdate.timer Source3: geoipupdate.service ++++++ _service ++++++ --- /var/tmp/diff_new_pack.OIhbCt/_old 2020-12-17 17:08:43.849936662 +0100 +++ /var/tmp/diff_new_pack.OIhbCt/_new 2020-12-17 17:08:43.853936665 +0100 @@ -2,7 +2,7 @@ <service name="tar_scm" mode="disabled"> <param name="scm">git</param> <param name="url">git://github.com/maxmind/geoipupdate</param> - <param name="revision">v4.5.0</param> + <param name="revision">v4.6.0</param> <param name="exclude">.git</param> <param name="versionformat">@PARENT_TAG@</param> <param name="changesgenerate">enable</param> ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.OIhbCt/_old 2020-12-17 17:08:43.869936681 +0100 +++ /var/tmp/diff_new_pack.OIhbCt/_new 2020-12-17 17:08:43.869936681 +0100 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">git://github.com/maxmind/geoipupdate</param> - <param name="changesrevision">65d57e3edea7798cd72adaafca50e96f34a61d59</param> + <param name="changesrevision">7b5009fa4cd889997783005cea56180b4b85974e</param> </service> </servicedata> ++++++ geoipupdate-4.5.0.tar.gz -> geoipupdate-4.6.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/geoipupdate-4.5.0/CHANGELOG.md new/geoipupdate-4.6.0/CHANGELOG.md --- old/geoipupdate-4.5.0/CHANGELOG.md 2020-10-28 22:03:38.000000000 +0100 +++ new/geoipupdate-4.6.0/CHANGELOG.md 2020-12-14 17:54:50.000000000 +0100 @@ -1,5 +1,11 @@ # CHANGELOG +## 4.6.0 (2020-12-14) + +* Show version number in verbose output. +* Retry downloads in more scenarios. Previously we would not retry failures + occurring when reading the response body, but now we do. + ## 4.5.0 (2020-10-28) * We no longer use a third party library for exponential backoff. This diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/geoipupdate-4.5.0/cmd/geoipupdate/main.go new/geoipupdate-4.6.0/cmd/geoipupdate/main.go --- old/geoipupdate-4.5.0/cmd/geoipupdate/main.go 2020-10-28 22:03:38.000000000 +0100 +++ new/geoipupdate-4.6.0/cmd/geoipupdate/main.go 2020-12-14 17:54:50.000000000 +0100 @@ -45,6 +45,7 @@ } if config.Verbose { + log.Printf("geoipupdate version %s", version) log.Printf("Using config file %s", args.ConfigFile) log.Printf("Using database directory %s", config.DatabaseDirectory) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/geoipupdate-4.5.0/pkg/geoipupdate/database/http_reader.go new/geoipupdate-4.6.0/pkg/geoipupdate/database/http_reader.go --- old/geoipupdate-4.5.0/pkg/geoipupdate/database/http_reader.go 2020-10-28 22:03:38.000000000 +0100 +++ new/geoipupdate-4.6.0/pkg/geoipupdate/database/http_reader.go 2020-12-14 17:54:50.000000000 +0100 @@ -10,6 +10,7 @@ "log" "net/http" "net/url" + "os" "time" "github.com/maxmind/geoipupdate/v4/pkg/geoipupdate" @@ -52,25 +53,107 @@ } }() - maxMindURL := fmt.Sprintf( + updateURL := fmt.Sprintf( "%s/geoip/databases/%s/update?db_md5=%s", reader.url, url.PathEscape(editionID), url.QueryEscape(destination.GetHash()), ) - req, err := http.NewRequest(http.MethodGet, maxMindURL, nil) // nolint: noctx + if reader.verbose { + log.Printf("Performing update request to %s", updateURL) + } + + var modified bool + // It'd be nice to not use a temporary file here. However the Writer API does + // not currently support multiple attempts to download the file (it assumes + // we'll begin writing once). Using a temporary file here should be a less + // disruptive alternative to changing the API. If we change that API in the + // future, adding something like Reset() may be desirable. + tempFile, err := ioutil.TempFile("", "geoipupdate") if err != nil { - return errors.Wrap(err, "error creating request") + return errors.Wrap(err, "error opening temporary file") + } + defer func() { + if err := tempFile.Close(); err != nil { + log.Printf("error closing temporary file: %s", err) + } + if err := os.Remove(tempFile.Name()); err != nil { + log.Printf("error removing temporary file: %s", err) + } + }() + var newMD5 string + var modificationTime time.Time + err = internal.RetryWithBackoff( + func() error { + newMD5, modificationTime, modified, err = reader.download( + updateURL, + editionID, + tempFile, + ) + return err + }, + reader.retryFor, + ) + if err != nil { + return err } - req.SetBasicAuth(fmt.Sprintf("%d", reader.accountID), reader.licenseKey) - if reader.verbose { - log.Printf("Performing update request to %s", maxMindURL) + if !modified { + return nil + } + + if _, err := tempFile.Seek(0, 0); err != nil { + return errors.Wrap(err, "error seeking") } - response, err := internal.MaybeRetryRequest(reader.client, reader.retryFor, req) + + if _, err = io.Copy(destination, tempFile); err != nil { + return errors.Wrap(err, "error writing response") + } + + if err := destination.ValidHash(newMD5); err != nil { + return err + } + + if err := destination.Commit(); err != nil { + return errors.Wrap(err, "encountered an issue committing database update") + } + + if reader.preserveFileTimes { + err = destination.SetFileModificationTime(modificationTime) + if err != nil { + return errors.Wrap(err, "unable to set modification time") + } + } + + return nil +} + +func (reader *HTTPDatabaseReader) download( + updateURL, + editionID string, + tempFile *os.File, +) (string, time.Time, bool, error) { + // Prepare a clean slate for this download attempt. + + if err := tempFile.Truncate(0); err != nil { + return "", time.Time{}, false, errors.Wrap(err, "error truncating") + } + if _, err := tempFile.Seek(0, 0); err != nil { + return "", time.Time{}, false, errors.Wrap(err, "error seeking") + } + + // Perform the download. + + req, err := http.NewRequest(http.MethodGet, updateURL, nil) // nolint: noctx + if err != nil { + return "", time.Time{}, false, errors.Wrap(err, "error creating request") + } + req.SetBasicAuth(fmt.Sprintf("%d", reader.accountID), reader.licenseKey) + + response, err := reader.client.Do(req) if err != nil { - return errors.Wrap(err, "error performing HTTP request") + return "", time.Time{}, false, errors.Wrap(err, "error performing HTTP request") } defer func() { if err := response.Body.Close(); err != nil { @@ -82,20 +165,20 @@ if reader.verbose { log.Printf("No new updates available for %s", editionID) } - return nil + return "", time.Time{}, false, nil } if response.StatusCode != http.StatusOK { buf, err := ioutil.ReadAll(io.LimitReader(response.Body, 256)) if err == nil { - return errors.Errorf("unexpected HTTP status code: %s: %s", response.Status, buf) + return "", time.Time{}, false, errors.Errorf("unexpected HTTP status code: %s: %s", response.Status, buf) } - return errors.Errorf("unexpected HTTP status code: %s", response.Status) + return "", time.Time{}, false, errors.Errorf("unexpected HTTP status code: %s", response.Status) } gzReader, err := gzip.NewReader(response.Body) if err != nil { - return errors.Wrap(err, "encountered an error creating GZIP reader") + return "", time.Time{}, false, errors.Wrap(err, "encountered an error creating GZIP reader") } defer func() { if err := gzReader.Close(); err != nil { @@ -103,34 +186,21 @@ } }() - if _, err = io.Copy(destination, gzReader); err != nil { //nolint:gosec - return errors.Wrap(err, "error writing response") + if _, err := io.Copy(tempFile, gzReader); err != nil { //nolint:gosec + return "", time.Time{}, false, errors.Wrap(err, "error writing response") } newMD5 := response.Header.Get("X-Database-MD5") if newMD5 == "" { - return errors.New("no X-Database-MD5 header found") - } - if err := destination.ValidHash(newMD5); err != nil { - return err - } - - if err := destination.Commit(); err != nil { - return errors.Wrap(err, "encountered an issue committing database update") + return "", time.Time{}, false, errors.New("no X-Database-MD5 header found") } - if reader.preserveFileTimes { - modificationTime, err := lastModified(response.Header.Get("Last-Modified")) - if err != nil { - return errors.Wrap(err, "unable to get last modified time") - } - err = destination.SetFileModificationTime(modificationTime) - if err != nil { - return errors.Wrap(err, "unable to set modification time") - } + modificationTime, err := lastModified(response.Header.Get("Last-Modified")) + if err != nil { + return "", time.Time{}, false, errors.Wrap(err, "unable to get last modified time") } - return nil + return newMD5, modificationTime, true, nil } // LastModified retrieves the date that the MaxMind database was last modified. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/geoipupdate-4.5.0/pkg/geoipupdate/geoip_updater.go new/geoipupdate-4.6.0/pkg/geoipupdate/geoip_updater.go --- old/geoipupdate-4.5.0/pkg/geoipupdate/geoip_updater.go 2020-10-28 22:03:38.000000000 +0100 +++ new/geoipupdate-4.6.0/pkg/geoipupdate/geoip_updater.go 2020-12-14 17:54:50.000000000 +0100 @@ -42,36 +42,49 @@ if config.Verbose { log.Printf("Performing get filename request to %s", maxMindURL) } - req, err := http.NewRequest(http.MethodGet, maxMindURL, nil) // nolint: noctx - if err != nil { - return "", errors.Wrap(err, "error creating HTTP request") - } - res, err := internal.MaybeRetryRequest(client, config.RetryFor, req) - if err != nil { - return "", errors.Wrap(err, "error performing HTTP request") - } - defer func() { - if err := res.Body.Close(); err != nil { - log.Fatalf("error closing response body: %+v", errors.Wrap(err, "closing body")) - } - }() - buf, err := ioutil.ReadAll(io.LimitReader(res.Body, 256)) - if err != nil { - return "", errors.Wrap(err, "error reading response body") - } + var buf []byte + err := internal.RetryWithBackoff( + func() error { + req, err := http.NewRequest(http.MethodGet, maxMindURL, nil) // nolint: noctx + if err != nil { + return errors.Wrap(err, "error creating HTTP request") + } - if res.StatusCode != http.StatusOK { - return "", errors.Errorf("unexpected HTTP status code: %s: %s", res.Status, buf) - } + res, err := client.Do(req) + if err != nil { + return errors.Wrap(err, "error performing HTTP request") + } + defer func() { + if err := res.Body.Close(); err != nil { + log.Fatalf("error closing response body: %+v", errors.Wrap(err, "closing body")) + } + }() - if len(buf) == 0 { - return "", errors.New("response body is empty") - } + buf, err = ioutil.ReadAll(io.LimitReader(res.Body, 256)) + if err != nil { + return errors.Wrap(err, "error reading response body") + } + + if res.StatusCode != http.StatusOK { + return errors.Errorf("unexpected HTTP status code: %s: %s", res.Status, buf) + } - if bytes.Count(buf, []byte("\n")) > 0 || - bytes.Count(buf, []byte("\x00")) > 0 { - return "", errors.New("invalid characters in filename") + if len(buf) == 0 { + return errors.New("response body is empty") + } + + if bytes.Count(buf, []byte("\n")) > 0 || + bytes.Count(buf, []byte("\x00")) > 0 { + return errors.New("invalid characters in filename") + } + + return nil + }, + config.RetryFor, + ) + if err != nil { + return "", err } return string(buf), nil diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/geoipupdate-4.5.0/pkg/geoipupdate/internal/retry.go new/geoipupdate-4.6.0/pkg/geoipupdate/internal/retry.go --- old/geoipupdate-4.5.0/pkg/geoipupdate/internal/retry.go 2020-10-28 22:03:38.000000000 +0100 +++ new/geoipupdate-4.6.0/pkg/geoipupdate/internal/retry.go 2020-12-14 17:54:50.000000000 +0100 @@ -2,46 +2,31 @@ package internal import ( - "net/http" "time" - - "github.com/pkg/errors" ) -// MaybeRetryRequest is an internal implementation detail of this module. It -// shouldn't be used by users of the geoipupdate Go library. You can use the -// RetryFor field of geoipupdate.Config if you'd like to retry failed requests -// when using the library directly. -func MaybeRetryRequest(c *http.Client, retryFor time.Duration, req *http.Request) (*http.Response, error) { - if retryFor < 0 { - return nil, errors.New("negative retry duration") - } - if req.Body != nil { - return nil, errors.New("can't retry requests with bodies") - } - var resp *http.Response - var err error - +// RetryWithBackoff calls the provided function repeatedly until it succeeds or +// until the retry duration is up. +func RetryWithBackoff( + fn func() error, + retryFor time.Duration, +) error { start := time.Now() + for i := uint(0); ; i++ { - resp, err = c.Do(req) - if err == nil && resp.StatusCode < 500 { - break + err := fn() + if err == nil { + return nil } currentDuration := time.Since(start) waitDuration := 200 * time.Millisecond * (1 << i) + if currentDuration+waitDuration > retryFor { - break - } - if err == nil { - _ = resp.Body.Close() + return err } + time.Sleep(waitDuration) } - if err != nil { - return nil, errors.Wrap(err, "error performing http request") - } - return resp, nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/geoipupdate-4.5.0/pkg/geoipupdate/internal/retry_test.go new/geoipupdate-4.6.0/pkg/geoipupdate/internal/retry_test.go --- old/geoipupdate-4.5.0/pkg/geoipupdate/internal/retry_test.go 2020-10-28 22:03:38.000000000 +0100 +++ new/geoipupdate-4.6.0/pkg/geoipupdate/internal/retry_test.go 2020-12-14 17:54:50.000000000 +0100 @@ -1,71 +1,41 @@ package internal import ( - "net/http" - "net/http/httptest" "testing" "time" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func TestRetry(t *testing.T) { - { - n, resp, err := testRetry( // nolint: bodyclose - t, - func(n int, causeError func()) int { - causeError() - return http.StatusOK +func TestRetryWithBackoff(t *testing.T) { + t.Run("never succeeds", func(t *testing.T) { + var n int + err := RetryWithBackoff( + func() error { + n++ + return errors.New("foo") }, + 6*time.Second, ) assert.Equal(t, 5, n) assert.Error(t, err) - assert.Nil(t, resp) - } + }) - { - n, resp, err := testRetry( - t, - func(n int, causeError func()) int { + t.Run("succeeds after failures", func(t *testing.T) { + var n int + err := RetryWithBackoff( + func() error { + n++ if n < 3 { - causeError() + return errors.New("foo") } - return http.StatusOK + return nil }, + 6*time.Second, ) assert.Equal(t, 3, n) assert.NoError(t, err) - assert.NotNil(t, resp) - require.NoError(t, resp.Body.Close()) - } - - { - n, resp, err := testRetry( - t, - func(int, func()) int { return http.StatusInternalServerError }, - ) - assert.Equal(t, 5, n) - assert.NoError(t, err) - require.NoError(t, resp.Body.Close()) - } -} - -func testRetry(t *testing.T, cb func(int, func()) int) (int, *http.Response, error) { - var server *httptest.Server - requests := 0 - server = httptest.NewServer( - http.HandlerFunc( - func(rw http.ResponseWriter, r *http.Request) { - requests++ - rw.WriteHeader(cb(requests, func() { server.CloseClientConnections() })) - }, - ), - ) - - req, err := http.NewRequest(http.MethodGet, server.URL, nil) // nolint: noctx - require.NoError(t, err) - resp, err := MaybeRetryRequest(server.Client(), 6*time.Second, req) - server.Close() - return requests, resp, err + }) } ++++++ vendor.tar.gz ++++++ /work/SRC/openSUSE:Factory/geoipupdate/vendor.tar.gz /work/SRC/openSUSE:Factory/.geoipupdate.new.5145/vendor.tar.gz differ: char 1, line 1 _______________________________________________ openSUSE Commits mailing list -- [email protected] To unsubscribe, email [email protected] List Netiquette: https://en.opensuse.org/openSUSE:Mailing_list_netiquette List Archives: https://lists.opensuse.org/archives/list/[email protected]
