Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package minio-client for openSUSE:Factory checked in at 2023-12-04 23:00:49 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/minio-client (Old) and /work/SRC/openSUSE:Factory/.minio-client.new.25432 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "minio-client" Mon Dec 4 23:00:49 2023 rev:58 rq:1130615 version:20231202T020328Z Changes: -------- --- /work/SRC/openSUSE:Factory/minio-client/minio-client.changes 2023-11-21 21:32:36.224640725 +0100 +++ /work/SRC/openSUSE:Factory/.minio-client.new.25432/minio-client.changes 2023-12-04 23:01:12.634957945 +0100 @@ -1,0 +2,13 @@ +Sun Dec 03 18:52:47 UTC 2023 - ka...@b1-systems.de + +- Update to version 20231202T020328Z: + * do not auto-decompress objects with content-encoding (#4777) + * Update minio-go to incl ilm fix (#4779) + * chore update minio/pkg and madmin-go (#4774) + * set-json to fail on custom policies without .json (#4745) + * make STS support for 'mc' alias specific (#4771) + * ilm: Add support for size based filters (#4770) + * Option to enable disable ILM expiry for site replication + (#4698) + +------------------------------------------------------------------- Old: ---- mc-20231120T163059Z.obscpio New: ---- mc-20231202T020328Z.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ minio-client.spec ++++++ --- /var/tmp/diff_new_pack.UbA33u/_old 2023-12-04 23:01:14.627031296 +0100 +++ /var/tmp/diff_new_pack.UbA33u/_new 2023-12-04 23:01:14.631031443 +0100 @@ -22,7 +22,7 @@ %define binary_name minio-client Name: minio-client -Version: 20231120T163059Z +Version: 20231202T020328Z Release: 0 Summary: Client for MinIO License: AGPL-3.0-only ++++++ _service ++++++ --- /var/tmp/diff_new_pack.UbA33u/_old 2023-12-04 23:01:14.663032621 +0100 +++ /var/tmp/diff_new_pack.UbA33u/_new 2023-12-04 23:01:14.663032621 +0100 @@ -5,7 +5,7 @@ <param name="exclude">.git</param> <param name="changesgenerate">enable</param> <param name="versionformat">@PARENT_TAG@</param> - <param name="revision">RELEASE.2023-11-20T16-30-59Z</param> + <param name="revision">RELEASE.2023-12-02T02-03-28Z</param> <param name="match-tag">RELEASE.*</param> <param name="versionrewrite-pattern">RELEASE\.(.*)-(.*)-(.*)-(.*)-(.*)</param> <param name="versionrewrite-replacement">\1\2\3\4\5</param> @@ -19,7 +19,7 @@ <param name="compression">gz</param> </service> <service name="go_modules" mode="manual"> - <param name="archive">mc-20231120T163059Z.obscpio</param> + <param name="archive">mc-20231202T020328Z.obscpio</param> </service> </services> ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.UbA33u/_old 2023-12-04 23:01:14.687033505 +0100 +++ /var/tmp/diff_new_pack.UbA33u/_new 2023-12-04 23:01:14.687033505 +0100 @@ -1,6 +1,6 @@ <servicedata> <service name="tar_scm"> <param name="url">https://github.com/minio/mc</param> - <param name="changesrevision">937b34616f012ad30e1cd6fd61e1da25ff931648</param></service></servicedata> + <param name="changesrevision">f5f7147b9ec4cf78eb67f1cdc91b63d191852e6a</param></service></servicedata> (No newline at EOF) ++++++ mc-20231120T163059Z.obscpio -> mc-20231202T020328Z.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/access-perms.go new/mc-20231202T020328Z/cmd/access-perms.go --- old/mc-20231120T163059Z/cmd/access-perms.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/access-perms.go 2023-12-02 03:03:28.000000000 +0100 @@ -17,7 +17,12 @@ package cmd -import "path/filepath" +import ( + "os" + + json "github.com/minio/colorjson" + "github.com/minio/minio-go/v7/pkg/policy" +) // isValidAccessPERM - is provided access perm string supported. func (b accessPerms) isValidAccessPERM() bool { @@ -29,7 +34,32 @@ } func (b accessPerms) isValidAccessFile() bool { - return filepath.Ext(string(b)) == ".json" + file, err := os.Open(string(b)) + if err != nil { + fatalIf(errDummy().Trace(), "Unable to open access file.") + return false + } + defer file.Close() + + var policy policy.BucketAccessPolicy + if json.NewDecoder(file).Decode(&policy) != nil { + fatalIf(errDummy().Trace(), "Unable to parse access file.") + return false + } + + if policy.Version != "2012-10-17" { + fatalIf(errDummy().Trace(), "Invalid policy version. Only 2012-10-17 is supported.") + return false + } + + for _, statement := range policy.Statements { + if statement.Effect != "Allow" && statement.Effect != "Deny" { + fatalIf(errDummy().Trace(), "Invalid policy effect. Only Allow and Deny are supported.") + return false + } + } + + return true } // accessPerms - access level. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/admin-replicate-add.go new/mc-20231202T020328Z/cmd/admin-replicate-add.go --- old/mc-20231120T163059Z/cmd/admin-replicate-add.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/admin-replicate-add.go 2023-12-02 03:03:28.000000000 +0100 @@ -28,13 +28,20 @@ "github.com/minio/pkg/v2/console" ) +var adminReplicateAddFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "replicate-ilm-expiry", + Usage: "replicate ILM expiry rules", + }, +} + var adminReplicateAddCmd = cli.Command{ Name: "add", Usage: "add one or more sites for replication", Action: mainAdminReplicateAdd, OnUsageError: onUsageError, Before: setGlobalsFromContext, - Flags: globalFlags, + Flags: append(globalFlags, adminReplicateAddFlags...), CustomHelpTemplate: `NAME: {{.HelpName}} - {{.Usage}} @@ -48,6 +55,9 @@ EXAMPLES: 1. Add a site for cluster-level replication: {{.Prompt}} {{.HelpName}} minio1 minio2 + + 2. Add a site for cluster-level replication with replication of ILM expiry rules: + {{.Prompt}} {{.HelpName}} minio1 minio2 --replicate-ilm-expiry `, } @@ -106,7 +116,9 @@ }) } - res, e := client.SiteReplicationAdd(globalContext, ps, madmin.SRAddOptions{}) + var opts madmin.SRAddOptions + opts.ReplicateILMExpiry = ctx.Bool("replicate-ilm-expiry") + res, e := client.SiteReplicationAdd(globalContext, ps, opts) fatalIf(probe.NewError(e).Trace(args...), "Unable to add sites for replication") printMsg(successMessage(res)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/admin-replicate-info.go new/mc-20231202T020328Z/cmd/admin-replicate-info.go --- old/mc-20231120T163059Z/cmd/admin-replicate-info.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/admin-replicate-info.go 2023-12-02 03:03:28.000000000 +0100 @@ -19,6 +19,7 @@ import ( "fmt" + "strconv" "strings" humanize "github.com/dustin/go-humanize" @@ -75,7 +76,8 @@ Field{"Endpoint", 46}, Field{"Sync", 4}, Field{"Bandwidth", 10}, - ).buildRow("Deployment ID", "Site Name", "Endpoint", "Sync", "Bandwidth")) + Field{"ILM Expiry Replication", 25}, + ).buildRow("Deployment ID", "Site Name", "Endpoint", "Sync", "Bandwidth", "ILM Expiry Replication")) messages = append(messages, r) r = console.Colorize("THeaders", newPrettyTable(" | ", @@ -85,7 +87,8 @@ Field{"Endpoint", 46}, Field{"Sync", 4}, Field{"Bandwidth", 10}, - ).buildRow("", "", "", "", "Per Bucket")) + Field{"ILM Expiry Replication", 25}, + ).buildRow("", "", "", "", "Per Bucket", "")) messages = append(messages, r) for _, peer := range info.Sites { var chk string @@ -103,7 +106,8 @@ Field{"Endpoint", 46}, Field{"Sync", 4}, Field{"Bandwidth", 10}, - ).buildRow(peer.DeploymentID, peer.Name, peer.Endpoint, chk, limit)) + Field{"ILM Expiry Replication", 25}, + ).buildRow(peer.DeploymentID, peer.Name, peer.Endpoint, chk, limit, strconv.FormatBool(peer.ReplicateILMExpiry))) messages = append(messages, r) } } else { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/admin-replicate-status.go new/mc-20231202T020328Z/cmd/admin-replicate-status.go --- old/mc-20231120T163059Z/cmd/admin-replicate-status.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/admin-replicate-status.go 2023-12-02 03:03:28.000000000 +0100 @@ -51,6 +51,10 @@ Usage: "display only groups", }, cli.BoolFlag{ + Name: "ilm-expiry-rules", + Usage: "display only ilm expiry rules", + }, + cli.BoolFlag{ Name: "all", Usage: "display all available site replication status", }, @@ -70,6 +74,10 @@ Name: "group", Usage: "display group sync status", }, + cli.StringFlag{ + Name: "ilm-expiry-rule", + Usage: "display ILM expiry rule sync status", + }, } // Some cell values @@ -312,6 +320,42 @@ } } } + if i.opts.ILMExpiryRules { + messages = append(messages, + console.Colorize("SummaryHdr", "ILM Expiry Rules replication status:")) + switch { + case i.MaxILMExpiryRules == 0: + messages = append(messages, console.Colorize("Summary", "No ILM Expiry Rules present\n")) + default: + msg := console.Colorize(i.getTheme(len(info.ILMExpiryStats) == 0), fmt.Sprintf("%d/%d ILM Expiry Rules in sync", info.MaxILMExpiryRules-len(info.ILMExpiryStats), info.MaxILMExpiryRules)) + "\n" + messages = append(messages, fmt.Sprintf("%s %s", coloredDot, msg)) + if len(i.ILMExpiryStats) > 0 { + messages = append(messages, i.siteHeader(siteNames, "ILM Expiry Rules")) + } + var detailFields []Field + for b, ssMap := range i.ILMExpiryStats { + var details []string + details = append(details, b) + detailFields = append(detailFields, legendFields[0]) + for _, sname := range siteNames { + detailFields = append(detailFields, legendFields[0]) + dID := nameIDMap[sname] + ss := ssMap[dID] + switch { + case !ss.HasILMExpiryRules: + details = append(details, blankCell) + case ss.ILMExpiryRuleMismatch: + details = append(details, fmt.Sprintf("%s in-sync", crossTickCell)) + default: + details = append(details, fmt.Sprintf("%s in-sync", tickCell)) + } + } + messages = append(messages, newPrettyTable(" | ", + detailFields...).buildRow(details...)) + messages = append(messages, "") + } + } + } switch i.opts.Entity { case madmin.SRBucketEntity: @@ -322,7 +366,8 @@ messages = append(messages, i.getUserStatusSummary(siteNames, nameIDMap, "User")...) case madmin.SRGroupEntity: messages = append(messages, i.getGroupStatusSummary(siteNames, nameIDMap, "Group")...) - + case madmin.SRILMExpiryRuleEntity: + messages = append(messages, i.getILMExpiryStatusSummary(siteNames, nameIDMap, "ILMExpiryRule")...) } if i.opts.Metrics { uiFn := func(theme string) func(string) string { @@ -716,29 +761,85 @@ return messages } +func (i srStatus) getILMExpiryStatusSummary(siteNames []string, nameIDMap map[string]string, legend string) []string { + var messages []string + coloredDot := console.Colorize("Status", dot) + var found bool + for _, st := range i.SRStatusInfo.ILMExpiryStats[i.opts.EntityValue] { + if st.HasILMExpiryRules { + found = true + break + } + } + if !found { + messages = append(messages, console.Colorize("Summary", fmt.Sprintf("ILM Expiry Rule %s not found\n", i.opts.EntityValue))) + return messages + } + + rowLegend := []string{"ILM Expiry Rule"} + detailFields := make([][]Field, len(rowLegend)) + + var rules []string + detailFields[0] = make([]Field, len(siteNames)+1) + detailFields[0][0] = Field{"Entity", 15} + rules = append(rules, "ILM Expiry Rule") + rows := make([]string, len(rowLegend)) + for j, sname := range siteNames { + dID := nameIDMap[sname] + ss := i.SRStatusInfo.ILMExpiryStats[i.opts.EntityValue][dID] + var theme, msgStr string + for r := range rowLegend { + switch r { + case 0: + theme, msgStr = syncStatus(ss.ILMExpiryRuleMismatch, ss.HasILMExpiryRules) + rules = append(rules, msgStr) + detailFields[r][j+1] = Field{theme, fieldLen} + } + } + } + for r := range rowLegend { + switch r { + case 0: + rows[r] = newPrettyTable(" | ", + detailFields[r]...).buildRow(rules...) + } + } + messages = append(messages, + console.Colorize("SummaryHdr", fmt.Sprintf("%s %s\n", coloredDot, console.Colorize("Summary", "ILM Expiry Rule replication summary for: ")+console.Colorize("UserMessage", i.opts.EntityValue)))) + siteHdr := i.siteHeader(siteNames, legend) + messages = append(messages, siteHdr) + + messages = append(messages, rows...) + return messages +} + // Calculate srstatus options for command line flags func srStatusOpts(ctx *cli.Context) (opts madmin.SRStatusOptions) { if !(ctx.IsSet("buckets") || ctx.IsSet("users") || ctx.IsSet("groups") || ctx.IsSet("policies") || + ctx.IsSet("ilm-expiry-rules") || ctx.IsSet("bucket") || ctx.IsSet("user") || ctx.IsSet("group") || ctx.IsSet("policy") || + ctx.IsSet("ilm-expiry-rule") || ctx.IsSet("all")) || ctx.IsSet("all") { opts.Buckets = true opts.Users = true opts.Groups = true opts.Policies = true opts.Metrics = true + opts.ILMExpiryRules = true return } opts.Buckets = ctx.Bool("buckets") opts.Policies = ctx.Bool("policies") opts.Users = ctx.Bool("users") opts.Groups = ctx.Bool("groups") - for _, name := range []string{"bucket", "user", "group", "policy"} { + opts.ILMExpiryRules = ctx.Bool("ilm-expiry-rules") + for _, name := range []string{"bucket", "user", "group", "policy", "ilm-expiry-rule"} { if ctx.IsSet(name) { opts.Entity = madmin.GetSREntityType(name) opts.EntityValue = ctx.String(name) @@ -756,13 +857,13 @@ fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...), "Need exactly one alias argument.") } - groupStatus := ctx.IsSet("buckets") || ctx.IsSet("groups") || ctx.IsSet("users") || ctx.IsSet("policies") - indivStatus := ctx.IsSet("bucket") || ctx.IsSet("group") || ctx.IsSet("user") || ctx.IsSet("policy") + groupStatus := ctx.IsSet("buckets") || ctx.IsSet("groups") || ctx.IsSet("users") || ctx.IsSet("policies") || ctx.IsSet("ilm-expiry-rules") + indivStatus := ctx.IsSet("bucket") || ctx.IsSet("group") || ctx.IsSet("user") || ctx.IsSet("policy") || ctx.IsSet("ilm-expiry-rule") if groupStatus && indivStatus { fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...), - "Cannot specify both (bucket|group|policy|user) flag and one or more of buckets|groups|policies|users) flag(s)") + "Cannot specify both (bucket|group|policy|user|ilm-expiry-rule) flag and one or more of buckets|groups|policies|users|ilm-expiry-rules) flag(s)") } - setSlc := []bool{ctx.IsSet("bucket"), ctx.IsSet("user"), ctx.IsSet("group"), ctx.IsSet("policy")} + setSlc := []bool{ctx.IsSet("bucket"), ctx.IsSet("user"), ctx.IsSet("group"), ctx.IsSet("policy"), ctx.IsSet("ilm-expiry-rule")} count := 0 for _, s := range setSlc { if s { @@ -771,7 +872,7 @@ } if count > 1 { fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...), - "Cannot specify more than one of --bucket, --policy, --user, --group flags at the same time") + "Cannot specify more than one of --bucket, --policy, --user, --group, --ilm-expiry-rule flags at the same time") } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/admin-replicate-update.go new/mc-20231202T020328Z/cmd/admin-replicate-update.go --- old/mc-20231120T163059Z/cmd/admin-replicate-update.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/admin-replicate-update.go 2023-12-02 03:03:28.000000000 +0100 @@ -53,6 +53,14 @@ Name: "bucket-bandwidth", Usage: "Set default bandwidth limit for bucket in bits per second (K,B,G,T for metric and Ki,Bi,Gi,Ti for IEC units)", }, + cli.BoolFlag{ + Name: "disable-ilm-expiry-replication", + Usage: "disable ILM expiry rules replication", + }, + cli.BoolFlag{ + Name: "enable-ilm-expiry-replication", + Usage: "enable ILM expiry rules replication", + }, } var adminReplicateUpdateCmd = cli.Command{ @@ -80,6 +88,12 @@ 2. Edit a site in cluster-level replication to set default bandwidth limit for bucket: {{.Prompt}} {{.HelpName}} myminio --deployment-id c1758167-4426-454f-9aae-5c3dfdf6df64 --bucket-bandwidth "2G" + + 3. Disable replication of ILM expiry in cluster-level replication: + {{.Prompt}} {{.HelpName}} myminio --disable-ilm-expiry-replication + + 4. Enable replication of ILM expiry in cluster-level replication: + {{.Prompt}} {{.HelpName}} myminio --enable-ilm-expiry-replication `, } @@ -125,15 +139,21 @@ client, err := newAdminClient(aliasedURL) fatalIf(err, "Unable to initialize admin connection.") - if !ctx.IsSet("deployment-id") { + if !ctx.IsSet("deployment-id") && !ctx.IsSet("disable-ilm-expiry-replication") && !ctx.IsSet("enable-ilm-expiry-replication") { fatalIf(errInvalidArgument(), "--deployment-id is a required flag") } - if !ctx.IsSet("endpoint") && !ctx.IsSet("mode") && !ctx.IsSet("sync") && !ctx.IsSet("bucket-bandwidth") { - fatalIf(errInvalidArgument(), "--endpoint, --mode or --bucket-bandwidth is a required flag") + if !ctx.IsSet("endpoint") && !ctx.IsSet("mode") && !ctx.IsSet("sync") && !ctx.IsSet("bucket-bandwidth") && !ctx.IsSet("disable-ilm-expiry-replication") && !ctx.IsSet("enable-ilm-expiry-replication") { + fatalIf(errInvalidArgument(), "--endpoint, --mode, --bucket-bandwidth, --disable-ilm-expiry-replication or --enable-ilm-expiry-replication is a required flag") } if ctx.IsSet("mode") && ctx.IsSet("sync") { fatalIf(errInvalidArgument(), "either --sync or --mode flag should be specified") } + if ctx.IsSet("disable-ilm-expiry-replication") && ctx.IsSet("enable-ilm-expiry-replication") { + fatalIf(errInvalidArgument(), "either --disable-ilm-expiry-replication or --enable-ilm-expiry-replication flag should be specified") + } + if (ctx.IsSet("disable-ilm-expiry-replication") || ctx.IsSet("enable-ilm-expiry-replication")) && ctx.IsSet("deployment-id") { + fatalIf(errInvalidArgument(), "--deployment-id should not be set with --disable-ilm-expiry-replication or --enable-ilm-expiry-replication") + } var syncState string if ctx.IsSet("sync") { // for backward compatibility - deprecated Jul 2023 @@ -175,12 +195,15 @@ } ep = u.String() } + var opts madmin.SREditOptions + opts.DisableILMExpiryReplication = ctx.Bool("disable-ilm-expiry-replication") + opts.EnableILMExpiryReplication = ctx.Bool("enable-ilm-expiry-replication") res, e := client.SiteReplicationEdit(globalContext, madmin.PeerInfo{ DeploymentID: ctx.String("deployment-id"), Endpoint: ep, SyncState: madmin.SyncStatus(syncState), DefaultBandwidth: bwDefaults, - }, madmin.SREditOptions{}) + }, opts) fatalIf(probe.NewError(e).Trace(args...), "Unable to edit cluster replication site endpoint") printMsg(updateSuccessMessage(res)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/alias-set.go new/mc-20231202T020328Z/cmd/alias-set.go --- old/mc-20231120T163059Z/cmd/alias-set.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/alias-set.go 2023-12-02 03:03:28.000000000 +0100 @@ -235,8 +235,8 @@ // BuildS3Config constructs an S3 Config and does // signature auto-probe when needed. -func BuildS3Config(ctx context.Context, url, accessKey, secretKey, api, path string, peerCert *x509.Certificate) (*Config, *probe.Error) { - s3Config := NewS3Config(url, &aliasConfigV10{ +func BuildS3Config(ctx context.Context, alias, url, accessKey, secretKey, api, path string, peerCert *x509.Certificate) (*Config, *probe.Error) { + s3Config := NewS3Config(alias, url, &aliasConfigV10{ AccessKey: accessKey, SecretKey: secretKey, URL: url, @@ -339,7 +339,7 @@ fatalIf(err.Trace(alias, url, accessKey), "Unable to initialize new alias from the provided credentials.") } - s3Config, err := BuildS3Config(ctx, url, accessKey, secretKey, api, path, peerCert) + s3Config, err := BuildS3Config(ctx, alias, url, accessKey, secretKey, api, path, peerCert) fatalIf(err.Trace(alias, url, accessKey), "Unable to initialize new alias from the provided credentials.") msg := setAlias(alias, aliasConfigV10{ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/anonymous-main.go new/mc-20231202T020328Z/cmd/anonymous-main.go --- old/mc-20231120T163059Z/cmd/anonymous-main.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/anonymous-main.go 2023-12-02 03:03:28.000000000 +0100 @@ -52,7 +52,7 @@ USAGE: {{.HelpName}} [FLAGS] set PERMISSION TARGET - {{.HelpName}} [FLAGS] set-json TARGET FILE + {{.HelpName}} [FLAGS] set-json FILE TARGET {{.HelpName}} [FLAGS] get TARGET {{.HelpName}} [FLAGS] get-json TARGET {{.HelpName}} [FLAGS] list TARGET diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/client-admin.go new/mc-20231202T020328Z/cmd/client-admin.go --- old/mc-20231120T163059Z/cmd/client-admin.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/client-admin.go 2023-12-02 03:03:28.000000000 +0100 @@ -108,7 +108,7 @@ return nil, probe.NewError(fmt.Errorf("No valid configuration found for '%s' host alias", urlStrFull)) } - s3Config := NewS3Config(urlStrFull, aliasCfg) + s3Config := NewS3Config(alias, urlStrFull, aliasCfg) s3Client, err := s3AdminNew(s3Config) if err != nil { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/client-admin_test.go new/mc-20231202T020328Z/cmd/client-admin_test.go --- old/mc-20231120T163059Z/cmd/client-admin_test.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/client-admin_test.go 2023-12-02 03:03:28.000000000 +0100 @@ -19,15 +19,9 @@ import ( "bytes" - "context" "io" - "log" "net/http" - "net/http/httptest" - "os" "strconv" - - checkv1 "gopkg.in/check.v1" ) type adminPolicyHandler struct { @@ -67,61 +61,3 @@ w.WriteHeader(http.StatusForbidden) } } - -func (s *TestSuite) TestAdminSTSOperation(c *checkv1.C) { - sts := stsHandler{ - endpoint: "/", - jwt: []byte("eyJhbGciOiJSUzI1NiIsImtpZCI6Inc0dFNjMEc5Tk0wQWhGaWJYaWIzbkpRZkRKeDc1dURRTUVpOTNvTHJ0OWcifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzMxMTg3NzEwLCJpYXQiOjE2OTk2NTE3MTAsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtaW5pby10ZW5hbnQtMSIsInBvZCI6eyJuYW1lIjoic2V0dXAtYnVja2V0LXQ4eGdjIiwidWlkIjoiNjZhYjlkZWItNzkwMC00YTFlLTgzMDgtMTkwODIwZmQ3NDY5In0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJtYy1qb2Itc2EiLCJ1aWQiOiI3OTc4NzJjZC1kMjkwLTRlM2EtYjYyMC00ZGFkYzZhNzUyMTYifSwid2FybmFmdGVyIjoxNjk5NjU1MzE3fSwibmJmIjoxNjk5NjUxNzEwLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tdGVuYW50LTE6bWMtam9iLXNhIn0.rY7dpAh8GBTViH9Ges7tRhgyihdFWEN0DwXchelmZg58VOI526S-YfbCqrxksTs8Iu0fp1rmk1cUj7FGDh3AOv2RphHjoWci1802zKkHgH0iOEbKMp3jHXwfyHda8CyrSCPycGzClueCf1ae91wd_0lgK9lOR1qqY1HuDeXqSEAUIGrfh1VcP2n95Zc07EY-Uh3XjJE4drtgusACEK5n3P3WtN9s0m0GomEGQzF5ZJczxLGpHBKMQ5VDhMksVKdBAsx9xHgSx84aUhKQViYilAL-8PRj-R ZA9s_IpEymAh5R37dKzAO8Fqq0nG7fVbH_ifzw3xhHiG92BhHldBDqEQ"), - } - - tmpfile, errFs := os.CreateTemp("", "jwt") - if errFs != nil { - log.Fatal(errFs) - } - defer os.Remove(tmpfile.Name()) // clean up - - if _, errFs := tmpfile.Write(sts.jwt); errFs != nil { - log.Fatal(errFs) - } - if errFs := tmpfile.Close(); errFs != nil { - log.Fatal(errFs) - } - - stsServer := httptest.NewServer(sts) - defer stsServer.Close() - os.Setenv("MC_STS_ENDPOINT", stsServer.URL+sts.endpoint) - os.Setenv("MC_WEB_IDENTITY_TOKEN_FILE", tmpfile.Name()) - handler := adminPolicyHandler{ - endpoint: "/minio/admin/v3/add-canned-policy?name=", - name: "test", - policy: []byte(` -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:aws:s3:::test-bucket", - "arn:aws:s3:::test-bucket/*" - ] - } - ] - -}`), - } - server := httptest.NewServer(handler) - defer server.Close() - - conf := new(Config) - conf.Debug = true - conf.Insecure = true - conf.HostURL = server.URL + handler.endpoint + handler.name - s3c, err := s3AdminNew(conf) - c.Assert(err, checkv1.IsNil) - - policyErr := s3c.AddCannedPolicy(context.Background(), handler.name, handler.policy) - c.Assert(policyErr, checkv1.IsNil) -} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/client-s3.go new/mc-20231202T020328Z/cmd/client-s3.go --- old/mc-20231120T163059Z/cmd/client-s3.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/client-s3.go 2023-12-02 03:03:28.000000000 +0100 @@ -216,14 +216,14 @@ func getCredentialsChainForConfig(config *Config, transport http.RoundTripper) ([]credentials.Provider, *probe.Error) { var credsChain []credentials.Provider // if an STS endpoint is set, we will add that to the chain - if stsEndpoint := env.Get("MC_STS_ENDPOINT", ""); stsEndpoint != "" { + if stsEndpoint := env.Get("MC_STS_ENDPOINT_"+config.Alias, ""); stsEndpoint != "" { // set AWS_WEB_IDENTITY_TOKEN_FILE is MC_WEB_IDENTITY_TOKEN_FILE is set - if val := env.Get("MC_WEB_IDENTITY_TOKEN_FILE", ""); val != "" { + if val := env.Get("MC_WEB_IDENTITY_TOKEN_FILE_"+config.Alias, ""); val != "" { os.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", val) - if val := env.Get("MC_ROLE_ARN", ""); val != "" { + if val := env.Get("MC_ROLE_ARN_"+config.Alias, ""); val != "" { os.Setenv("AWS_ROLE_ARN", val) } - if val := env.Get("MC_ROLE_SESSION_NAME", randString(32, rand.NewSource(time.Now().UnixNano()), "mc-session-name-")); val != "" { + if val := env.Get("MC_ROLE_SESSION_NAME_"+config.Alias, randString(32, rand.NewSource(time.Now().UnixNano()), "mc-session-name-")); val != "" { os.Setenv("AWS_ROLE_SESSION_NAME", val) } } @@ -957,6 +957,8 @@ return nil, probe.NewError(err) } } + // Disallow automatic decompression for some objects with content-encoding set. + o.Set("Accept-Encoding", "identity") reader, e := c.api.GetObject(ctx, bucket, object, o) if e != nil { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/client-s3_test.go new/mc-20231202T020328Z/cmd/client-s3_test.go --- old/mc-20231120T163059Z/cmd/client-s3_test.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/client-s3_test.go 2023-12-02 03:03:28.000000000 +0100 @@ -22,10 +22,8 @@ "bytes" "context" "io" - "log" "net/http" "net/http/httptest" - "os" "strconv" minio "github.com/minio/minio-go/v7" @@ -88,7 +86,7 @@ } switch { - case r.Method == "PUT": + case r.Method == http.MethodPut: // Handler for PUT object request. length, e := strconv.Atoi(r.Header.Get("Content-Length")) if e != nil { @@ -102,7 +100,7 @@ } w.Header().Set("ETag", "9af2f8218b150c351ad802c6f3d66abe") w.WriteHeader(http.StatusOK) - case r.Method == "HEAD": + case r.Method == http.MethodHead: // Handler for Stat object request. if r.URL.Path != h.resource { w.WriteHeader(http.StatusNotFound) @@ -112,7 +110,7 @@ w.Header().Set("Last-Modified", UTCNow().Format(http.TimeFormat)) w.Header().Set("ETag", "9af2f8218b150c351ad802c6f3d66abe") w.WriteHeader(http.StatusOK) - case r.Method == "POST": + case r.Method == http.MethodPost: // Handler for multipart upload request. if _, ok := r.URL.Query()["uploads"]; ok { if r.URL.Path == h.resource { @@ -134,7 +132,7 @@ w.WriteHeader(http.StatusNotFound) return } - case r.Method == "GET": + case r.Method == http.MethodGet: // Handler for get bucket location request. if _, ok := r.URL.Query()["location"]; ok { response := []byte("<LocationConstraint xmlns=\"http://doc.s3.amazonaws.com/2006-03-01\"></LocationConstraint>") @@ -174,7 +172,7 @@ return } switch { - case r.Method == "POST": + case r.Method == http.MethodPost: token := r.Form.Get("WebIdentityToken") if token == string(h.jwt) { response := []byte("<AssumeRoleWithWebIdentityResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleWithWebIdentityResult><AssumedRoleUser><Arn></Arn><AssumeRoleId></AssumeRoleId></AssumedRoleUser><Credentials><AccessKeyId>7NL5BR739GUQ0ZOD4JNB</AccessKeyId><SecretAccessKey>A2mxZSxPnHNhSduedUHczsXZpVSSssOLpDruUmTV</SecretAccessKey><Expiration>0001-01-01T00:00:00Z</Expiration><SessionToken>eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI3Tkw1QlI3MzlHVVEwWk9ENEpOQiIsImV4cCI6MTY5OTYwMzMwNiwicGFyZW50IjoibWluaW8iLCJzZXNzaW9uUG9saWN5IjoiZXlKV1pYSnphVzl1SWpvaU1qQXhNaTB4TUMweE55SXNJbE4wWVhSbGJXVnVkQ0k2VzNzaVJXWm1aV04wSWpvaVFXeHNiM2NpTENKQlkzUnBiMjRpT2xzaVlXUnRhVzQ2S2lKZGZTeDdJa1ZtWm1WamRDSTZJa0ZzYkc5M0lpd2lRV04wYVc5dUlqcGJJbXR0Y3pvcUlsMTlMSHNpUldabVpXTjBJam9pUVd4c2IzY2lMQ0pCWTNScGIyNGlPbHNpY3pNNktpSmRMQ0pTWlhOdmRYSmpaU0k2V3lKaGNtNDZZWGR6T25Nek9qbzZLaUpkZlYxOSJ9.uuE_x7PO8QoPfUk9KzUELoAqxihIknZAvJLl5aYJjwpSjJYFTPLp6EvuyJX2hc18s9HzeiJ-vU0dPzsy50dXmg</SessionToken></ Credentials></AssumeRoleWithWebIdentityResult><ResponseMetadata></ResponseMetadata></AssumeRoleWithWebIdentityResponse>") @@ -182,14 +180,12 @@ w.Header().Set("Content-Type", "application/xml") w.Header().Set("Server", "MinIO") w.Write(response) - w.WriteHeader(http.StatusOK) return } else { response := []byte("<ErrorResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><Error><Type></Type><Code>AccessDenied</Code><Message>Access denied: Invalid Token</Message></Error><RequestId></RequestId></ErrorResponse>") w.Header().Set("Content-Length", strconv.Itoa(len(response))) w.Header().Set("Content-Type", "application/xml") w.Write(response) - w.WriteHeader(http.StatusForbidden) return } } @@ -279,52 +275,6 @@ } } -func (s *TestSuite) TestSTSOperation(c *checkv1.C) { - sts := stsHandler{ - endpoint: "/", - jwt: []byte("eyJhbGciOiJSUzI1NiIsImtpZCI6Inc0dFNjMEc5Tk0wQWhGaWJYaWIzbkpRZkRKeDc1dURRTUVpOTNvTHJ0OWcifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzMxMTIyNjg0LCJpYXQiOjE2OTk1ODY2ODQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtaW5pby10ZW5hbnQtMSIsInBvZCI6eyJuYW1lIjoic2V0dXAtYnVja2V0LXJ4aHhiIiwidWlkIjoiNmNhMzhjMmItYTdkMC00M2Y0LWE0NjMtZjdlNjU4MGUyZDdiIn0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJtYy1qb2Itc2EiLCJ1aWQiOiI3OTc4NzJjZC1kMjkwLTRlM2EtYjYyMC00ZGFkYzZhNzUyMTYifSwid2FybmFmdGVyIjoxNjk5NTkwMjkxfSwibmJmIjoxNjk5NTg2Njg0LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tdGVuYW50LTE6bWMtam9iLXNhIn0.fBJckmoQFyJ9bUgKZv6jzBESd9ccX_HFPPBZ17Gz_CsQ5wXrMqnvoMs1mcv6QKWsDsvSnWnw_tcW0cjvVkXb2mKmioKLzqV4ihGbiWzwk2e1xDohn8fizdQkf64bXpncjGdEGv8oi9A4300jfLMfg53POriMyEAQMeIDKPOI9qx913xjGni2w2H49mjLfnFnRaj9osvy17425dNIrMC6GDFq3rcq6Z_cdDmL18Jwsjy1xDsAhUzmOclr-VI3AeSnuD4fbf6jhbKE14qVUjLmIBf__B5NhE SiaFNwxFYjonZyi357Nx93CD1wai28tNRSODx7BiPHLxk8SyzY0CP0sQ"), - } - - tmpfile, errFs := os.CreateTemp("", "jwt") - if errFs != nil { - log.Fatal(errFs) - } - defer os.Remove(tmpfile.Name()) // clean up - - if _, errFs := tmpfile.Write(sts.jwt); errFs != nil { - log.Fatal(errFs) - } - if errFs := tmpfile.Close(); errFs != nil { - log.Fatal(errFs) - } - - stsServer := httptest.NewServer(sts) - defer stsServer.Close() - os.Setenv("MC_STS_ENDPOINT", stsServer.URL+sts.endpoint) - os.Setenv("MC_WEB_IDENTITY_TOKEN_FILE", tmpfile.Name()) - object := objectHandler{ - resource: "/bucket/object", - data: []byte("Hello, World"), - } - server := httptest.NewServer(object) - defer server.Close() - - conf := new(Config) - conf.HostURL = server.URL + object.resource - s3c, err := S3New(conf) - c.Assert(err, checkv1.IsNil) - - var reader io.Reader - reader = bytes.NewReader(object.data) - n, err := s3c.Put(context.Background(), reader, int64(len(object.data)), nil, PutOptions{ - metadata: map[string]string{ - "Content-Type": "application/octet-stream", - }, - }) - c.Assert(err, checkv1.IsNil) - c.Assert(n, checkv1.Equals, int64(len(object.data))) -} - var testSelectCompressionTypeCases = []struct { opts SelectObjectOpts object string diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/client-sts_test.go new/mc-20231202T020328Z/cmd/client-sts_test.go --- old/mc-20231120T163059Z/cmd/client-sts_test.go 1970-01-01 01:00:00.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/client-sts_test.go 2023-12-02 03:03:28.000000000 +0100 @@ -0,0 +1,144 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package cmd + +import ( + "bytes" + "context" + "io" + "log" + "net/http/httptest" + "os" + "testing" +) + +func TestSTSS3Operation(t *testing.T) { + sts := stsHandler{ + endpoint: "/", + jwt: []byte("eyJhbGciOiJSUzI1NiIsImtpZCI6Inc0dFNjMEc5Tk0wQWhGaWJYaWIzbkpRZkRKeDc1dURRTUVpOTNvTHJ0OWcifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzMxMTIyNjg0LCJpYXQiOjE2OTk1ODY2ODQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtaW5pby10ZW5hbnQtMSIsInBvZCI6eyJuYW1lIjoic2V0dXAtYnVja2V0LXJ4aHhiIiwidWlkIjoiNmNhMzhjMmItYTdkMC00M2Y0LWE0NjMtZjdlNjU4MGUyZDdiIn0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJtYy1qb2Itc2EiLCJ1aWQiOiI3OTc4NzJjZC1kMjkwLTRlM2EtYjYyMC00ZGFkYzZhNzUyMTYifSwid2FybmFmdGVyIjoxNjk5NTkwMjkxfSwibmJmIjoxNjk5NTg2Njg0LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tdGVuYW50LTE6bWMtam9iLXNhIn0.fBJckmoQFyJ9bUgKZv6jzBESd9ccX_HFPPBZ17Gz_CsQ5wXrMqnvoMs1mcv6QKWsDsvSnWnw_tcW0cjvVkXb2mKmioKLzqV4ihGbiWzwk2e1xDohn8fizdQkf64bXpncjGdEGv8oi9A4300jfLMfg53POriMyEAQMeIDKPOI9qx913xjGni2w2H49mjLfnFnRaj9osvy17425dNIrMC6GDFq3rcq6Z_cdDmL18Jwsjy1xDsAhUzmOclr-VI3AeSnuD4fbf6jhbKE14qVUjLmIBf__B5NhE SiaFNwxFYjonZyi357Nx93CD1wai28tNRSODx7BiPHLxk8SyzY0CP0sQ"), + } + + tmpfile, errFs := os.CreateTemp("", "jwt") + if errFs != nil { + log.Fatal(errFs) + } + defer os.Remove(tmpfile.Name()) // clean up + + if _, errFs := tmpfile.Write(sts.jwt); errFs != nil { + log.Fatal(errFs) + } + if errFs := tmpfile.Close(); errFs != nil { + log.Fatal(errFs) + } + + stsServer := httptest.NewServer(sts) + defer stsServer.Close() + t.Setenv("MC_STS_ENDPOINT_test", stsServer.URL+sts.endpoint) + t.Setenv("MC_WEB_IDENTITY_TOKEN_FILE_test", tmpfile.Name()) + object := objectHandler{ + resource: "/bucket/object", + data: []byte("Hello, World"), + } + server := httptest.NewServer(object) + defer server.Close() + + conf := new(Config) + conf.Alias = "test" + conf.HostURL = server.URL + object.resource + s3c, err := S3New(conf) + if err != nil { + t.Fatal(err) + } + + var reader io.Reader + reader = bytes.NewReader(object.data) + n, err := s3c.Put(context.Background(), reader, int64(len(object.data)), nil, PutOptions{ + metadata: map[string]string{ + "Content-Type": "application/octet-stream", + }, + }) + if err != nil { + t.Fatal(err) + } + if n != int64(len(object.data)) { + t.Fatalf("expected %d, got %d", n, len(object.data)) + } +} + +func TestAdminSTSOperation(t *testing.T) { + sts := stsHandler{ + endpoint: "/", + jwt: []byte("eyJhbGciOiJSUzI1NiIsImtpZCI6Inc0dFNjMEc5Tk0wQWhGaWJYaWIzbkpRZkRKeDc1dURRTUVpOTNvTHJ0OWcifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNzMxMTg3NzEwLCJpYXQiOjE2OTk2NTE3MTAsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtaW5pby10ZW5hbnQtMSIsInBvZCI6eyJuYW1lIjoic2V0dXAtYnVja2V0LXQ4eGdjIiwidWlkIjoiNjZhYjlkZWItNzkwMC00YTFlLTgzMDgtMTkwODIwZmQ3NDY5In0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJtYy1qb2Itc2EiLCJ1aWQiOiI3OTc4NzJjZC1kMjkwLTRlM2EtYjYyMC00ZGFkYzZhNzUyMTYifSwid2FybmFmdGVyIjoxNjk5NjU1MzE3fSwibmJmIjoxNjk5NjUxNzEwLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tdGVuYW50LTE6bWMtam9iLXNhIn0.rY7dpAh8GBTViH9Ges7tRhgyihdFWEN0DwXchelmZg58VOI526S-YfbCqrxksTs8Iu0fp1rmk1cUj7FGDh3AOv2RphHjoWci1802zKkHgH0iOEbKMp3jHXwfyHda8CyrSCPycGzClueCf1ae91wd_0lgK9lOR1qqY1HuDeXqSEAUIGrfh1VcP2n95Zc07EY-Uh3XjJE4drtgusACEK5n3P3WtN9s0m0GomEGQzF5ZJczxLGpHBKMQ5VDhMksVKdBAsx9xHgSx84aUhKQViYilAL-8PRj-R ZA9s_IpEymAh5R37dKzAO8Fqq0nG7fVbH_ifzw3xhHiG92BhHldBDqEQ"), + } + + tmpfile, errFs := os.CreateTemp("", "jwt") + if errFs != nil { + log.Fatal(errFs) + } + defer os.Remove(tmpfile.Name()) // clean up + + if _, errFs := tmpfile.Write(sts.jwt); errFs != nil { + log.Fatal(errFs) + } + if errFs := tmpfile.Close(); errFs != nil { + log.Fatal(errFs) + } + + stsServer := httptest.NewServer(sts) + defer stsServer.Close() + t.Setenv("MC_STS_ENDPOINT_test", stsServer.URL+sts.endpoint) + t.Setenv("MC_WEB_IDENTITY_TOKEN_FILE_test", tmpfile.Name()) + handler := adminPolicyHandler{ + endpoint: "/minio/admin/v3/add-canned-policy?name=", + name: "test", + policy: []byte(` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::test-bucket", + "arn:aws:s3:::test-bucket/*" + ] + } + ] + +}`), + } + server := httptest.NewServer(handler) + defer server.Close() + + conf := new(Config) + conf.Alias = "test" + conf.Debug = true + conf.Insecure = true + conf.HostURL = server.URL + handler.endpoint + handler.name + s3c, err := s3AdminNew(conf) + if err != nil { + t.Fatal(err) + } + + e := s3c.AddCannedPolicy(context.Background(), handler.name, handler.policy) + if e != nil { + t.Fatal(e) + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/client.go new/mc-20231202T020328Z/cmd/client.go --- old/mc-20231120T163059Z/cmd/client.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/client.go 2023-12-02 03:03:28.000000000 +0100 @@ -216,6 +216,7 @@ // Config - see http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html type Config struct { + Alias string AccessKey string SecretKey string SessionToken string diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/common-methods.go new/mc-20231202T020328Z/cmd/common-methods.go --- old/mc-20231120T163059Z/cmd/common-methods.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/common-methods.go 2023-12-02 03:03:28.000000000 +0100 @@ -629,8 +629,7 @@ return fsClient, nil } - s3Config := NewS3Config(urlStr, hostCfg) - + s3Config := NewS3Config(alias, urlStr, hostCfg) s3Client, err := S3New(s3Config) if err != nil { return nil, err.Trace(alias, urlStr) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/ilm/options.go new/mc-20231202T020328Z/cmd/ilm/options.go --- old/mc-20231120T163059Z/cmd/ilm/options.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/ilm/options.go 2023-12-02 03:03:28.000000000 +0100 @@ -20,9 +20,11 @@ import ( "errors" "fmt" + "math" "strconv" "strings" + "github.com/dustin/go-humanize" "github.com/minio/cli" "github.com/minio/mc/pkg/probe" "github.com/minio/minio-go/v7/pkg/lifecycle" @@ -61,13 +63,15 @@ Status *bool - Prefix *string - Tags *string - ExpiryDate *string - ExpiryDays *string - TransitionDate *string - TransitionDays *string - StorageClass *string + Prefix *string + Tags *string + ObjectSizeLessThan *int64 + ObjectSizeGreaterThan *int64 + ExpiryDate *string + ExpiryDays *string + TransitionDate *string + TransitionDays *string + StorageClass *string ExpiredObjectDeleteMarker *bool NoncurrentVersionExpirationDays *int @@ -77,13 +81,58 @@ NoncurrentVersionTransitionStorageClass *string } +// Filter returns lifecycle.Filter appropriate for opts +func (opts LifecycleOptions) Filter() lifecycle.Filter { + var f lifecycle.Filter + var tags []lifecycle.Tag + var predCount int + if opts.Tags != nil { + tags = extractILMTags(*opts.Tags) + predCount += len(tags) + } + var prefix string + if opts.Prefix != nil { + prefix = *opts.Prefix + predCount++ + } + + var szLt, szGt int64 + if opts.ObjectSizeLessThan != nil { + szLt = *opts.ObjectSizeLessThan + predCount++ + } + + if opts.ObjectSizeGreaterThan != nil { + szGt = *opts.ObjectSizeGreaterThan + predCount++ + } + + if predCount >= 2 { + f.And = lifecycle.And{ + Tags: tags, + Prefix: prefix, + ObjectSizeLessThan: szLt, + ObjectSizeGreaterThan: szGt, + } + } else { + // In a valid lifecycle rule filter at most one of the + // following will only be set. + f.Prefix = prefix + f.ObjectSizeGreaterThan = szGt + f.ObjectSizeLessThan = szLt + if len(tags) >= 1 { + f.Tag = tags[0] + } + } + + return f +} + // ToILMRule creates lifecycle.Configuration based on LifecycleOptions func (opts LifecycleOptions) ToILMRule() (lifecycle.Rule, *probe.Error) { var ( id, status string - filter lifecycle.Filter - nonCurrentVersionExpirationDays lifecycle.ExpirationDays newerNonCurrentExpirationVersions int nonCurrentVersionTransitionDays lifecycle.ExpirationDays @@ -110,23 +159,6 @@ return lifecycle.Rule{}, err } - andVal := lifecycle.And{} - if opts.Tags != nil { - andVal.Tags = extractILMTags(*opts.Tags) - } - - if opts.Prefix != nil { - filter.Prefix = *opts.Prefix - } - - if len(andVal.Tags) > 0 { - filter.And = andVal - if opts.Prefix != nil { - filter.And.Prefix = *opts.Prefix - } - filter.Prefix = "" - } - if opts.NoncurrentVersionExpirationDays != nil { nonCurrentVersionExpirationDays = lifecycle.ExpirationDays(*opts.NoncurrentVersionExpirationDays) } @@ -145,7 +177,7 @@ newRule := lifecycle.Rule{ ID: id, - RuleFilter: filter, + RuleFilter: opts.Filter(), Status: status, Expiration: expiry, Transition: transition, @@ -177,6 +209,10 @@ return &ptr } +func int64Ptr(i int64) *int64 { + return &i +} + func boolPtr(b bool) *bool { ptr := b return &ptr @@ -191,6 +227,8 @@ prefix *string tags *string + sizeLt *int64 + sizeGt *int64 expiryDate *string expiryDays *string transitionDate *string @@ -232,6 +270,24 @@ } } + if ctx.IsSet("size-lt") { + szStr := ctx.String("size-lt") + szLt, err := humanize.ParseBytes(szStr) + if err != nil || szLt > math.MaxInt64 { + return LifecycleOptions{}, probe.NewError(fmt.Errorf("size-lt value %s is invalid", szStr)) + } + + sizeLt = int64Ptr(int64(szLt)) + } + if ctx.IsSet("size-gt") { + szStr := ctx.String("size-gt") + szGt, err := humanize.ParseBytes(szStr) + if err != nil || szGt > math.MaxInt64 { + return LifecycleOptions{}, probe.NewError(fmt.Errorf("size-gt value %s is invalid", szStr)) + } + sizeGt = int64Ptr(int64(szGt)) + } + // For backward-compatibility if ctx.IsSet("storage-class") { tier = strPtr(strings.ToUpper(ctx.String("storage-class"))) @@ -318,6 +374,8 @@ Status: status, Prefix: prefix, Tags: tags, + ObjectSizeLessThan: sizeLt, + ObjectSizeGreaterThan: sizeGt, ExpiryDate: expiryDate, ExpiryDays: expiryDays, TransitionDate: transitionDate, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/ilm/options_test.go new/mc-20231202T020328Z/cmd/ilm/options_test.go --- old/mc-20231120T163059Z/cmd/ilm/options_test.go 1970-01-01 01:00:00.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/ilm/options_test.go 2023-12-02 03:03:28.000000000 +0100 @@ -0,0 +1,155 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package ilm + +import ( + "fmt" + "testing" + + "github.com/dustin/go-humanize" + "github.com/minio/minio-go/v7/pkg/lifecycle" +) + +func TestOptionFilter(t *testing.T) { + emptyFilter := lifecycle.Filter{} + emptyOpts := LifecycleOptions{} + + filterWithPrefix := lifecycle.Filter{ + Prefix: "doc/", + } + optsWithPrefix := LifecycleOptions{ + Prefix: strPtr("doc/"), + } + + filterWithTag := lifecycle.Filter{ + Tag: lifecycle.Tag{ + Key: "key1", + Value: "value1", + }, + } + optsWithTag := LifecycleOptions{ + Tags: strPtr("key1=value1"), + } + + filterWithSzLt := lifecycle.Filter{ + ObjectSizeLessThan: 100 * humanize.MiByte, + } + optsWithSzLt := LifecycleOptions{ + ObjectSizeLessThan: int64Ptr(100 * humanize.MiByte), + } + + filterWithSzGt := lifecycle.Filter{ + ObjectSizeGreaterThan: 1 * humanize.MiByte, + } + optsWithSzGt := LifecycleOptions{ + ObjectSizeGreaterThan: int64Ptr(1 * humanize.MiByte), + } + + filterWithAnd := lifecycle.Filter{ + And: lifecycle.And{ + Prefix: "doc/", + Tags: []lifecycle.Tag{ + { + Key: "key1", + Value: "value1", + }, + }, + ObjectSizeLessThan: 100 * humanize.MiByte, + ObjectSizeGreaterThan: 1 * humanize.MiByte, + }, + } + optsWithAnd := LifecycleOptions{ + Prefix: strPtr("doc/"), + Tags: strPtr("key1=value1"), + ObjectSizeLessThan: int64Ptr(100 * humanize.MiByte), + ObjectSizeGreaterThan: int64Ptr(1 * humanize.MiByte), + } + + tests := []struct { + opts LifecycleOptions + want lifecycle.Filter + }{ + { + opts: emptyOpts, + want: emptyFilter, + }, + { + opts: optsWithPrefix, + want: filterWithPrefix, + }, + { + opts: optsWithTag, + want: filterWithTag, + }, + { + opts: optsWithSzGt, + want: filterWithSzGt, + }, + { + opts: optsWithSzLt, + want: filterWithSzLt, + }, + { + opts: optsWithAnd, + want: filterWithAnd, + }, + } + + filterEq := func(a, b lifecycle.Filter) bool { + if a.ObjectSizeGreaterThan != b.ObjectSizeGreaterThan { + return false + } + if a.ObjectSizeLessThan != b.ObjectSizeLessThan { + return false + } + if a.Prefix != b.Prefix { + return false + } + if a.Tag != b.Tag { + return false + } + + if a.And.ObjectSizeGreaterThan != b.And.ObjectSizeGreaterThan { + return false + } + if a.And.ObjectSizeLessThan != b.And.ObjectSizeLessThan { + return false + } + if a.And.Prefix != b.And.Prefix { + return false + } + if len(a.And.Tags) != len(b.And.Tags) { + return false + } + for i := range a.And.Tags { + if a.And.Tags[i] != b.And.Tags[i] { + return false + } + } + + return true + } + + for i, test := range tests { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + if got := test.opts.Filter(); !filterEq(got, test.want) { + t.Fatalf("Expected %#v but got %#v", test.want, got) + } + }) + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/ilm-rule-add.go new/mc-20231202T020328Z/cmd/ilm-rule-add.go --- old/mc-20231120T163059Z/cmd/ilm-rule-add.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/ilm-rule-add.go 2023-12-02 03:03:28.000000000 +0100 @@ -49,13 +49,13 @@ Add a lifecycle configuration rule. EXAMPLES: - 1. Add a lifecycle rule with a transition and a noncurrent version transition action for objects with prefix doc/ in mybucket. + 1. Add a lifecycle rule with a transition and a noncurrent version transition action for objects with prefix doc/ whose size is greater than 1MiB in mybucket. Tiers must exist in MinIO. Use existing tiers or add new tiers. {{.Prompt}} mc ilm tier add minio myminio MINIOTIER-1 --endpoint https://warm-minio-1.com \ --access-key ACCESSKEY --secret-key SECRETKEY --bucket bucket1 --prefix prefix1 - {{.Prompt}} {{.HelpName}} --prefix "doc/" --transition-days "90" --transition-tier "MINIOTIER-1" \ + {{.Prompt}} {{.HelpName}} --prefix "doc/" --size-gt 1MiB --transition-days "90" --transition-tier "MINIOTIER-1" \ --noncurrent-transition-days "45" --noncurrent-transition-tier "MINIOTIER-1" \ myminio/mybucket/ @@ -78,6 +78,14 @@ Usage: "key value pairs of the form '<key1>=<value1>&<key2>=<value2>&<key3>=<value3>'", }, cli.StringFlag{ + Name: "size-lt", + Usage: "objects with size less than this value will be selected for the lifecycle action", + }, + cli.StringFlag{ + Name: "size-gt", + Usage: "objects with size greater than this value will be selected for the lifecycle action", + }, + cli.StringFlag{ Name: "expiry-date", Usage: "format 'YYYY-MM-DD' the date of expiration", Hidden: true, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/cmd/utils.go new/mc-20231202T020328Z/cmd/utils.go --- old/mc-20231120T163059Z/cmd/utils.go 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/cmd/utils.go 2023-12-02 03:03:28.000000000 +0100 @@ -137,7 +137,7 @@ // NewS3Config simply creates a new Config struct using the passed // parameters. -func NewS3Config(urlStr string, aliasCfg *aliasConfigV10) *Config { +func NewS3Config(alias, urlStr string, aliasCfg *aliasConfigV10) *Config { // We have a valid alias and hostConfig. We populate the // credentials from the match found in the config file. s3Config := new(Config) @@ -152,6 +152,7 @@ s3Config.DownloadLimit = int64(globalLimitDownload) s3Config.HostURL = urlStr + s3Config.Alias = alias if aliasCfg != nil { s3Config.AccessKey = aliasCfg.AccessKey s3Config.SecretKey = aliasCfg.SecretKey diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/go.mod new/mc-20231202T020328Z/go.mod --- old/mc-20231120T163059Z/go.mod 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/go.mod 2023-12-02 03:03:28.000000000 +0100 @@ -20,7 +20,7 @@ github.com/minio/colorjson v1.0.6 github.com/minio/filepath v1.0.0 github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.63 + github.com/minio/minio-go/v7 v7.0.65-0.20231130060928-54e115c40e66 github.com/minio/selfupdate v0.6.0 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 @@ -46,8 +46,8 @@ github.com/gdamore/tcell/v2 v2.6.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/juju/ratelimit v1.0.2 - github.com/minio/madmin-go/v3 v3.0.29 - github.com/minio/pkg/v2 v2.0.2 + github.com/minio/madmin-go/v3 v3.0.34 + github.com/minio/pkg/v2 v2.0.4 github.com/muesli/reflow v0.3.0 github.com/navidys/tvxwidgets v0.3.0 github.com/olekukonko/tablewriter v0.0.5 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mc-20231120T163059Z/go.sum new/mc-20231202T020328Z/go.sum --- old/mc-20231120T163059Z/go.sum 2023-11-20 17:30:59.000000000 +0100 +++ new/mc-20231202T020328Z/go.sum 2023-12-02 03:03:28.000000000 +0100 @@ -140,16 +140,18 @@ github.com/minio/colorjson v1.0.6/go.mod h1:LUXwS5ZGNb6Eh9f+t+3uJiowD3XsIWtsvTriUBeqgYs= github.com/minio/filepath v1.0.0 h1:fvkJu1+6X+ECRA6G3+JJETj4QeAYO9sV43I79H8ubDY= github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEXx9T/Bw= -github.com/minio/madmin-go/v3 v3.0.29 h1:3bNLArtxIFud5wyb5/DnF5DGLBvcSJyzCA44EclX1Ow= -github.com/minio/madmin-go/v3 v3.0.29/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8= +github.com/minio/madmin-go/v3 v3.0.34 h1:MGPQYIWm52liSubofK24FhrznPYnRpQrDNddZJEyBPA= +github.com/minio/madmin-go/v3 v3.0.34/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/minio-go/v7 v7.0.65-0.20231122233251-1f7dd6b7e3e1 h1:oGua0HckYk7JuIW/c1IuInHe9MCef1U6Q2Qf9ea/V5U= +github.com/minio/minio-go/v7 v7.0.65-0.20231122233251-1f7dd6b7e3e1/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= +github.com/minio/minio-go/v7 v7.0.65-0.20231130060928-54e115c40e66 h1:9pgbcTmr0yvwlQJHsS72QDyNkZajKXCxnszpJtM5B4I= +github.com/minio/minio-go/v7 v7.0.65-0.20231130060928-54e115c40e66/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= -github.com/minio/pkg/v2 v2.0.2 h1:cytXmC21fBNS+0NVKEE5FuYmQfY+HFTqis6Kkj3U9ac= -github.com/minio/pkg/v2 v2.0.2/go.mod h1:6xTAr5M9yobpUroXAAaTrGJ9fhOZIqKYOT0I87u2yZ4= +github.com/minio/pkg/v2 v2.0.4 h1:vHQOqxD7AdsJq3X9GylO0PO6WCpcK7aePsVfk0oRYh0= +github.com/minio/pkg/v2 v2.0.4/go.mod h1:ya1jPY/rtVp3HarxwDQJkclR8Rd2QxwxA9kwS436ovs= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= ++++++ mc.obsinfo ++++++ --- /var/tmp/diff_new_pack.UbA33u/_old 2023-12-04 23:01:14.995044846 +0100 +++ /var/tmp/diff_new_pack.UbA33u/_new 2023-12-04 23:01:14.999044993 +0100 @@ -1,5 +1,5 @@ name: mc -version: 20231120T163059Z -mtime: 1700497859 -commit: 937b34616f012ad30e1cd6fd61e1da25ff931648 +version: 20231202T020328Z +mtime: 1701482608 +commit: f5f7147b9ec4cf78eb67f1cdc91b63d191852e6a ++++++ vendor.tar.gz ++++++ /work/SRC/openSUSE:Factory/minio-client/vendor.tar.gz /work/SRC/openSUSE:Factory/.minio-client.new.25432/vendor.tar.gz differ: char 5, line 1