This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-tck.git
The following commit(s) were added to refs/heads/main by this push:
new 4d88b55 feat: update snapshot revisions through CLI (#7)
4d88b55 is described below
commit 4d88b55cfa4d7e2e7cf1c8f90372c4078d57c672
Author: tison <[email protected]>
AuthorDate: Tue Sep 1 09:32:26 2026 +0800
feat: update snapshot revisions through CLI (#7)
Signed-off-by: tison <[email protected]>
---
cmd/tck/__snapshots__/root_test.snap | 3 +-
cmd/tck/output.go | 25 ++++++
cmd/tck/snapshots.go | 74 +++++++++++++---
config.toml | 26 ++++++
docs/serialization-snapshots.md | 83 ++++++++++--------
go.mod | 1 +
go.sum | 2 +
internal/snapshots/config.go | 166 +++++++++++++++++++++++++++++++++++
internal/snapshots/generator.go | 22 ++---
internal/snapshots/reconcile.go | 86 ++++++++++++++++--
internal/snapshots/runner.go | 46 ++++++++--
11 files changed, 454 insertions(+), 80 deletions(-)
diff --git a/cmd/tck/__snapshots__/root_test.snap
b/cmd/tck/__snapshots__/root_test.snap
index 101cf9e..7442f13 100644
--- a/cmd/tck/__snapshots__/root_test.snap
+++ b/cmd/tck/__snapshots__/root_test.snap
@@ -8,7 +8,8 @@ Usage:
Available Commands:
check Check the snapshot set and stable snapshot contents
- update Replace committed snapshots with newly generated snapshots
+ sync Synchronize snapshots from config.toml
+ update Adopt one upstream revision and update its snapshots
Flags:
-h, --help help for snapshots
diff --git a/cmd/tck/output.go b/cmd/tck/output.go
index 8676101..401e04b 100644
--- a/cmd/tck/output.go
+++ b/cmd/tck/output.go
@@ -32,6 +32,13 @@ import (
func printResult(output io.Writer, root string, mode snapshots.Mode, result
snapshots.Result) error {
target := displayPath(root, result.Target)
if len(result.Changes) == 0 {
+ if result.RevisionChanged() {
+ if err := printRevisionChange(output, result); err !=
nil {
+ return err
+ }
+ _, err := fmt.Fprintf(output, "✓ Updated %s.\n", target)
+ return err
+ }
_, err := fmt.Fprintf(output, "✓ %s is up to date.\n", target)
return err
}
@@ -71,11 +78,19 @@ func printResult(output io.Writer, root string, mode
snapshots.Mode, result snap
); err != nil {
return err
}
+ if result.RevisionChanged() {
+ if err := printRevisionChange(output, result); err != nil {
+ return err
+ }
+ }
switch {
case mode == snapshots.ModeUpdate:
_, err := fmt.Fprintf(output, "✓ Updated %s.\n", target)
return err
+ case mode == snapshots.ModeSync:
+ _, err := fmt.Fprintf(output, "✓ Synced %s.\n", target)
+ return err
case result.HasBlockingChanges():
count := result.BlockingChangeCount()
_, err := fmt.Fprintf(output, "✗ %s has %d blocking %s.\n",
target, count, plural(count, "change", "changes"))
@@ -91,6 +106,16 @@ func printResult(output io.Writer, root string, mode
snapshots.Mode, result snap
}
}
+func printRevisionChange(output io.Writer, result snapshots.Result) error {
+ _, err := fmt.Fprintf(
+ output,
+ "Source revision: %s -> %s\n",
+ result.PreviousRevision,
+ result.Revision,
+ )
+ return err
+}
+
func changeDetails(change snapshots.Change) string {
switch change.Status {
case snapshots.ChangeAdded:
diff --git a/cmd/tck/snapshots.go b/cmd/tck/snapshots.go
index 0df9b74..be11fc6 100644
--- a/cmd/tck/snapshots.go
+++ b/cmd/tck/snapshots.go
@@ -42,33 +42,72 @@ func newSnapshotsCommand() *cobra.Command {
},
}
command.AddCommand(
- newSnapshotReconcileCommand(snapshots.ModeCheck),
- newSnapshotReconcileCommand(snapshots.ModeUpdate),
+ newSnapshotCheckCommand(),
+ newSnapshotSyncCommand(),
+ newSnapshotUpdateCommand(),
)
return command
}
-func newSnapshotReconcileCommand(mode snapshots.Mode) *cobra.Command {
- verb := string(mode)
+func newSnapshotCheckCommand() *cobra.Command {
validLanguages := append(snapshots.Languages(), "all")
return &cobra.Command{
- Use: verb + " <cpp|go|java|all>",
- Short: reconcileDescription(mode),
- Long: reconcileDetails(mode),
+ Use: "check <cpp|go|java|all>",
+ Short: reconcileDescription(snapshots.ModeCheck),
+ Long: reconcileDetails(snapshots.ModeCheck),
Args: cobra.MatchAll(cobra.ExactArgs(1),
cobra.OnlyValidArgs),
ValidArgs: validLanguages,
RunE: func(command *cobra.Command, args []string) error {
- return reconcileSnapshots(command, mode, args[0])
+ languages := []string{args[0]}
+ if args[0] == "all" {
+ languages = snapshots.Languages()
+ }
+ return reconcileSnapshots(command, snapshots.ModeCheck,
languages, "")
},
}
}
-func reconcileSnapshots(command *cobra.Command, mode snapshots.Mode,
requestedLanguage string) error {
- languages := []string{requestedLanguage}
- if requestedLanguage == "all" {
- languages = snapshots.Languages()
+func newSnapshotSyncCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "sync",
+ Short: reconcileDescription(snapshots.ModeSync),
+ Long: reconcileDetails(snapshots.ModeSync),
+ Args: cobra.NoArgs,
+ RunE: func(command *cobra.Command, _ []string) error {
+ return reconcileSnapshots(command, snapshots.ModeSync,
snapshots.Languages(), "")
+ },
}
+}
+
+func newSnapshotUpdateCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "update <cpp|go|java> <revision>",
+ Short: reconcileDescription(snapshots.ModeUpdate),
+ Long: reconcileDetails(snapshots.ModeUpdate),
+ Args: func(command *cobra.Command, args []string) error {
+ if err := cobra.ExactArgs(2)(command, args); err != nil
{
+ return err
+ }
+ for _, language := range snapshots.Languages() {
+ if args[0] == language {
+ return nil
+ }
+ }
+ return fmt.Errorf("unsupported snapshot language %q",
args[0])
+ },
+ ValidArgs: snapshots.Languages(),
+ RunE: func(command *cobra.Command, args []string) error {
+ return reconcileSnapshots(command,
snapshots.ModeUpdate, []string{args[0]}, args[1])
+ },
+ }
+}
+func reconcileSnapshots(
+ command *cobra.Command,
+ mode snapshots.Mode,
+ languages []string,
+ requestedRevision string,
+) error {
root, err := repositoryRoot(command.Context())
if err != nil {
return err
@@ -95,6 +134,7 @@ func reconcileSnapshots(command *cobra.Command, mode
snapshots.Mode, requestedLa
root,
language,
mode,
+ requestedRevision,
command.OutOrStdout(),
command.ErrOrStderr(),
)
@@ -116,8 +156,10 @@ func reconcileDescription(mode snapshots.Mode) string {
switch mode {
case snapshots.ModeCheck:
return "Check the snapshot set and stable snapshot contents"
+ case snapshots.ModeSync:
+ return "Synchronize snapshots from config.toml"
case snapshots.ModeUpdate:
- return "Replace committed snapshots with newly generated
snapshots"
+ return "Adopt one upstream revision and update its snapshots"
default:
panic(fmt.Sprintf("unsupported snapshot mode %q", mode))
}
@@ -127,8 +169,10 @@ func reconcileDetails(mode snapshots.Mode) string {
switch mode {
case snapshots.ModeCheck:
return "Generate snapshots and verify that the file set and
stable contents match. Content changes to existing probabilistic snapshots are
reported but do not fail the check."
+ case snapshots.ModeSync:
+ return "Regenerate all snapshot directories from the
repositories and commits in config.toml without changing the config."
case snapshots.ModeUpdate:
- return "Generate snapshots and atomically replace the selected
committed snapshot directory with the complete generated file set."
+ return "Resolve an upstream commit, branch, or tag, record its
exact commit ID in config.toml, and regenerate that source's snapshots."
default:
panic(fmt.Sprintf("unsupported snapshot mode %q", mode))
}
@@ -138,6 +182,8 @@ func modeHeading(mode snapshots.Mode) string {
switch mode {
case snapshots.ModeCheck:
return "Check"
+ case snapshots.ModeSync:
+ return "Sync"
case snapshots.ModeUpdate:
return "Update"
default:
diff --git a/config.toml b/config.toml
new file mode 100644
index 0000000..316318c
--- /dev/null
+++ b/config.toml
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[snapshot.cpp]
+repository = "https://github.com/apache/datasketches-cpp.git";
+commit = "401423367055acdf7502e8ed3126730a08039d91"
+
+[snapshot.go]
+repository = "https://github.com/apache/datasketches-go.git";
+commit = "730c0ca31e00b8becf8b70591ae8ca73954912d0"
+
+[snapshot.java]
+repository = "https://github.com/apache/datasketches-java.git";
+commit = "f3b334b380feee9f928500de9adb6bbf763fc104"
diff --git a/docs/serialization-snapshots.md b/docs/serialization-snapshots.md
index d44c415..95b1af9 100644
--- a/docs/serialization-snapshots.md
+++ b/docs/serialization-snapshots.md
@@ -19,69 +19,78 @@
The serialization corpus is a compatibility boundary between DataSketches
implementations. Each directory under `serialization/<language>/snapshots`
contains sketches produced by one implementation and intended to be read and
validated by the others.
-This repository generates snapshots from exact upstream commits instead of
following the latest branch. Pinning makes a checkout reproducible and ensures
that changes to the compatibility corpus receive normal code review.
+This repository generates snapshots from the upstream sources configured in
`config.toml` instead of following the latest branch. Each source records its
repository and exact commit so a checkout remains reproducible.
-## Set up the toolchain
+```toml
+[snapshot.go]
+repository = "https://github.com/apache/datasketches-go.git";
+commit = "730c0ca31e00b8becf8b70591ae8ca73954912d0"
+```
-Install [mise](https://mise.jdx.dev/), then install the pinned toolchain and
inspect the available snapshot commands:
+## Update one source
+
+With the [toolchain installed](#set-up-the-toolchain), adopt a different
upstream branch, tag, or commit by specifying one source language and revision:
```shell
-mise install
-mise run tck -- snapshots --help
+mise run tck -- snapshots update go main
```
-Mise supplies Go, CMake and CTest, Java, and Maven. Git is required for every
source language, a C++ compiler is required for C++, and Make is required for
Go.
+The command resolves the supplied revision to an exact commit ID, writes it to
`[snapshot.go]` in `config.toml`, and regenerates `serialization/go/snapshots`.
Generation happens before either file set is changed, so a generation failure
leaves the repository untouched.
-Commands accept `cpp`, `go`, `java`, or `all` as the source language.
+Updates intentionally handle one source at a time so each upstream change can
be reviewed independently. If the upstream build or output layout changed,
update the corresponding adapter in `internal/snapshots/<language>.go` before
running the command.
-## Update snapshots from upstream
+Review the resolved pin and corpus together:
-Updating the corpus is an intentional maintainer operation because selecting
an upstream revision and accepting compatibility changes require review. There
is no GitHub Actions workflow that discovers newer revisions or opens snapshot
update pull requests.
+```shell
+git diff --stat
+git diff -- config.toml serialization/go/snapshots
+```
-To update one source language:
+Added and deleted files change the set of compatibility cases. Unexpected
changes to deterministic files should be understood from the upstream change
before they are accepted.
-1. Choose the upstream commit to adopt. Prefer a commit on the
implementation's main development branch whose generator represents the
compatibility behavior being adopted.
-2. Change that language's `commit` field in `internal/snapshots/generator.go`.
If the upstream build or output layout changed, update the corresponding
adapter in `internal/snapshots/<language>.go` as well.
-3. Regenerate the complete snapshot directory:
+Verify that generation is reproducible at the new pin and run the repository
checks:
- ```shell
- mise run tck -- snapshots update go
- ```
+```shell
+mise run tck -- snapshots check go
+mise run check
+```
- Update mode generates from the new pin and atomically replaces
`serialization/go/snapshots`; it is not an incremental copy, so removed
upstream outputs become visible as deletions.
+A second generation may report allowed modifications for known probabilistic
snapshots. The file set and deterministic contents must reproduce.
-4. Review the pin and corpus together:
+## Check the pinned corpus
- ```shell
- git diff --stat
- git diff -- internal/snapshots/generator.go
- git status --short serialization/go/snapshots
- ```
+Use check mode to regenerate snapshots from the currently pinned commit and
compare them with the committed corpus:
- Added and deleted files change the set of compatibility cases. Unexpected
changes to deterministic files should be understood from the upstream change
before they are accepted.
+```shell
+mise run tck -- snapshots check go
+```
-5. Verify that generation is reproducible at the new pin and run the
repository checks:
+Check mode does not modify the repository. It fails for added or deleted files
and for content changes to deterministic snapshots. It reports, but allows,
content changes to existing snapshots classified as probabilistic by
`internal/snapshots/stability.go`.
- ```shell
- mise run tck -- snapshots check go
- mise run check
- ```
+This command answers whether the repository matches its pin; it does not
determine whether the pin is the latest upstream commit.
- A second generation may report allowed modifications for known
probabilistic snapshots. The file set and deterministic contents must reproduce.
+## Synchronize configured snapshots
-Use `all` instead of a language only when intentionally refreshing every
source implementation. Updating languages separately usually produces smaller,
easier-to-review pull requests.
+Regenerate every snapshot directory from the repositories and commits
currently recorded in `config.toml`:
-## Check the pinned corpus
+```shell
+mise run tck -- snapshots sync
+```
-Use check mode to regenerate snapshots from the currently pinned commit and
compare them with the committed corpus:
+Sync may change the committed snapshot directories, but it never changes
`config.toml`.
+
+## Set up the toolchain
+
+Install [mise](https://mise.jdx.dev/), then install the pinned toolchain and
inspect the available snapshot commands:
```shell
-mise run tck -- snapshots check go
+mise install
+mise run tck -- snapshots --help
```
-Check mode does not modify the repository. It fails for added or deleted files
and for content changes to deterministic snapshots. It reports, but allows,
content changes to existing snapshots classified as probabilistic by
`internal/snapshots/stability.go`.
+Mise supplies Go, CMake and CTest, Java, and Maven. Git is required for every
source language, a C++ compiler is required for C++, and Make is required for
Go.
-This command answers whether the repository matches its pin; it does not
determine whether the pin is the latest upstream commit.
+Check mode accepts `cpp`, `go`, `java`, or `all`. Synchronization takes no
arguments, while update requires one source language and revision.
## Use the corpus from an implementation
@@ -105,6 +114,6 @@ Snapshot generation is kept out of the required workflow
because it clones and b
## Implementation notes
-For each requested language, the `tck` command reads the repository and commit
from `internal/snapshots/generator.go`, checks out that revision in a temporary
workspace, and invokes the source-specific adapter in
`internal/snapshots/<language>.go`. It then compares the generated output with
`serialization/<language>/snapshots`; update mode atomically replaces that
directory.
+For each requested language, the `tck` command reads the repository and commit
from `config.toml`, checks out that revision in a temporary workspace, and
invokes the source-specific adapter in `internal/snapshots/<language>.go`. It
then compares the generated output with `serialization/<language>/snapshots`.
The command-line interface and change report live in `cmd/tck`. Reconciliation
and file comparison live in `internal/snapshots`, where `stability.go`
classifies deterministic and known probabilistic outputs.
diff --git a/go.mod b/go.mod
index ad8dc28..202eab9 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ toolchain go1.26.5
require (
github.com/gkampitakis/go-snaps v0.5.23
github.com/jedib0t/go-pretty/v6 v6.8.3
+ github.com/pelletier/go-toml/v2 v2.4.3
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
)
diff --git a/go.sum b/go.sum
index 9bb4d11..e33cfbc 100644
--- a/go.sum
+++ b/go.sum
@@ -24,6 +24,8 @@ github.com/maruel/natural v1.3.0
h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg=
github.com/maruel/natural v1.3.0/go.mod
h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mattn/go-runewidth v0.0.16
h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod
h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pelletier/go-toml/v2 v2.4.3
h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
+github.com/pelletier/go-toml/v2 v2.4.3/go.mod
h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod
h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0
h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod
h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
diff --git a/internal/snapshots/config.go b/internal/snapshots/config.go
new file mode 100644
index 0000000..2746df4
--- /dev/null
+++ b/internal/snapshots/config.go
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package snapshots
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+
+ "github.com/pelletier/go-toml/v2"
+)
+
+const configFilename = "config.toml"
+
+const configFileHeader = `# Licensed to the Apache Software Foundation (ASF)
under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+`
+
+var commitIDPattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
+
+type repositoryConfig struct {
+ Snapshot snapshotSources `toml:"snapshot"`
+}
+
+type snapshotSources struct {
+ CPP snapshotSource `toml:"cpp"`
+ Go snapshotSource `toml:"go"`
+ Java snapshotSource `toml:"java"`
+}
+
+type snapshotSource struct {
+ Repository string `toml:"repository"`
+ Commit string `toml:"commit"`
+}
+
+func loadConfig(repositoryRoot string) (repositoryConfig, []byte, error) {
+ filename := filepath.Join(repositoryRoot, configFilename)
+ content, err := os.ReadFile(filename)
+ if err != nil {
+ return repositoryConfig{}, nil, fmt.Errorf("read TCK config:
%w", err)
+ }
+
+ var config repositoryConfig
+ decoder :=
toml.NewDecoder(bytes.NewReader(content)).DisallowUnknownFields()
+ if err := decoder.Decode(&config); err != nil {
+ return repositoryConfig{}, nil, fmt.Errorf("decode TCK config:
%w", err)
+ }
+ if err := config.validate(); err != nil {
+ return repositoryConfig{}, nil, err
+ }
+ return config, content, nil
+}
+
+func (config repositoryConfig) validate() error {
+ for _, language := range Languages() {
+ source, found := config.source(language)
+ if !found {
+ return fmt.Errorf("snapshot source for %s is not
configured", language)
+ }
+ if source.Repository == "" {
+ return fmt.Errorf("snapshot repository for %s must not
be empty", language)
+ }
+ if !commitIDPattern.MatchString(source.Commit) {
+ return fmt.Errorf("snapshot commit for %s must be a
40-character lowercase commit ID", language)
+ }
+ }
+ return nil
+}
+
+func (config repositoryConfig) source(language string) (snapshotSource, bool) {
+ switch language {
+ case "cpp":
+ return config.Snapshot.CPP, true
+ case "go":
+ return config.Snapshot.Go, true
+ case "java":
+ return config.Snapshot.Java, true
+ default:
+ return snapshotSource{}, false
+ }
+}
+
+func (config *repositoryConfig) setCommit(language, commit string) error {
+ switch language {
+ case "cpp":
+ config.Snapshot.CPP.Commit = commit
+ case "go":
+ config.Snapshot.Go.Commit = commit
+ case "java":
+ config.Snapshot.Java.Commit = commit
+ default:
+ return fmt.Errorf("unsupported snapshot language %q", language)
+ }
+ return config.validate()
+}
+
+func encodeConfig(config repositoryConfig) ([]byte, error) {
+ if err := config.validate(); err != nil {
+ return nil, err
+ }
+ return []byte(fmt.Sprintf(
+ "%s[snapshot.cpp]\nrepository = %q\ncommit = %q\n\n"+
+ "[snapshot.go]\nrepository = %q\ncommit = %q\n\n"+
+ "[snapshot.java]\nrepository = %q\ncommit = %q\n",
+ configFileHeader,
+ config.Snapshot.CPP.Repository,
+ config.Snapshot.CPP.Commit,
+ config.Snapshot.Go.Repository,
+ config.Snapshot.Go.Commit,
+ config.Snapshot.Java.Repository,
+ config.Snapshot.Java.Commit,
+ )), nil
+}
+
+func replaceConfigFile(repositoryRoot string, content []byte) error {
+ target := filepath.Join(repositoryRoot, configFilename)
+ transaction, err := os.MkdirTemp(repositoryRoot, ".tck-config-")
+ if err != nil {
+ return fmt.Errorf("create config update transaction: %w", err)
+ }
+ defer func() { _ = os.RemoveAll(transaction) }()
+
+ next := filepath.Join(transaction, "next")
+ if err := os.WriteFile(next, content, 0o644); err != nil {
+ return fmt.Errorf("stage TCK config: %w", err)
+ }
+ previous := filepath.Join(transaction, "previous")
+ if err := os.Rename(target, previous); err != nil {
+ return fmt.Errorf("preserve current TCK config: %w", err)
+ }
+ if err := os.Rename(next, target); err != nil {
+ _ = os.Rename(previous, target)
+ return fmt.Errorf("install TCK config: %w", err)
+ }
+ return nil
+}
diff --git a/internal/snapshots/generator.go b/internal/snapshots/generator.go
index 4d6786f..9fccb7f 100644
--- a/internal/snapshots/generator.go
+++ b/internal/snapshots/generator.go
@@ -33,8 +33,6 @@ type generationPaths struct {
type generateFunc func(context.Context, generationPaths, commandRunner) error
type generator struct {
- repository string
- commit string
requirements []string
generate generateFunc
stability func(string) Stability
@@ -42,22 +40,16 @@ type generator struct {
var generators = map[string]generator{
"cpp": {
- repository: "https://github.com/apache/datasketches-cpp.git";,
- commit: "401423367055acdf7502e8ed3126730a08039d91",
requirements: []string{"git", "cmake", "ctest"},
generate: generateCPP,
stability: cppStability,
},
"go": {
- repository: "https://github.com/apache/datasketches-go.git";,
- commit: "730c0ca31e00b8becf8b70591ae8ca73954912d0",
requirements: []string{"git", "go", "make"},
generate: generateGo,
stability: goStability,
},
"java": {
- repository: "https://github.com/apache/datasketches-java.git";,
- commit: "f3b334b380feee9f928500de9adb6bbf763fc104",
requirements: []string{"git", "java", "mvn"},
generate: generateJava,
stability: javaStability,
@@ -75,18 +67,22 @@ func Languages() []string {
func (definition generator) run(
ctx context.Context,
- workspace, destination string,
+ workspace, destination, repository, revision string,
runner commandRunner,
-) error {
+) (string, error) {
paths := generationPaths{
workspace: workspace,
source: filepath.Join(workspace, "source"),
destination: destination,
}
- if err := cloneAtCommit(ctx, runner, definition.repository,
definition.commit, paths.source); err != nil {
- return err
+ resolvedRevision, err := cloneAtRevision(ctx, runner, repository,
revision, paths.source)
+ if err != nil {
+ return "", err
}
- return definition.generate(ctx, paths, runner)
+ if err := definition.generate(ctx, paths, runner); err != nil {
+ return "", err
+ }
+ return resolvedRevision, nil
}
type commandRunner struct {
diff --git a/internal/snapshots/reconcile.go b/internal/snapshots/reconcile.go
index 168e125..f87a6b4 100644
--- a/internal/snapshots/reconcile.go
+++ b/internal/snapshots/reconcile.go
@@ -31,12 +31,19 @@ type Mode string
const (
ModeCheck Mode = "check"
+ ModeSync Mode = "sync"
ModeUpdate Mode = "update"
)
type Result struct {
- Target string
- Changes []Change
+ Target string
+ PreviousRevision string
+ Revision string
+ Changes []Change
+}
+
+func (result Result) RevisionChanged() bool {
+ return result.PreviousRevision != result.Revision
}
func (result Result) HasBlockingChanges() bool {
@@ -57,15 +64,34 @@ func Reconcile(
ctx context.Context,
repositoryRoot, language string,
mode Mode,
+ requestedRevision string,
stdout, stderr io.Writer,
) (Result, error) {
- if mode != ModeCheck && mode != ModeUpdate {
+ if mode != ModeCheck && mode != ModeSync && mode != ModeUpdate {
return Result{}, fmt.Errorf("unsupported reconciliation mode
%q", mode)
}
generator, found := generators[language]
if !found {
return Result{}, fmt.Errorf("unsupported snapshot language %q",
language)
}
+ if mode == ModeUpdate && requestedRevision == "" {
+ return Result{}, fmt.Errorf("a source revision is required in
update mode")
+ }
+ if mode != ModeUpdate && requestedRevision != "" {
+ return Result{}, fmt.Errorf("a source revision can only be
selected in update mode")
+ }
+ config, originalConfig, err := loadConfig(repositoryRoot)
+ if err != nil {
+ return Result{}, err
+ }
+ source, found := config.source(language)
+ if !found {
+ return Result{}, fmt.Errorf("snapshot source for unsupported
language %q", language)
+ }
+ sourceRevision := source.Commit
+ if requestedRevision != "" {
+ sourceRevision = requestedRevision
+ }
for _, requirement := range generator.requirements {
if _, err := exec.LookPath(requirement); err != nil {
return Result{}, fmt.Errorf("required command %q is not
installed or not on PATH", requirement)
@@ -80,7 +106,15 @@ func Reconcile(
generated := filepath.Join(workspace, "generated")
runner := commandRunner{stdout: stdout, stderr: stderr}
- if err := generator.run(ctx, workspace, generated, runner); err != nil {
+ resolvedRevision, err := generator.run(
+ ctx,
+ workspace,
+ generated,
+ source.Repository,
+ sourceRevision,
+ runner,
+ )
+ if err != nil {
return Result{}, fmt.Errorf("generate %s snapshots: %w",
language, err)
}
@@ -90,9 +124,49 @@ func Reconcile(
return Result{}, err
}
- result := Result{Target: target, Changes: changes}
- if mode == ModeUpdate && len(changes) > 0 {
+ result := Result{
+ Target: target,
+ PreviousRevision: source.Commit,
+ Revision: resolvedRevision,
+ Changes: changes,
+ }
+ if mode == ModeCheck {
+ return result, nil
+ }
+ if mode == ModeSync {
+ if len(changes) > 0 {
+ if err := replaceDirectory(target, generated); err !=
nil {
+ return Result{}, err
+ }
+ }
+ return result, nil
+ }
+
+ revisionChanged := result.RevisionChanged()
+ if revisionChanged {
+ if err := config.setCommit(language, resolvedRevision); err !=
nil {
+ return Result{}, err
+ }
+ updatedConfig, err := encodeConfig(config)
+ if err != nil {
+ return Result{}, err
+ }
+ if err := replaceConfigFile(repositoryRoot, updatedConfig); err
!= nil {
+ return Result{}, err
+ }
+ }
+ if len(changes) > 0 {
if err := replaceDirectory(target, generated); err != nil {
+ if !revisionChanged {
+ return Result{}, err
+ }
+ if restoreErr := replaceConfigFile(repositoryRoot,
originalConfig); restoreErr != nil {
+ return Result{}, fmt.Errorf(
+ "%w; restoring the previous config also
failed: %v",
+ err,
+ restoreErr,
+ )
+ }
return Result{}, err
}
}
diff --git a/internal/snapshots/runner.go b/internal/snapshots/runner.go
index 91e3759..f316b5c 100644
--- a/internal/snapshots/runner.go
+++ b/internal/snapshots/runner.go
@@ -18,6 +18,7 @@
package snapshots
import (
+ "bytes"
"context"
"fmt"
"os/exec"
@@ -40,22 +41,49 @@ func (runner commandRunner) run(ctx context.Context,
directory, name string, arg
return nil
}
-func cloneAtCommit(
+func (runner commandRunner) output(ctx context.Context, directory, name
string, arguments ...string) (string, error) {
+ commandLine := strings.Join(append([]string{name}, arguments...), " ")
+ if _, err := fmt.Fprintf(runner.stdout, "$ %s\n", commandLine); err !=
nil {
+ return "", fmt.Errorf("write command trace: %w", err)
+ }
+
+ var output bytes.Buffer
+ command := exec.CommandContext(ctx, name, arguments...)
+ command.Dir = directory
+ command.Stdout = &output
+ command.Stderr = runner.stderr
+ if err := command.Run(); err != nil {
+ return "", fmt.Errorf("run %s: %w", commandLine, err)
+ }
+ return strings.TrimSpace(output.String()), nil
+}
+
+func cloneAtRevision(
ctx context.Context,
runner commandRunner,
- repository, commit, destination string,
-) error {
+ repository, revision, destination string,
+) (string, error) {
+ if revision == "" || strings.TrimSpace(revision) != revision ||
strings.HasPrefix(revision, "-") {
+ return "", fmt.Errorf("invalid source revision %q", revision)
+ }
if err := runner.run(ctx, "", "git", "init", "--quiet", destination);
err != nil {
- return err
+ return "", err
}
if err := runner.run(ctx, "", "git", "-C", destination, "remote",
"add", "origin", repository); err != nil {
- return err
+ return "", err
}
- if err := runner.run(ctx, "", "git", "-C", destination, "fetch",
"--depth", "1", "--no-tags", "origin", commit); err != nil {
- return err
+ if err := runner.run(ctx, "", "git", "-C", destination, "fetch",
"--depth", "1", "--no-tags", "origin", revision); err != nil {
+ return "", err
}
if err := runner.run(ctx, "", "git", "-C", destination, "checkout",
"--quiet", "--detach", "FETCH_HEAD"); err != nil {
- return err
+ return "", err
}
- return nil
+ resolvedRevision, err := runner.output(ctx, "", "git", "-C",
destination, "rev-parse", "HEAD")
+ if err != nil {
+ return "", err
+ }
+ if !commitIDPattern.MatchString(resolvedRevision) {
+ return "", fmt.Errorf("resolved source revision %q is not a
40-character lowercase commit ID", resolvedRevision)
+ }
+ return resolvedRevision, nil
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
