This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch issue-12-latest-snapshots in repository https://gitbox.apache.org/repos/asf/datasketches-tck.git
commit e13811ab69d87254474299f59d3e1397c93fa475 Author: Lee Rhodes <[email protected]> AuthorDate: Sun Oct 4 16:06:38 2026 -0700 Make the latest snapshots from every language the default `snapshots update` now takes `all` and an optional revision. Without a revision, a source is updated to its default branch, so `snapshots update all` refreshes every source in one command. Pinning a specific branch, tag, or commit for one source remains available for reproducing a release or holding a source back. When several sources are processed, a failure in one no longer stops the others; the command reports every failed source and exits non-zero. An update whose commit is unchanged and whose output differs only in probabilistic snapshots no longer rewrites them, so repeated updates do not add binary churn. A new, non-required workflow runs `snapshots update all` every Monday and on demand, commits the result to main, and fails visibly when a source cannot be generated. The docs now describe main as the latest self-tested snapshots from every source and recommend that consumers test against it by default. Closes #12 Co-Authored-By: Claude Opus 5.5 <[email protected]> --- .github/workflows/update-snapshots.yml | 87 ++++++++++++++++++++++++++++++++++ cmd/tck/__snapshots__/output_test.snap | 12 +++++ cmd/tck/__snapshots__/root_test.snap | 2 +- cmd/tck/output.go | 9 ++++ cmd/tck/output_test.go | 20 ++++++++ cmd/tck/root_test.go | 24 ++++++++++ cmd/tck/snapshots.go | 47 +++++++++++++----- docs/serialization-snapshots.md | 39 +++++++++++---- internal/snapshots/reconcile.go | 13 +++++ 9 files changed, 230 insertions(+), 23 deletions(-) diff --git a/.github/workflows/update-snapshots.yml b/.github/workflows/update-snapshots.yml new file mode 100644 index 0000000..8ec1500 --- /dev/null +++ b/.github/workflows/update-snapshots.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Publishes the latest self-tested snapshots from every source language, so +# that main always holds the newest snapshot set. This workflow is not a +# required check. + +name: Update snapshots + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + update: + name: Update snapshots from every source + if: github.repository == 'apache/datasketches-tck' + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + + - name: Setup mise + uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3.6.3 + + - name: Update snapshots + id: update + run: | + set +e + mise run tck -- snapshots update all 2>&1 | tee update.log + status=${PIPESTATUS[0]} + set -e + echo "status=${status}" >> "$GITHUB_OUTPUT" + { + echo '## Snapshot update' + echo + echo '```' + grep -E '^(Update |Summary:|Source revision:|✓|✗|Error:)' update.log || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + # Sources that updated successfully are published even when another + # source failed, so one broken upstream does not hold back the others. + - name: Commit updated snapshots + run: | + if [[ -z "$(git status --porcelain -- config.toml serialization)" ]]; then + echo "Snapshots are already up to date." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -- config.toml serialization + { + echo "chore: update snapshots to the latest upstream revisions" + echo + grep -E '^(Update |Summary:|Source revision:)' update.log || true + } > commit-message.txt + git commit --quiet --file commit-message.txt + git push origin HEAD:${{ github.ref_name }} + + - name: Report failed sources + if: steps.update.outputs.status != '0' + run: | + grep -E '^(✗|Error:)' update.log || true + exit 1 diff --git a/cmd/tck/__snapshots__/output_test.snap b/cmd/tck/__snapshots__/output_test.snap index 4521ec0..36a5bf0 100644 --- a/cmd/tck/__snapshots__/output_test.snap +++ b/cmd/tck/__snapshots__/output_test.snap @@ -24,3 +24,15 @@ Summary: 1 added · 1 modified (1 probabilistic) · 1 deleted ✗ serialization/go/snapshots has 2 blocking changes. --- + +[TestPrintResultReportsUnwrittenUpdate - 1] +╭────────┬───────────┬───────┬──────────────────┬─────────────────────────╮ +│ CHANGE │ STABILITY │ CHECK │ SNAPSHOT │ CONTENT (SIZE · SHA256) │ +├────────┼───────────┼───────┼──────────────────┼─────────────────────────┤ +│ M │ unstable │ allow │ probabilistic.sk │ old 12 B · aaaaaaaaaaaa │ +│ │ │ │ │ new 12 B · bbbbbbbbbbbb │ +╰────────┴───────────┴───────┴──────────────────┴─────────────────────────╯ +Summary: 0 added · 1 modified (1 probabilistic) · 0 deleted +✓ snapshots already matches its source revision; 1 probabilistic content modification was not written. + +--- diff --git a/cmd/tck/__snapshots__/root_test.snap b/cmd/tck/__snapshots__/root_test.snap index 7442f13..c578f57 100644 --- a/cmd/tck/__snapshots__/root_test.snap +++ b/cmd/tck/__snapshots__/root_test.snap @@ -9,7 +9,7 @@ Usage: Available Commands: check Check the snapshot set and stable snapshot contents sync Synchronize snapshots from config.toml - update Adopt one upstream revision and update its snapshots + update Adopt upstream revisions and update their snapshots Flags: -h, --help help for snapshots diff --git a/cmd/tck/output.go b/cmd/tck/output.go index 401e04b..4776f46 100644 --- a/cmd/tck/output.go +++ b/cmd/tck/output.go @@ -85,6 +85,15 @@ func printResult(output io.Writer, root string, mode snapshots.Mode, result snap } switch { + case (mode == snapshots.ModeUpdate || mode == snapshots.ModeSync) && !result.Written: + _, err := fmt.Fprintf( + output, + "✓ %s already matches its source revision; %d probabilistic content %s not written.\n", + target, + unstableModified, + plural(unstableModified, "modification was", "modifications were"), + ) + return err case mode == snapshots.ModeUpdate: _, err := fmt.Fprintf(output, "✓ Updated %s.\n", target) return err diff --git a/cmd/tck/output_test.go b/cmd/tck/output_test.go index 3552291..20d59d7 100644 --- a/cmd/tck/output_test.go +++ b/cmd/tck/output_test.go @@ -74,6 +74,26 @@ func TestPrintResultAllowsOnlyUnstableModifications(t *testing.T) { snaps.WithConfig(snaps.Raw()).MatchSnapshot(t, output.String()) } +func TestPrintResultReportsUnwrittenUpdate(t *testing.T) { + t.Parallel() + + before := testMetadata(12, 0xaa) + after := testMetadata(12, 0xbb) + result := snapshots.Result{ + Target: "snapshots", + PreviousRevision: "a", + Revision: "a", + Changes: []snapshots.Change{{ + Status: snapshots.ChangeModified, Stability: snapshots.Unstable, + Path: "probabilistic.sk", Before: &before, After: &after, + }}, + } + + var output bytes.Buffer + require.NoError(t, printResult(&output, ".", snapshots.ModeUpdate, result)) + snaps.WithConfig(snaps.Raw()).MatchSnapshot(t, output.String()) +} + func testMetadata(size int64, fill byte) snapshots.FileMetadata { metadata := snapshots.FileMetadata{Size: size} for index := range metadata.SHA256 { diff --git a/cmd/tck/root_test.go b/cmd/tck/root_test.go index 5741851..ce7dd05 100644 --- a/cmd/tck/root_test.go +++ b/cmd/tck/root_test.go @@ -46,3 +46,27 @@ func TestRunRejectsUnsupportedLanguage(t *testing.T) { require.Empty(t, output.String()) require.Contains(t, stderr.String(), `invalid argument "python" for "tck snapshots check"`) } + +func TestRunRejectsRevisionForAllUpdates(t *testing.T) { + t.Parallel() + + var output bytes.Buffer + var stderr bytes.Buffer + exitCode := run(t.Context(), []string{"snapshots", "update", "all", "main"}, &output, &stderr) + + require.Equal(t, 1, exitCode) + require.Empty(t, output.String()) + require.Contains(t, stderr.String(), "a revision cannot be selected with all") +} + +func TestRunRejectsUnsupportedUpdateLanguage(t *testing.T) { + t.Parallel() + + var output bytes.Buffer + var stderr bytes.Buffer + exitCode := run(t.Context(), []string{"snapshots", "update", "python"}, &output, &stderr) + + require.Equal(t, 1, exitCode) + require.Empty(t, output.String()) + require.Contains(t, stderr.String(), `unsupported snapshot language "python"`) +} diff --git a/cmd/tck/snapshots.go b/cmd/tck/snapshots.go index be11fc6..3379a6f 100644 --- a/cmd/tck/snapshots.go +++ b/cmd/tck/snapshots.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "github.com/apache/datasketches-tck/internal/snapshots" @@ -80,24 +81,34 @@ func newSnapshotSyncCommand() *cobra.Command { } func newSnapshotUpdateCommand() *cobra.Command { + validLanguages := append(snapshots.Languages(), "all") return &cobra.Command{ - Use: "update <cpp|go|java> <revision>", + Use: "update <cpp|go|java|all> [revision]", Short: reconcileDescription(snapshots.ModeUpdate), Long: reconcileDetails(snapshots.ModeUpdate), Args: func(command *cobra.Command, args []string) error { - if err := cobra.ExactArgs(2)(command, args); err != nil { + if err := cobra.RangeArgs(1, 2)(command, args); err != nil { return err } - for _, language := range snapshots.Languages() { - if args[0] == language { - return nil - } + if !slices.Contains(validLanguages, args[0]) { + return fmt.Errorf("unsupported snapshot language %q", args[0]) + } + if args[0] == "all" && len(args) == 2 { + return errors.New("a revision cannot be selected with all; each source is updated to its default branch") } - return fmt.Errorf("unsupported snapshot language %q", args[0]) + return nil }, - ValidArgs: snapshots.Languages(), + ValidArgs: validLanguages, RunE: func(command *cobra.Command, args []string) error { - return reconcileSnapshots(command, snapshots.ModeUpdate, []string{args[0]}, args[1]) + languages := []string{args[0]} + if args[0] == "all" { + languages = snapshots.Languages() + } + revision := snapshots.DefaultBranch + if len(args) == 2 { + revision = args[1] + } + return reconcileSnapshots(command, snapshots.ModeUpdate, languages, revision) }, } } @@ -114,6 +125,7 @@ func reconcileSnapshots( } outOfDate := false + var failed []string for index, language := range languages { if index > 0 { if _, err := fmt.Fprintln(command.OutOrStdout()); err != nil { @@ -139,13 +151,24 @@ func reconcileSnapshots( command.ErrOrStderr(), ) if err != nil { - return err + if len(languages) == 1 { + return err + } + // Keep going so one broken upstream does not hold back the others. + failed = append(failed, languageHeading(language)) + if _, printErr := fmt.Fprintf(command.ErrOrStderr(), "✗ %v\n", err); printErr != nil { + return fmt.Errorf("print snapshot failure: %w", printErr) + } + continue } if err := printResult(command.OutOrStdout(), root, mode, result); err != nil { return fmt.Errorf("print snapshot result: %w", err) } outOfDate = outOfDate || (mode == snapshots.ModeCheck && result.HasBlockingChanges()) } + if len(failed) > 0 { + return fmt.Errorf("%s snapshots failed for %s", strings.ToLower(modeHeading(mode)), strings.Join(failed, ", ")) + } if outOfDate { return errSnapshotsOutOfDate } @@ -159,7 +182,7 @@ func reconcileDescription(mode snapshots.Mode) string { case snapshots.ModeSync: return "Synchronize snapshots from config.toml" case snapshots.ModeUpdate: - return "Adopt one upstream revision and update its snapshots" + return "Adopt upstream revisions and update their snapshots" default: panic(fmt.Sprintf("unsupported snapshot mode %q", mode)) } @@ -172,7 +195,7 @@ func reconcileDetails(mode snapshots.Mode) string { case snapshots.ModeSync: return "Regenerate all snapshot directories from the repositories and commits in config.toml without changing the config." case snapshots.ModeUpdate: - return "Resolve an upstream commit, branch, or tag, record its exact commit ID in config.toml, and regenerate that source's snapshots." + return "Resolve an upstream commit, branch, or tag, record its exact commit ID in config.toml, and regenerate that source's snapshots. Without a revision, the source's default branch is used; all updates every source to its default branch. Snapshots are left untouched when the revision is unchanged and only probabilistic snapshots differ." default: panic(fmt.Sprintf("unsupported snapshot mode %q", mode)) } diff --git a/docs/serialization-snapshots.md b/docs/serialization-snapshots.md index 95b1af9..50ee001 100644 --- a/docs/serialization-snapshots.md +++ b/docs/serialization-snapshots.md @@ -19,7 +19,9 @@ The serialization corpus is a compatibility boundary between DataSketches implementations. Each directory under `serialization/<language>/snapshots` contains sketches produced by one implementation and intended to be read and validated by the others. -This repository generates snapshots from the upstream sources configured in `config.toml` instead of following the latest branch. Each source records its repository and exact commit so a checkout remains reproducible. +The `main` branch of this repository publishes the latest self-tested snapshots from every source language. Each language validates its own snapshots with its generators and round-trip tests before they reach the TCK, so a snapshot set that passes there is ready to publish. Whether each implementation can read the others' snapshots is then answered by running its cross-language tests against the latest set, not by reviewing the `.sk` files by hand. + +`config.toml` records the repository and exact commit each snapshot directory was generated from, so any TCK commit remains reproducible: ```toml [snapshot.go] @@ -27,26 +29,41 @@ repository = "https://github.com/apache/datasketches-go.git"; commit = "730c0ca31e00b8becf8b70591ae8ca73954912d0" ``` +## Update to the latest snapshots + +With the [toolchain installed](#set-up-the-toolchain), update every source to its default branch: + +```shell +mise run tck -- snapshots update all +``` + +For each source, the command resolves the default branch to an exact commit ID, writes it to `config.toml`, and regenerates that source's snapshot directory. Generation happens before either file set is changed, so a source whose generation fails is left untouched; the remaining sources are still updated and the command exits with an error naming the failed sources. + +When a source's commit has not changed and only probabilistic snapshots differ, its snapshots are left as they are, so repeated updates do not add binary churn to the history. + +The [Update snapshots](#github-actions) workflow runs this command weekly and commits the result to `main`. + ## Update one source -With the [toolchain installed](#set-up-the-toolchain), adopt a different upstream branch, tag, or commit by specifying one source language and revision: +Update a single source to its default branch, or to a specific branch, tag, or commit: ```shell -mise run tck -- snapshots update go main +mise run tck -- snapshots update go +mise run tck -- snapshots update go v0.2.0 ``` -The command resolves the supplied revision to an exact commit ID, writes it to `[snapshot.go]` in `config.toml`, and regenerates `serialization/go/snapshots`. Generation happens before either file set is changed, so a generation failure leaves the repository untouched. +Pinning an older revision is useful for reproducing a release or for holding a source back while an upstream problem is fixed. If the upstream build or output layout changed, update the corresponding adapter in `internal/snapshots/<language>.go` before running the command. -Updates intentionally handle one source at a time so each upstream change can be reviewed independently. If the upstream build or output layout changed, update the corresponding adapter in `internal/snapshots/<language>.go` before running the command. +## Review an update -Review the resolved pin and corpus together: +Review the resolved pins and the corpus together: ```shell git diff --stat git diff -- config.toml serialization/go/snapshots ``` -Added and deleted files change the set of compatibility cases. Unexpected changes to deterministic files should be understood from the upstream change before they are accepted. +Most modified files are probabilistic snapshots, which are expected to change on every generation. The update report classifies each change, so the meaningful part of a review is short: added and deleted files change the set of compatibility cases, and changes to deterministic snapshots, such as a flag bit, should be understood from the upstream change. Verify that generation is reproducible at the new pin and run the repository checks: @@ -67,7 +84,7 @@ mise run tck -- snapshots check go Check mode does not modify the repository. It fails for added or deleted files and for content changes to deterministic snapshots. It reports, but allows, content changes to existing snapshots classified as probabilistic by `internal/snapshots/stability.go`. -This command answers whether the repository matches its pin; it does not determine whether the pin is the latest upstream commit. +This command answers whether the repository matches its pin; it does not determine whether the pin is the latest upstream commit. Use update mode for that. ## Synchronize configured snapshots @@ -90,7 +107,7 @@ mise run tck -- snapshots --help Mise supplies Go, CMake and CTest, Java, and Maven. Git is required for every source language, a C++ compiler is required for C++, and Make is required for Go. -Check mode accepts `cpp`, `go`, `java`, or `all`. Synchronization takes no arguments, while update requires one source language and revision. +Check mode accepts `cpp`, `go`, `java`, or `all`. Synchronization takes no arguments. Update accepts `cpp`, `go`, `java`, or `all`, and an optional revision for a single source. ## Use the corpus from an implementation @@ -98,6 +115,8 @@ An implementation consumes the `.sk` files as test fixtures. Its compatibility t This repository centralizes the fixture corpus and source-side generation. Consumer tests remain in the individual DataSketches implementation repositories. +By default, an implementation should test against the latest snapshot set on `main`. A failure then shows exactly which source language and sketch disagree, as early as possible. To keep a release or a CI run from changing underneath it, an implementation can instead pin a specific TCK commit. + ## Review policy for probabilistic snapshots Some upstream generators contain randomness, so byte-for-byte reproduction is not a valid invariant for every file. The stability policy is source-specific because upstream implementations do not always seed or exercise an algorithm in the same way. @@ -110,7 +129,7 @@ The probabilistic classification only controls byte-level comparison in this rep `.github/workflows/check.yml` runs `mise run check` for pull requests and pushes to `main`. It validates the Go implementation of the TCK tooling, but it does not run the upstream snapshot generators or modify committed snapshots. -Snapshot generation is kept out of the required workflow because it clones and builds three external projects, and because an automated update cannot decide whether an upstream compatibility change should be adopted. A pull request that updates a pin should include the generated corpus changes and record which source-language checks were run locally. +`.github/workflows/update-snapshots.yml` runs `snapshots update all` every Monday and on manual dispatch, and commits any updated pins and snapshots directly to `main`. It is not a required check. The job summary lists each source's revision change and change counts. If a source fails to generate, the other sources are still committed and the run fails so that the broken upstream is noticed. ## Implementation notes diff --git a/internal/snapshots/reconcile.go b/internal/snapshots/reconcile.go index f87a6b4..84b554c 100644 --- a/internal/snapshots/reconcile.go +++ b/internal/snapshots/reconcile.go @@ -29,6 +29,10 @@ import ( type Mode string +// DefaultBranch is the revision that selects a source repository's default +// branch: fetching HEAD from a remote resolves to the branch it points to. +const DefaultBranch = "HEAD" + const ( ModeCheck Mode = "check" ModeSync Mode = "sync" @@ -40,6 +44,8 @@ type Result struct { PreviousRevision string Revision string Changes []Change + // Written reports whether the snapshot directory was replaced. + Written bool } func (result Result) RevisionChanged() bool { @@ -138,11 +144,17 @@ func Reconcile( if err := replaceDirectory(target, generated); err != nil { return Result{}, err } + result.Written = true } return result, nil } revisionChanged := result.RevisionChanged() + if !revisionChanged && !result.HasBlockingChanges() { + // Nothing new upstream. Rewriting the probabilistic snapshots would only + // add binary churn to the history without adopting anything. + return result, nil + } if revisionChanged { if err := config.setCommit(language, resolvedRevision); err != nil { return Result{}, err @@ -169,6 +181,7 @@ func Reconcile( } return Result{}, err } + result.Written = true } return result, nil } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
