zeroshade commented on a change in pull request #10951: URL: https://github.com/apache/arrow/pull/10951#discussion_r700441345
########## File path: go/parquet/metadata/app_version.go ########## @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "regexp" + "strconv" + "strings" + + "github.com/apache/arrow/go/parquet" + "github.com/apache/arrow/go/parquet/schema" +) + +var ( + // Regular expression for the version format + // major . minor . patch unknown - prerelease.x + build info + // Eg: 1.5.0ab-cdh5.5.0+cd + versionRx = regexp.MustCompile(`^(\d+)\.(\d+)\.(\d+)([^-+]*)?(?:-([^+]*))?(?:\+(.*))?$`) + // Regular expression for the application format + // application_name version VERSION_FORMAT (build build_name) + // Eg: parquet-cpp version 1.5.0ab-xyz5.5.0+cd (build abcd) + applicationRx = regexp.MustCompile(`^(.*?)\s*(?:(version\s*(?:([^(]*?)\s*(?:\(\s*build\s*([^)]*?)\s*\))?)?)?)$`) + + // Parquet816FixedVersion is the version used for fixing PARQUET-816 + // that changed the padding calculations for dictionary headers on row groups. + Parquet816FixedVersion = NewAppVersionExplicit("parquet-mr", 1, 2, 9) + parquet251FixedVersion = NewAppVersionExplicit("parquet-mr", 1, 8, 0) + parquetCPPFixedStatsVersion = NewAppVersionExplicit("parquet-cpp", 1, 3, 0) + parquetMRFixedStatsVersion = NewAppVersionExplicit("parquet-mr", 1, 10, 0) +) + +// AppVersion represents a specific application version either read from +// or written to a parquet file. +type AppVersion struct { + App string + Build string + Version struct { + Major int + Minor int + Patch int + Unknown string + PreRelease string + BuildInfo string + } +} + +// NewAppVersionExplicit is a convenience function to construct a specific +// application version from the given app string and version +func NewAppVersionExplicit(app string, major, minor, patch int) *AppVersion { + v := &AppVersion{App: app} + v.Version.Major = major + v.Version.Minor = minor + v.Version.Patch = patch + return v +} + +// NewAppVersion parses a "created by" string such as "parquet-go 1.0.0". +// +// It also supports handling pre-releases and build info such as +// parquet-cpp version 1.5.0ab-xyz5.5.0+cd (build abcd) +func NewAppVersion(createdby string) *AppVersion { + v := &AppVersion{} + + var ver []string + + m := applicationRx.FindStringSubmatch(strings.ToLower(createdby)) + if len(m) >= 4 { + v.App = m[1] + v.Build = m[4] + ver = versionRx.FindStringSubmatch(m[3]) + } else { + v.App = "unknown" + } + + if len(ver) >= 7 { + v.Version.Major, _ = strconv.Atoi(ver[1]) + v.Version.Minor, _ = strconv.Atoi(ver[2]) + v.Version.Patch, _ = strconv.Atoi(ver[3]) + v.Version.Unknown = ver[4] + v.Version.PreRelease = ver[5] + v.Version.BuildInfo = ver[6] + } + return v +} + +// LessThan compares the app versions and returns true if this version +// is "less than" the passed version. +// +// If the apps don't match, this always returns false. Otherwise it compares +// the major versions first, then the minor versions, and finally the patch +// versions. +// +// Pre-release and build info are not considered. +func (v AppVersion) LessThan(other *AppVersion) bool { + switch { + case v.App != other.App: + return false + case v.Version.Major < other.Version.Major: + return true + case v.Version.Major > other.Version.Major: + return false + case v.Version.Minor < other.Version.Minor: + return true + case v.Version.Minor > other.Version.Minor: + return false + } + + return v.Version.Patch < other.Version.Patch +} + +// Equal only compares the Application and major/minor/patch versions. +// +// Pre-release and build info are not considered. +func (v AppVersion) Equal(other *AppVersion) bool { + return v.App == other.App && + v.Version.Major == other.Version.Major && + v.Version.Minor == other.Version.Minor && + v.Version.Patch == other.Version.Patch +} + +// HasCorrectStatistics checks whether or not the statistics are valid to be used +// based on the primitive type and the version since previous versions had issues with +// properly computing stats. +// +// Reference: parquet-cpp/src/parquet/metadata.cc +// +// PARQUET-686 has more discussion on statistics Review comment: added check for parquet-cpp-arrow version 4.0.0 to check decimal stats etc. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
