This is an automated email from the ASF dual-hosted git repository. Amar3tto pushed a commit to branch icerbergio-benchmark in repository https://gitbox.apache.org/repos/asf/beam.git
commit 7fb07784bf2c7b01480d65cc1d8319599982f58b Author: Vitaly Terentyev <[email protected]> AuthorDate: Thu Jul 9 14:42:45 2026 +0400 Add looker metrics for IcebergIO --- .../beam_PostCommit_Java_IO_Performance_Tests.yml | 2 +- .test-infra/tools/refresh_looker_metrics.py | 2 + website/www/site/content/en/performance/_index.md | 1 + .../content/en/performance/icebergio/_index.md | 50 ++++++++++++++++++++++ website/www/site/data/performance.yaml | 31 ++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml index 0bc0a37ea4e..9bda3d1bf31 100644 --- a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml +++ b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml @@ -64,7 +64,7 @@ jobs: matrix: job_name: ["beam_PostCommit_Java_IO_Performance_Tests"] job_phrase: ["Run Java PostCommit IO Performance Tests"] - test_case: ["IcebergPerformanceTest"] + test_case: ["IcebergPerformanceTest", "GCSPerformanceTest", "BigTablePerformanceTest", "BigQueryStorageApiStreamingPerformanceTest"] steps: - uses: actions/checkout@v7 - name: Setup repository diff --git a/.test-infra/tools/refresh_looker_metrics.py b/.test-infra/tools/refresh_looker_metrics.py index c8d66f4a4bd..573cee0d4e2 100644 --- a/.test-infra/tools/refresh_looker_metrics.py +++ b/.test-infra/tools/refresh_looker_metrics.py @@ -34,6 +34,8 @@ LOOKS_TO_DOWNLOAD = [ ("33", ["21", "70", "116", "69", "115"]), # BigTableIO_Write ("34", ["22", "56", "96", "55", "95"]), # TextIO_Read ("35", ["23", "64", "110", "63", "109"]), # TextIO_Write + ("113", ["386", "388", "390", "392", "394"]), # IcebergIO_Read + ("114", ["387", "389", "391", "393", "395"]), # IcebergIO_Write ("75", ["258", "259", "260", "261", "262"]), # TensorFlow MNIST ("76", ["233", "234", "235", "236", "237"]), # PyTorch BERT base uncased ("77", ["238", "239", "240", "241", "242"]), # PyTorch BERT large uncased diff --git a/website/www/site/content/en/performance/_index.md b/website/www/site/content/en/performance/_index.md index a0eaba2aa0e..0181965fc10 100644 --- a/website/www/site/content/en/performance/_index.md +++ b/website/www/site/content/en/performance/_index.md @@ -38,6 +38,7 @@ writing to various Beam IOs. - [BigQuery](/performance/bigquery) - [BigTable](/performance/bigtable) - [TextIO](/performance/textio) +- [IcebergIO](/performance/icebergio) # Measured Beam Python ML Pipelines diff --git a/website/www/site/content/en/performance/icebergio/_index.md b/website/www/site/content/en/performance/icebergio/_index.md new file mode 100644 index 00000000000..e618865097b --- /dev/null +++ b/website/www/site/content/en/performance/icebergio/_index.md @@ -0,0 +1,50 @@ +--- +title: "IcebergIO Performance" +--- + +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +# IcebergIO Performance + +The following graphs show various metrics when reading from or writing to Apache Iceberg tables using +IcebergIO. See the [glossary](/performance/glossary) for definitions. + +## Read + +### What is the estimated cost of reading from Apache Iceberg tables using IcebergIO? + +{{< performance_looks io="icebergio" read_or_write="read" section="test_name" >}} + +### How has various metrics changed when reading from Apache Iceberg tables using IcebergIO for different Beam SDK versions? + +{{< performance_looks io="icebergio" read_or_write="read" section="version" >}} + +### How has various metrics changed over time when reading from Apache Iceberg tables using IcebergIO? + +{{< performance_looks io="icebergio" read_or_write="read" section="date" >}} + +## Write + +### What is the estimated cost of writing to Apache Iceberg tables using IcebergIO? + +{{< performance_looks io="icebergio" read_or_write="write" section="test_name" >}} + +### How has various metrics changed when writing to Apache Iceberg tables using IcebergIO for different Beam SDK versions? + +{{< performance_looks io="icebergio" read_or_write="write" section="version" >}} + +### How has various metrics changed over time when writing to Apache Iceberg tables using IcebergIO? + +{{< performance_looks io="icebergio" read_or_write="write" section="date" >}} diff --git a/website/www/site/data/performance.yaml b/website/www/site/data/performance.yaml index 42fb5bee92f..e996ea26291 100644 --- a/website/www/site/data/performance.yaml +++ b/website/www/site/data/performance.yaml @@ -107,6 +107,37 @@ looks: title: AvgInputThroughputBytesPerSec by Version - id: fVVHhXCrHNgBG52TJsTjR8VbmWCCQnVN title: AvgInputThroughputElementsPerSec by Version + icebergio: + read: + folder: 113 + test_name: + - id: x5R7w8VnzkccgzrBTWQ82J7JBNtkGxH2 + title: Read IcebergIO RunTime and EstimatedCost + date: + - id: HQsKkmn7cS7m5bCYzPBYqnnFM6MXvJgV + title: AvgOutputThroughputBytesPerSec by Date + - id: MNYqD2ZCwV2XfYjwDg4NDVT7vr4Ytpz2 + title: AvgOutputThroughputElementsPerSec by Date + version: + - id: Cf7wGQfrM4hn8Wdf64ZjwshrHHdZKmKs + title: AvgOutputThroughputBytesPerSec by Version + - id: MGSX85MhCnCFFWYzSCGS36JRWxcBccqd + title: AvgOutputThroughputElementsPerSec by Version + write: + folder: 114 + test_name: + - id: yvvWB2JbbbFQ4R5MTwF2hbVW2X4bDxnY + title: Write IcebergIO RunTime and EstimatedCost + date: + - id: Gcw6jfhvvkkHKDwqqTkhZtYYNNH2Z8DZ + title: AvgInputThroughputBytesPerSec by Date + - id: RYtRGKsxBRkyRgWk6RGjshFnZPk76vhM + title: AvgInputThroughputElementsPerSec by Date + version: + - id: B5q25ktnXNhHxpYrGmqzgcGMgX26zvM3 + title: AvgInputThroughputBytesPerSec by Version + - id: VfMWk6rk26JQNr8wRpsK5Gk6XQPfd7vW + title: AvgInputThroughputElementsPerSec by Version pytorchbertbase: write: folder: 76
