This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch new-site-dev
in repository https://gitbox.apache.org/repos/asf/pinot-site.git
The following commit(s) were added to refs/heads/new-site-dev by this push:
new 69cc60d7 Added blog posts (#189)
69cc60d7 is described below
commit 69cc60d7a75eb030d92449d8f09077f0516811a4
Author: James Dilworth <[email protected]>
AuthorDate: Wed Aug 19 11:53:29 2026 -0700
Added blog posts (#189)
---
components/MDXComponents.tsx | 2 +
components/YouTube.tsx | 36 +++
data/authors/hongkun.mdx | 13 ++
data/authors/raghav.mdx | 9 +
data/blog/2026-02-14-Year-in-Review-2025.mdx | 140 +++++++++++
data/blog/2026-07-15-Materialized-Views.mdx | 255 +++++++++++++++++++++
public/static/images/authors/hongkun.jpg | Bin 0 -> 98604 bytes
public/static/images/authors/raghav.jpeg | Bin 0 -> 68592 bytes
.../images/blogs/materialized-views/image1.png | Bin 0 -> 83441 bytes
.../images/blogs/materialized-views/image2.png | Bin 0 -> 183640 bytes
.../images/blogs/materialized-views/image3.png | Bin 0 -> 138310 bytes
.../images/blogs/materialized-views/image4.png | Bin 0 -> 19849 bytes
tailwind.config.ts | 4 +-
13 files changed, 458 insertions(+), 1 deletion(-)
diff --git a/components/MDXComponents.tsx b/components/MDXComponents.tsx
index 2fc938af..e9d6e2c8 100644
--- a/components/MDXComponents.tsx
+++ b/components/MDXComponents.tsx
@@ -6,6 +6,7 @@ import Image from './Image';
import CustomLink from './Link';
import TableWrapper from './TableWrapper';
import VideoEmbed from './VideoEmbed';
+import YouTube from './YouTube';
export const components: MDXComponents = {
Image,
@@ -14,5 +15,6 @@ export const components: MDXComponents = {
pre: Pre,
table: TableWrapper,
VideoEmbed,
+ YouTube,
BlogNewsletterForm
};
diff --git a/components/YouTube.tsx b/components/YouTube.tsx
new file mode 100644
index 00000000..3af7712f
--- /dev/null
+++ b/components/YouTube.tsx
@@ -0,0 +1,36 @@
+// Simple YouTube embed for the blgo
+
+type YouTubeProps = {
+ id: string;
+ title?: string;
+};
+
+export function YouTube({ id, title = 'YouTube video' }: YouTubeProps) {
+ return (
+ <div
+ style={{
+ width: '100%',
+ aspectRatio: '16 / 9',
+ marginBlock: '2rem',
+ overflow: 'hidden',
+ borderRadius: '0.5rem'
+ }}
+ >
+ <iframe
+ src={`https://www.youtube-nocookie.com/embed/${id}`}
+ title={title}
+ allow="accelerometer; autoplay; clipboard-write;
encrypted-media; gyroscope; picture-in-picture; web-share"
+ allowFullScreen
+ loading="lazy"
+ style={{
+ display: 'block',
+ width: '100%',
+ height: '100%',
+ border: 0
+ }}
+ />
+ </div>
+ );
+}
+
+export default YouTube;
diff --git a/data/authors/hongkun.mdx b/data/authors/hongkun.mdx
new file mode 100644
index 00000000..efc08e1c
--- /dev/null
+++ b/data/authors/hongkun.mdx
@@ -0,0 +1,13 @@
+---
+name: Hongkun Xu
+avatar: /static/images/authors/hongkun.jpg
+occupation: Senior Software Engineer
+company: Cisco
+linkedin: https://www.linkedin.com/in/hongkun-xu-5517721a6/
+---
+
+I have been actively contributing to the Apache Pinot and StarRocks
communities, with a strong focus on OLAP systems, real-time analytics, and
distributed query engines.
+
+My work and interests center around query optimization, resource management,
execution efficiency, and cross-cluster management. Through long-term
involvement in open-source communities and practical engineering scenarios, I
have developed a deep understanding of how to improve query performance,
resource utilization, and system stability in large-scale distributed
environments.
+
+I am passionate about open-source collaboration and enjoy working on
technically challenging problems at the intersection of distributed systems,
data infrastructure, and analytical databases. I am particularly interested in
building scalable, efficient, and production-ready data platforms that can
support complex analytical workloads with high performance and reliability.
diff --git a/data/authors/raghav.mdx b/data/authors/raghav.mdx
new file mode 100644
index 00000000..3ddea7ef
--- /dev/null
+++ b/data/authors/raghav.mdx
@@ -0,0 +1,9 @@
+---
+name: Raghvendra Yadav
+avatar: /static/images/authors/raghav.jpeg
+occupation: Software Engineer
+company: StarTree
+linkedin: https://www.linkedin.com/in/raghvendra-yadav-b996a520/
+---
+
+Software Engineer @StarTree.ai | Empowering real-time decisions with the speed
of Apache Pinot with StarTree.
diff --git a/data/blog/2026-02-14-Year-in-Review-2025.mdx
b/data/blog/2026-02-14-Year-in-Review-2025.mdx
new file mode 100644
index 00000000..c5c0ae52
--- /dev/null
+++ b/data/blog/2026-02-14-Year-in-Review-2025.mdx
@@ -0,0 +1,140 @@
+---
+title: '2025 Year in Review'
+date: 2026-02-15
+authors: ['raghav']
+summary: 'Unleashing Real-Time Analytics: Insights from the "Apache Pinot Year
in Review 2025 and Look Ahead'
+
+tags: [Apache Pinot, releases, real-time analytics, open source, community
updates]
+---
+
+The "Apache Pinot Year in Review 2025 and Look Ahead" session offered
highlights from the Apache Pinot community, showcasing technical improvements,
community growth, and the transformative adoption of Pinot’s capabilities
across industries. Here's a summary of the key takeaways, with a professional
breakdown of features, organizational use cases, and collaborative endeavors
from contributors that make this video a must-watch for data innovators.
+
+<YouTube
+ id="14GezNkbq3E"
+ title="Apache Pinot in 2025: What’s Advancing Across Query, Ingestion,
and the Core Engine"
+/>
+
+## Community Growth and Adoption
+
+Hosted by Raghav Yadav, an Apache Pinot contributor and Staff Software
Engineer at StarTree, the session began with an overview of the community's
accomplishments. Pinot experienced more growth last year, including:
+
+- 5,800 members actively engaging on Slack
+- 120 contributors delivering approximately 2,000 commits
+- Achieving 6,000 stars on GitHub and impressive numbers for Docker pulls
+
+Raghav highlighted this widespread adoption, including by well-known companies
like Nvidia, Goldman Sachs, DoorDash, Grab, Wix, Slack, and Walmart. He praised
the new committers added to Pinot, Chris Peck (Uber) and Sonam Mandal
(LinkedIn), alongside the new PMC member, Gunjalo.
+
+Key Quote:
+
+> "Adoption across all sectors has been amazing, with new use cases helping us
to evolve and innovate as a community."
+
+## Real-Time Insights from Slack
+
+Sai Tarun Tadakamalla and Zebin Kang from Slack, shared an inspiring success
story about Slack’s migration from batch-based data processing to real-time
analytics using Pinot. This shift improved enterprise dashboards, allowing
customers to monitor message and file exports with exceptional speed.
+
+Slack reduced ingestion latency below 1 second and query latency to 10 seconds
for massive data loads of over 400-500M records at scale. They leveraged
Pinot’s native functions like
[[DistinctCountHLL]](https://docs.pinot.apache.org/functions-1/distinctcounthll)
for hyper log log approximations. And they implemented sorted and inverted
indexes to optimize scale and efficiency.
+
+<YouTube id="o_OYtkxkEMQ" title="Enhanced visibility and dashboards with
Apache Pinot at Slack" />
+
+In 2026, Slack is planning on onboarding Flink alongside Kafka and Pinot for
more comprehensive real-time analytics across their infrastructure. Sai
emphasized collaboration, inviting broader participation from engineers on the
open-source Pinot Slack channel.
+
+Key Quote:
+
+> "With Kafka, Flink, and Pinot, we'll address more real-time analytics needs
in the coming year, opening new opportunities for user monitoring and
engagement."
+
+## AngelOne: Scaling Real-Time Analytics Across Verticals
+
+Milan Chaudi revealed how AngelOne uses Apache Pinot to manage high-capacity
loads for stock trading, anomaly detection, and user onboarding. Alongside
this, custom dashboards provide personalized views for individual users at peak
TPS ingestion rates of 100k and 2M queries/day with P99 latency under 100
milliseconds. Milan highlighted the following enhancements Apache Pinot brought
to AngelOne
+
+- Personalized customer experience
+- Grip - a notification system enables swift decisions and campaign
automation.
+- Prism - a system for user onboarding: Powered by NiFi workflows leveraging
Pinot databases, Prism self-heals and addresses bottlenecks without manual
input.
+- Scalable performance: Codebase contributions improved group-by query
correctness, multi-tenant support, and integration with external libraries like
Roaring Bitmaps.
+
+Milan emphasized the cost and time savings benefits of using Pinot, and
AngelOne's plans to expand Pinot’s role in their business across additional
verticals in the next year.
+
+Key Quote:
+
+> "Pinot’s speed and scalability have driven business transformation, enabling
personalized user workflows and data-driven decision-making across verticals."
+
+<YouTube
+ id="bRyMHHqx4_Y"
+ title="How AngelOne scales real-time analytics across verticals with
Apache Pinot"
+/>
+
+## Walmart: AI-Powered Analytics
+
+Arsheena, representing Walmart, discussed how Pinot is revolutionizing their
last mile delivery system. AI integrations, real-time state tracking, and
reduced resolution time by 50% allow Walmart teams to pinpoint gaps instantly.
Their deployment on GCP supports 50+ tables and plans are underway to expand
this infrastructure to international markets such as Canada and Mexico.
Real-time ingestion via Kafka and Cosmos with visualization and alerting
managed through Apache Superset and Airflow.
+
+<YouTube id="aNllv8rBIWU" title="Enhancing last mile delivery with Apache
Pinot at Walmart" />
+
+Key Quote:
+
+> "Pinot is our one-stop solution for lifecycle tracking, helping reduce
resolution times and enabling intelligent decision-making in real time."
+
+## Enterprise Features and Technical Advancements
+
+Yash Mayya from StarTree introduced advancements to the query engine including:
+
+- Performance: Hundreds of commits added, including improvements for ORDER
BY descending performance and memory safeguards for high-cardinality GROUP BY
queries.
+- Observability: Enhanced query stats visualization with flame graphs and
stage-specific metrics for error reporting and debugging.
+- Join Improvements: Smarter IO reduction through hints for distribution
type, replicated joins, and parallel processing.
+- Various aggregation and window improvements
+
+Key Quote:
+
+> "Lite mode is perfect for dashboards and top queries where the results need
to be bounded but performance is non-negotiable."
+
+### MSE Physical Optimizer:
+
+Shaurya Chatruvedi of Uber, shared how Pinot’s MSE Physical Optimizer reshapes
query plans based on partition metadata, segment placement, and worker
topology, optimizing distributed execution. Key updates include data shuffle
elimination for co-located tables, and intelligent worker assignment through
Helix-based metadata. Another improvement is Lite Mode which follows the
scatter-gather principles of the single-stage engine, but still supports JOIN
and WINDOW functions with limited datasets.
+
+Key Quote:
+
+> "By optimizing physical plans, we reduce IO bottlenecks and streamline
resource allocation. The result is a significantly faster query execution
cycle."
+
+## Real-Time Enhancements: Disaster Recovery
+
+Aman and Hanul revealed updates that boost Pinot’s real-time ingestion. These
include pause-less ingestion for ultra-fresh data pipelines and innovative
disaster recovery for failed segment builds.
+
+Other impactful changes include:
+
+- Byte-based throttling for better memory management.
+- Coordination mechanisms to ensure deduplication accuracy.
+- Real-time segment validation reducing manual intervention.
+
+Byte-based throttling for better memory management.
+
+Coordination mechanisms to ensure deduplication accuracy.
+
+Real-time segment validation reducing manual intervention.
+
+Key Quote:
+
+> "Pause-less ingestion helped us consistently ingest at rates of 300k events
per second while maintaining freshness within 5 seconds."
+
+## Looking Ahead
+
+The final segment featured Chinmay, who unveiled collaborative plans for
Pinot’s roadmap:
+
+- Arrow integration: Deeper in-memory format enhancements.
+- Query isolation: SLA adherence for mixed query workloads.
+- Cost-based optimizer: Cutting-edge improvements for query planning
efficiency.
+
+Arrow integration: Deeper in-memory format enhancements.
+
+Query isolation: SLA adherence for mixed query workloads.
+
+Cost-based optimizer: Cutting-edge improvements for query planning efficiency.
+
+Chinmay invited both feedback and participation, fostering open collaboration
as Pinot evolves.
+
+Key Quote:
+
+> "The roadmap reflects our mission to build a scalable, cutting-edge system
that aligns with the needs of developers and contributors alike."
+
+## Conclusion
+
+This session underscored Apache Pinot’s leadership in real-time analytics,
driven by user stories, community contributions, and technical advancements.
From Slack's latency breakthroughs to Walmart's AI integrations and AngelOne’s
anomaly detection, the ingenuity and collaboration within the Pinot community
promise even greater delivery in 2026\.
+
+If you’re working with real-time analytics or large-scale data processing,
this video will leave you inspired and equipped to leverage Apache Pinot’s
best-in-class offerings.
diff --git a/data/blog/2026-07-15-Materialized-Views.mdx
b/data/blog/2026-07-15-Materialized-Views.mdx
new file mode 100644
index 00000000..12711993
--- /dev/null
+++ b/data/blog/2026-07-15-Materialized-Views.mdx
@@ -0,0 +1,255 @@
+---
+title: 'Building Materialized Views for Predictable Analytics on Apache Pinot'
+date: 2026-07-01
+authors: ['hongkun']
+summary: 'Materialized Views are coming to Apache Pinot in 1.5'
+
+tags: [Apache Pinot, Materialized Views]
+---
+
+In a real-time analytics system, performance is only one part of the problem.
Operators need the cluster to stay healthy when the same expensive aggregation
arrives from a dozen dashboards at once. Users need to know how fresh the data
is — without standing up a separate ETL pipeline for every new metric.
Materialized Views are how we make that contract first-class inside Pinot —
owned, refreshed, and applied to queries by the engine itself.
+
+## Why Pinot needed materialized views
+
+Three scenarios kept showing up in our on-call rotation, our customer
escalations, and our backlog at Webex.
+
+**Cluster overload from repeated work.** A single large fact table receives
dozens of distinct-count aggregations within a short window — same dimensions,
same metrics, only filters or time ranges differ. Pinot recomputes every one
from scratch. P99 spikes within minutes; PagerDuty fires; on-call responds with
scale-up, throttling, or asking customers to stagger their dashboard refreshes.
All firefighting, never a fix.
+
+**Stable query patterns, unstable cost.** Behind every Webex Calling, Webex
Meetings, and Webex Devices dashboard sits the same handful of SQL templates,
varying only in time range, tenant ID, or dimension combination. The user-side
query pattern is fixed. The Pinot-side execution is from-scratch, every time.
It doesn’t trigger alerts — it just quietly burns through a meaningful slice of
cluster compute, every hour of every day.
+
+**External ETL as the only escape hatch.** Users who only need daily
aggregates, but want to keep the detail table as their single source of truth,
are forced to build a daily ETL pipeline _outside_ Pinot — computing aggregates
and writing them into a separate table. The cost is real: T-1 lag, an extra
pipeline to operate, two definitions for one data product, and a small
cross-team project for every new metric.
+
+Three different scenarios, one underlying gap:
+
+**Pinot computes fast, but it has no first-class way of not computing again.**
+
+That’s exactly the problem materialized views have solved in the database
world for decades. Inside Pinot, the acceleration story has historically lived
on the _execution_ side — better indexes, faster scans, stronger
pre-aggregation via Star-Tree. We set out to add the missing piece: a native
materialized view layer that the engine owns, refreshes, and applies to queries
transparently.
+
+## What Pinot already does — and what it doesn’t
+
+Pinot is not short on acceleration. Inverted, range, sorted, bloom, JSON, and
text indexes make narrow-predicate lookups fast at the segment level;
per-segment statistics and pruning keep work proportional to the data that
matters. For pre-aggregation specifically, the Star-tree index is the flagship
— a multi-dimensional cube materialized inside each segment that turns many
aggregation queries into index lookups instead of row scans.
+
+Star-tree shines when the workload is well-defined. Pick the dimensions a
dashboard filters and groups by, keep their cardinality modest, and the tree
turns aggregation queries into very fast lookups. For a fixed-shape dashboard
against a moderate-cardinality table, that is genuinely the right answer.
+
+Two real-world constraints push it past the point where it pays off.
+
+**High-cardinality dimensions blow up the cube.** Star-tree’s storage cost
grows roughly multiplicatively with the cardinality of the indexed dimensions.
Thirty columns at a cardinality of a few hundred each remains workable in
practice — maxLeafRecords truncation plus the fact that real data sits well
below the cartesian product keeps things bounded. Add a single column with
twenty thousand or more distinct values — a tenant ID, a device ID, a session
ID — and the tree explodes. The mit [...]
+
+**Ad-hoc query shapes outrun any predefined cube.** Star-Tree’s indexed
dimensions are set at segment-build time. That fits a fixed-shape dashboard. It
does not fit a workload where analysts are clicking through different filter
and group-by combinations on the same fact table — the dimension set that needs
to be accelerated changes faster than any cube definition can keep up with. You
cannot pre-enumerate all the questions an ad-hoc workload will ask.
+
+Both limits are structural, not configuration choices: they come from what
Star-Tree is — a per-segment, dimension-cube index that must know its
dimensions in advance.
+
+The remaining gap is a different abstraction altogether:
+
+Pinot has no native way to remember a query result shape and rewrite future
queries against it.
+
+Materialized views fit exactly that slot. They operate at a coarser grain,
match queries by subsumption at query time, and handle sketch aggregations
(HLL, theta, …) that Star-Tree’s pre-aggregation framework cannot combine.
Complementary to Star-Tree, not a replacement — and as we will see in the next
section, the entry cost for a team that wants one is a single DDL statement.
+
+## One DDL, no pipelines
+
+The whole point of materialized views is that adding one should not be an ops
project. In Pinot, an MV is created with a single DDL statement, in the same
SQL session the user is already in. No separate service to deploy, no pipeline
to wire up, no schema file to author.
+
+Here is the full DDL behind the cdr_events_daily_dcount view referenced in the
worked example below:
+
+```sql
+CREATE MATERIALIZED VIEW cdr_events_daily_dcount
+REFRESH EVERY 1 HOUR
+PROPERTIES (
+ 'timeColumnName' = 'day',
+ 'bucketTimePeriod' = '1d'
+)
+AS
+SELECT
+ DATE_TRUNC('DAY', callstarttimemillis) AS day,
+ orgid,
+ on_off_net,
+ DISTINCTCOUNTRAWHLL(correlationid) AS correlation_id_hll
+FROM cdr_events
+GROUP BY day, orgid, on_off_net
+```
+
+Three knobs worth noticing:
+
+- `REFRESH EVERY 1 HOUR` — this MV’s own refresh cadence. Omit it and the
cluster-wide MV cron applies.
+- `bucketTimePeriod \= '1d'` — the partition width, and the unit of
freshness tracking we will see in the next-to-next section.
+- No explicit column list — column names and types are inferred from the AS
`SELECT` body, so the DDL really is one statement.
+
+The companion DDL an operator already expects from other warehouses works out
of the box:
+
+- `SHOW MATERIALIZED VIEWS \[IN \<database\>\]`
+- `SHOW CREATE MATERIALIZED VIEW \<name\>`
+- `DROP MATERIALIZED VIEW \[IF EXISTS\] \<name\>`
+
+Same surface, same muscle memory.
+
+
+
+_Data Sources view in the Pinot controller UI: each materialized view is a
first-class entry alongside regular tables, listed with its base table,
freshness state, and last refresh time._
+
+### The column list writes itself （Schema Inference）
+
+That third bullet hides most of the actual implementation. When you omit the
column list, Pinot derives the MV schema from the `AS SELECT` projection at
DDL-compile time. The query is run through Calcite’s validator against the live
cluster catalog, so column typos, wrong arity, multi-source JOINs, and type
mismatches all fail as a 400 the moment you submit the DDL — not at the first
refresh, and not at the first query.
+
+Each output column then maps to one of three shapes:
+
+- **The time column** — the alias matching the `timeColumnName` property —
is pinned to canonical `TIMESTAMP` with `1:MILLISECONDS:TIMESTAMP` format and
`1:MILLISECONDS` granularity. The base column can be a millis-epoch `LONG`; the
MV always stores it as `TIMESTAMP`, and the analyzer separately enforces that
`bucketTimePeriod` divides it cleanly.
+- **Aggregations on the MV allow-list** (`SUM`, `MIN`, `MAX`, `COUNT`, plus
the raw-sketch family `DISTINCTCOUNTRAWHLL`, `DISTINCTCOUNTRAWHLLPLUS`,
`DISTINCTCOUNTRAWTHETASKETCH`) pick their storage type from a small central
catalog (MaterializedViewAggregationCatalog), not from the engine’s surface
type. That distinction is load-bearing for sketches: the engine reports STRING
for `DISTINCTCOUNTRAWHLL(...)` because that is the hex-encoded value a SQL
client receives, but the MV needs th [...]
+- **Everything else** — bare columns, scalar transforms, any function not on
the allow-list — takes Calcite’s validated type and lands as `DIMENSION`, `NOT
NULL`, no default.
+
+The few rules the user does have to follow are mechanical: every computed
expression needs an `AS alias`, aliases must be unique, the timeColumnName
alias must actually appear in the projection, and `SELECT \*` is rejected.
Anything that needs a non-`DIMENSION` role, a multi-value column, a custom
default, or a non-canonical time-column shape falls back to the explicit
column-list form — which we have not yet needed for any of our internal MVs.
+
+Behind the scenes, the compiler turns the DDL into the same `(Schema, OFFLINE
TableConfig)` pair Pinot uses for any offline table, with MV-specific
bookkeeping (`task.MaterializedViewTask.definedSQL`, the resolved Quartz cron)
folded into the task config. The watermark — “how far the MV has confirmed
coverage” — is intentionally not set by the compiler; it starts at zero and
only advances when the scheduler completes its first refresh. That is the
explicit handshake that keeps the rewrit [...]
+
+The rest of the post shows what the engine does the moment this view exists —
without the dashboard team changing a single line of SQL.
+
+## What runs where
+
+Three Pinot processes coordinate through ZooKeeper. The **Controller** owns
the DDL surface, flips MV partitions to `STALE` when base-table segments
change, and emits refresh tasks. The **Broker** reads MV metadata at compile
time to decide whether to rewrite a user query. The **Minion** runs the refresh
task, builds new segments, and advances the per-partition fingerprint and
watermark. Servers execute whichever table the Broker dispatches — unchanged.
+
+
+
+## Same SQL, different execution
+
+Take one of our largest fact tables. `cdr_events` is a Call Detail Record
table holding **more than ten billion rows** across **tens of terabytes** of
storage and **13 months of retention**, with **tens of millions of new rows
landing every day** across **tens of thousands of customer organizations**.
+
+This table sits behind a tenant-facing dashboard with a **time-range picker**
— the user chooses the window. Most stick to the defaults (last 7 or 30 days).
But it is entirely normal for less-experienced users to widen the window to 6
months, or to the full 13-month retention, to “see the long trend.” Every one
of those clicks is a full table scan over the chosen range.
+
+Here is the actual query behind the dashboard’s “Off-Net Calls” tile:
+
+```sql
+SELECT DISTINCTCOUNTHLL(correlationid) AS "Off Net"
+FROM cdr_events
+WHERE on_off_net = 'off_net'
+ AND orgid IN ('4***')
+ AND callstarttimemillis >= 1771804800000 -- 2026-02-23
+ AND callstarttimemillis <= 1774224000000 -- 2026-03-23
+```
+
+_Time bounds align with day boundaries here; off-boundary bounds would
introduce up to one day of drift per edge._
+
+At a 4-week window for a single tenant, this query scans **roughly 150 million
rows** to compute one number — the count of distinct off-net calls — with the
HLL sketch rebuilt from scratch on every replica that touches the request.
End-to-end latency: **just over one second**.
+
+That is for the 4-week default. **Widen the window to 6 months and the scan
grows to roughly 1 billion rows. Widen it to the full 13-month retention and
you are scanning more than 2 billion rows** — to render a single tile. Multiply
by tens of thousands of tenants refreshing their own dashboards independently,
and the cluster overload scenario from earlier is exactly what you get.
+
+Now consider this materialized view:
+
+```sql
+CREATE MATERIALIZED VIEW cdr_events_daily_dcount AS
+SELECT
+ DATE_TRUNC('DAY', callstarttimemillis) AS day,
+ orgid,
+ on_off_net,
+ DISTINCTCOUNTRAWHLL(correlationid) AS correlation_id_hll
+FROM cdr_events
+GROUP BY day, orgid, on_off_net
+```
+
+Three deliberate choices:
+
+- The MV stores `DISTINCTCOUNTRAWHLL` (raw HLL sketch bytes), not the
cardinality number. Numbers cannot be safely combined later; sketches can.
+- The grain is **day × orgid × on_off_net** — every dimension the dashboard
can filter or group by.
+- The partition column is **day**, so any query’s time range can be aligned
to whole-partition boundaries.
+
+With this MV in place, Pinot’s rewrite engine transforms the user’s query into:
+
+```sql
+SELECT DISTINCTCOUNTHLL(correlation_id_hll) AS "Off Net"
+FROM cdr_events_daily_dcount
+WHERE on_off_net = 'off_net'
+ AND orgid IN ('4***')
+ AND day >= 1771804800000
+ AND day <= 1774224000000
+```
+
+The output is identical. The work to compute it is not. The rewritten query
reads **29 rows instead of roughly 150 million** — one pre-computed sketch per
day, merged in microseconds. End-to-end latency drops to roughly **200 ms** — a
**\~5× user-side speedup**. And the long-window cases scale gracefully: at 6
months, the cluster reads **180 rows instead of \~1 billion**; at 13 months,
**around 400 rows instead of more than 2 billion**. The cost of “show me a
longer view” stops being scary.
+
+The MV itself is small. Across the full set of active orgs, two `on_off_net`
values, and roughly 400 days of retention, it produces on the order of **tens
of millions of rows / tens of GB** — **two orders of magnitude smaller** than
the source table. The cost of remembering is a small fraction of the cost of
recomputing.
+
+### The bigger win is not the latency
+
+A 5× speedup on a single dashboard query is the visible part of the story. The
invisible part is what matters more to operators: **the cluster scanned more
than five million times fewer rows for this one query.**
+
+Fewer rows scanned means less segment data loaded off disk, less heap churn
while building HLL sketches, less CPU on filter and projection, less network
traffic across replicas. Every byte the cluster does not have to read is a byte
not competing with ingestion throughput, concurrent queries, or background
compactions.
+
+These savings translate directly into the resources that govern cluster
stability. For a query of this class, **memory pressure drops by roughly 5× and
CPU consumption by roughly 20×** — direct consequences of the I/O collapse you
just saw. Per-query resource attribution for MV-rewritten queries is still
maturing in our observability stack, so we are not pinning these numbers to a
decimal point yet — but the order of magnitude is hard to miss.
+
+**Users get snappier dashboards. Operators get a quieter cluster.** That
double win — better user experience and better cluster stability from the same
change — is the reason this work exists.
+
+### How the engine actually finds the view
+
+When the user query arrives, the rewrite engine walks five quick steps:
+
+1. **Parse and identify the query shape**: aggregation function
(`DISTINCTCOUNTHLL`), filter columns (`on_off_net`, `orgid`, time), no GROUP BY.
+2. **Pull MV candidates** from `MaterializedViewMetadataCache` for the source
table.
+3. **Check aggregation subsumption** via `AggregationEquivalenceRegistry`. The
user’s `DISTINCTCOUNTHLL` matches `SketchMergeEquivalence("DISTINCTCOUNTHLL",
"DISTINCTCOUNTRAWHLL", "DISTINCTCOUNTHLL")` — meaning the MV’s raw HLL bytes
can be merged into the user’s cardinality result.
+4. **Align the time range** via `MaterializedViewTimeExpression`. The user’s
`callstarttimemillis` predicate is rewritten to reference the MV’s day bucket
column, with the epoch-ms literals carried over verbatim.
+5. **Emit the rewritten plan** — same SQL surface to the user, completely
different physical execution.
+
+**No code change on the dashboard side.** No new pipeline. One DDL statement
at MV creation time, and from that moment forward, every qualifying query takes
the cheap path automatically.
+
+## How the view earns trust
+
+A fast wrong answer is worse than a slow right one. The hardest engineering
problem in any materialized view system is not building the views — it is
deciding, at query time, when a given MV partition is safe to read. Pinot’s MV
layer makes that decision explicit, per partition, with a small state machine
that both the refresh path and the read path can reason about.
+
+Before the state machine itself, the picture: an MV is laid out as a
fixed-width grid of partitions over the base table’s timeline, with each bucket
fed by whichever base segments happen to fall into its window. The
`bucketTimePeriod` knob from the DDL is exactly what fixes the grid’s width —
and every freshness check, refresh task, and rewrite decision in the rest of
this section reduces to per-bucket reasoning over that grid.
+
+
+
+_The MV’s bucket grid laid over the base table’s segment timeline. Each MV
bucket aggregates the base segments whose data falls into its window; the
`bucketTimePeriod` knob from the DDL is what fixes the grid’s width._
+
+### **Per-partition freshness, not all-or-nothing**
+
+Every MV is sliced into time buckets of fixed width, and for each bucket Pinot
stores a small `PartitionInfo` record in ZooKeeper:
+
+```bash
+bucket = 2026-02-23 → { state: VALID, segmentCount: 47, crc: 0x5e3a…,
lastRefreshTime: 1771823412000 }
+bucket = 2026-03-23 → { state: STALE, segmentCount: 12, crc: 0x9b1f…,
lastRefreshTime: 1773002145000 }
+```
+
+
+
+_MV detail page in the Pinot controller UI: per-bucket VALID / STALE state,
the MV’s watermarkMs, and the last refresh time are exposed to operators
directly — the same data the rewrite engine reads at compile time to decide
whether to redirect a query._
+
+Freshness is a property of _each bucket_, not of the whole view. A user asking
about February does not care that March is in the middle of a refresh. The
`PartitionFingerprint` — segment count plus a CRC sum across the base segments
that fed the bucket — is what makes the freshness check robust: if a base
segment is swapped for a different one at the same count, the count alone would
not notice, but the CRC will.
+
+When a base-table segment is added, replaced, or deleted,
`MaterializedViewConsistencyManager` picks up the ZK event, debounces for five
seconds to coalesce bulk ingestion bursts, intersects the affected time range
with each MV’s bucket grid, and flips overlapping buckets to `STALE`. It is
event-driven, not polled — idle clusters do no work.
+
+### **Refresh without blocking reads**
+
+A scheduler (the Generator) periodically scans for `STALE` buckets. Before
submitting a refresh task it re-reads the current base-segment fingerprint and
compares it to the stored baseline; if they match, the stale marking was a
false positive (a segment churn that did not actually change data) and the
bucket is quietly returned to `VALID` without recomputation. Real changes go on
to the Executor, which materializes the bucket, writes the new fingerprint,
flips state back to VALID, and a [...]
+
+That watermark is what closes the loop with the query path. When a user’s time
range crosses it, the broker switches into `ExecutionMode.SPLIT*REWRITE:`` the
MV serves the cold half (`ts \< watermarkMs`), the base table serves the hot
half (`ts \>= watermarkMs`), and `BrokerReduceService` merges both sides using
the user’s original query as the merge key. The user gets the latest ingested
data \_and\* the pre-aggregation speedup on the cold half — they never wait for
a refresh to finish.
+
+**Bucket-level consistency is a deliberate choice.** Analytics dashboards look
at completed time buckets, not “as of two seconds ago,” and the bucket boundary
maps cleanly onto a freshness contract a human can reason about. For workloads
that need stricter freshness, operators can lower `stalenessThresholdMs` (the
rewrite engine will skip an MV whose `watermarkMs` falls outside the SLO) or
set `rewriteEnabled=false` to take the MV out of the rewrite path entirely.
Schema changes, partial [...]
+
+## **What it bought us**
+
+Pinot’s MV layer hasn’t been in production long enough to publish
quarter-over-quarter savings curves, but the shape of what it changes is
already visible at three different altitudes.
+
+**Cluster stability.** When a stable query pattern hits the same fact table
hundreds of times an hour from different dashboards, the rewrite engine
collapses that pattern down to a handful of pre-aggregated rows. Less data read
from disk, less scan work across replicas, less heap pressure during sketch
construction. The cluster-overload pattern from earlier — where unrelated
tenants pile onto the same expensive aggregation — becomes much harder to
trigger.
+
+**Query latency.** A user query that hits an MV reads a row count proportional
to its time range in buckets, not to the source table’s row count. The “what if
I widen the date picker to a year” case — the one users used to learn to avoid
— flattens out. Dashboards stay snappy at windows where they used to crawl.
+
+**One less pipeline.** The external daily-aggregate ETL pattern from earlier
collapses into a single `CREATE MATERIALIZED VIEW` statement that the cluster
owns end-to-end. No second schedule, no second failure mode, no second source
of truth.
+
+## What’s next
+
+Two pieces of work are in active design.
+
+**Hybrid table support.** MVs today are built over `OFFLINE` source tables,
which fits batch-ingested fact tables well but excludes any table whose
freshest data lives in a realtime segment. The roadmap here is to extend MV
bookkeeping — partition fingerprints, the consistency manager, the watermark
contract — to hybrid tables, so a realtime-backed dashboard can sit on the same
MV layer the offline tables already do today. The freshness state machine from
the previous section is designed [...]
+
+**Multi-Stage Engine (MSE) query support.** Transparent rewrite today runs on
Pinot’s Single-Stage Engine. The next milestone is to let MSE queries read MVs
directly — users referencing the MV table by name in an MSE query plan — before
the more invasive step of teaching MSE planning to find the right MV
automatically. Direct access first, transparent rewrite second.
+
+## Try it
+
+The Materialized View feature is merged into Apache Pinot. The code lives
under the `pinot-materialized-view` module and ships with a self-contained
quickstart that builds the base table, creates an MV, runs the minion refresh
task, and validates the rewrite end-to-end:
+
+```
+bin/pinot-admin.sh QuickStart -type MATERIALIZED_VIEW
+```
+
+Transparent rewrite is gated behind a broker switch —
`pinot.broker.query.enable.materialized.view.rewrite=true` — so it can be
enabled per cluster as teams evaluate it on their own workloads. Full reference
— supported aggregations, source-table requirements, REST endpoints, and the
Data Explorer UI for inspecting MV state — lives in the [Apache Pinot
materialized views
documentation](https://docs.pinot.apache.org/build-with-pinot/querying-and-sql/materialized-views).
+
+If your workload looks like the ones in this post, we would love to hear how
it lands.
+
+More details and updated information about <a
href="https://docs.pinot.apache.org/build-with-pinot/querying-and-sql/materialized-views";>Materialized
Views in Apache Pinot in the Docs</a>
diff --git a/public/static/images/authors/hongkun.jpg
b/public/static/images/authors/hongkun.jpg
new file mode 100644
index 00000000..952bb2e2
Binary files /dev/null and b/public/static/images/authors/hongkun.jpg differ
diff --git a/public/static/images/authors/raghav.jpeg
b/public/static/images/authors/raghav.jpeg
new file mode 100644
index 00000000..7ab08938
Binary files /dev/null and b/public/static/images/authors/raghav.jpeg differ
diff --git a/public/static/images/blogs/materialized-views/image1.png
b/public/static/images/blogs/materialized-views/image1.png
new file mode 100644
index 00000000..ffbdfab7
Binary files /dev/null and
b/public/static/images/blogs/materialized-views/image1.png differ
diff --git a/public/static/images/blogs/materialized-views/image2.png
b/public/static/images/blogs/materialized-views/image2.png
new file mode 100644
index 00000000..f62ecb4d
Binary files /dev/null and
b/public/static/images/blogs/materialized-views/image2.png differ
diff --git a/public/static/images/blogs/materialized-views/image3.png
b/public/static/images/blogs/materialized-views/image3.png
new file mode 100644
index 00000000..ae61169e
Binary files /dev/null and
b/public/static/images/blogs/materialized-views/image3.png differ
diff --git a/public/static/images/blogs/materialized-views/image4.png
b/public/static/images/blogs/materialized-views/image4.png
new file mode 100644
index 00000000..096d7d61
Binary files /dev/null and
b/public/static/images/blogs/materialized-views/image4.png differ
diff --git a/tailwind.config.ts b/tailwind.config.ts
index 7a72c4da..069f61ba 100644
--- a/tailwind.config.ts
+++ b/tailwind.config.ts
@@ -141,8 +141,10 @@ const config = {
fontWeight: '600'
},
code: {
- color: theme('colors.indigo.500')
+ color: theme('colors.gray.500')
},
+ 'code::before': { content: '""' },
+ 'code::after': { content: '""' },
p: {
overflowWrap: 'break-word',
wordBreak: 'break-word'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
