This is an automated email from the ASF dual-hosted git repository.
ptupitsyn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 8e864e6a32 IGNITE-18246 .NET: LINQ: Add support for Intersect, Union,
Except (#1386)
8e864e6a32 is described below
commit 8e864e6a3295f2fd49231265f7c04c83dd316abf
Author: Pavel Tupitsyn <[email protected]>
AuthorDate: Mon Nov 28 21:54:57 2022 +0200
IGNITE-18246 .NET: LINQ: Add support for Intersect, Union, Except (#1386)
---
.../dotnet/Apache.Ignite.Tests/JavaServer.cs | 3 +-
.../Linq/LinqSqlGenerationTests.cs | 26 ++++
.../Apache.Ignite.Tests/Linq/LinqTests.GroupBy.cs | 40 ++++++
.../Linq/LinqTests.UnionIntersectExcept.cs | 159 +++++++++++++++++++++
.../Internal/Linq/IgniteQueryExecutor.cs | 11 ++
.../Internal/Linq/IgniteQueryModelVisitor.cs | 87 +++++------
.../Table/Serialization/ReflectionUtils.cs | 11 +-
.../dotnet/Apache.Ignite/Sql/IResultSet.cs | 2 +
8 files changed, 292 insertions(+), 47 deletions(-)
diff --git a/modules/platforms/dotnet/Apache.Ignite.Tests/JavaServer.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/JavaServer.cs
index 97b5592323..ca60a14a03 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/JavaServer.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/JavaServer.cs
@@ -180,8 +180,7 @@ namespace Apache.Ignite.Tests
var cfg = new IgniteClientConfiguration("127.0.0.1:" + port);
using var client = await IgniteClient.StartAsync(cfg);
- var tables = await client.Tables.GetTablesAsync();
- return tables.Count > 0 ? null : new
InvalidOperationException("No tables found on server");
+ return null;
}
catch (Exception e)
{
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqSqlGenerationTests.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqSqlGenerationTests.cs
index 0a0ba656fd..4582114324 100644
---
a/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqSqlGenerationTests.cs
+++
b/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqSqlGenerationTests.cs
@@ -207,6 +207,32 @@ public partial class LinqSqlGenerationTests
ex!.Message);
}
+ [Test]
+ public void TestUnion() =>
+ AssertSql(
+ "select (_T0.KEY + ?), _T0.VAL from PUBLIC.tbl1 as _T0 " +
+ "union (select (_T1.KEY + ?), _T1.VAL from PUBLIC.tbl1 as _T1)",
+ q => q.Select(x => new { Key = x.Key + 1, x.Val })
+ .Union(q.Select(x => new { Key = x.Key + 100, x.Val }))
+ .ToList());
+
+ [Test]
+ public void TestIntersect() =>
+ AssertSql(
+ "select (_T0.KEY + ?), concat(_T0.VAL, ?) from PUBLIC.tbl1 as _T0
" +
+ "intersect (select (_T1.KEY + ?), concat(_T1.VAL, ?) from
PUBLIC.tbl1 as _T1)",
+ q => q.Select(x => new { Key = x.Key + 1, Val = x.Val + "_" })
+ .Intersect(q.Select(x => new { Key = x.Key + 100, Val = x.Val
+ "!" }))
+ .ToList());
+
+ [Test]
+ public void TestExcept() =>
+ AssertSql(
+ "select (_T0.KEY + ?) from PUBLIC.tbl1 as _T0 except (select
(_T1.KEY + ?) from PUBLIC.tbl1 as _T1)",
+ q => q.Select(x => x.Key + 1)
+ .Except(q.Select(x => x.Key + 5))
+ .ToList());
+
[OneTimeSetUp]
public async Task OneTimeSetUp()
{
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.GroupBy.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.GroupBy.cs
index c858d20cb0..7ae5615831 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.GroupBy.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.GroupBy.cs
@@ -19,6 +19,7 @@ namespace Apache.Ignite.Tests.Linq;
using System.Collections.Generic;
using System.Linq;
+using Internal.Linq;
using NUnit.Framework;
/// <summary>
@@ -146,4 +147,43 @@ public partial class LinqTests
"order by (_T0.VAL) asc",
query.ToString());
}
+
+ /// <summary>
+ /// Tests grouping combined with join in a reverse order followed by a
projection to an anonymous type with
+ /// custom projected column names.
+ /// <para />
+ /// Covers <see cref="ExpressionWalker.GetProjectedMember"/>.
+ /// </summary>
+ [Test]
+ public void TestGroupByWithReverseJoinAndAnonymousProjectionWithRename()
+ {
+ var query1 = PocoView.AsQueryable();
+ var query2 = PocoIntView.AsQueryable();
+
+ var query = query1.Join(
+ query2,
+ o => o.Key,
+ p => p.Key,
+ (org, person) => new
+ {
+ Cat = org.Val,
+ Price = person.Val
+ })
+ .GroupBy(x => x.Cat)
+ .Select(g => new {Category = g.Key, MaxPrice = g.Max(x =>
x.Price)})
+ .OrderByDescending(x => x.MaxPrice);
+
+ var res = query.ToList();
+
+ Assert.AreEqual("v-9", res[0].Category);
+ Assert.AreEqual(900, res[0].MaxPrice);
+
+ StringAssert.Contains(
+ "select _T0.VAL, max(_T1.VAL) " +
+ "from PUBLIC.TBL1 as _T0 " +
+ "inner join PUBLIC.TBL_INT32 as _T1 on (cast(_T1.KEY as bigint) =
_T0.KEY) " +
+ "group by (_T0.VAL) " +
+ "order by (max(_T1.VAL)) desc",
+ query.ToString());
+ }
}
diff --git
a/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.UnionIntersectExcept.cs
b/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.UnionIntersectExcept.cs
new file mode 100644
index 0000000000..f6bc7a9be2
--- /dev/null
+++
b/modules/platforms/dotnet/Apache.Ignite.Tests/Linq/LinqTests.UnionIntersectExcept.cs
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Tests.Linq;
+
+using System.Linq;
+using NUnit.Framework;
+
+/// <summary>
+/// Linq UNION/INTERSECT/EXCEPT tests.
+/// </summary>
+public partial class LinqTests
+{
+ [Test]
+ public void TestUnion()
+ {
+ var subQuery = PocoView.AsQueryable()
+ .Where(x => x.Key < 2)
+ .Select(x => new { Id = x.Key });
+
+ var query = PocoLongView.AsQueryable()
+ .Where(x => x.Key > 8)
+ .Select(x => new { Id = x.Key })
+ .Union(subQuery);
+
+ var res = query.ToList();
+
+ CollectionAssert.AreEquivalent(new[] { 0, 1, 9 }, res.Select(x =>
x.Id));
+
+ StringAssert.Contains(
+ "select _T0.KEY from PUBLIC.TBL_INT64 as _T0 where (_T0.KEY > ?) "
+
+ "union (select _T1.KEY from PUBLIC.TBL1 as _T1 where (_T1.KEY <
?)",
+ query.ToString());
+ }
+
+ [Test]
+ public void TestUnionWithOrderBy()
+ {
+ var subQuery = PocoView.AsQueryable()
+ .Where(x => x.Key < 2)
+ .Select(x => new { Id = x.Key });
+
+ var query = PocoLongView.AsQueryable()
+ .Where(x => x.Key > 8)
+ .Select(x => new { Id = x.Key })
+ .Union(subQuery)
+ .OrderBy(x => x.Id);
+
+ var res = query.ToList();
+
+ Assert.AreEqual(new[] { 0, 1, 9 }, res.Select(x => x.Id));
+
+ StringAssert.Contains(
+ "select * from " +
+ "(select _T0.KEY from PUBLIC.TBL_INT64 as _T0 where (_T0.KEY > ?)
" +
+ "union (select _T1.KEY from PUBLIC.TBL1 as _T1 where (_T1.KEY <
?))) as _T2 " +
+ "order by (_T2.KEY) asc",
+ query.ToString());
+ }
+
+ [Test]
+ public void TestUnionWithCast()
+ {
+ var subQuery = PocoIntView.AsQueryable()
+ .Select(x => new { x.Key })
+ .Where(x => x.Key > 3 && x.Key < 5);
+
+ var query = PocoByteView.AsQueryable()
+ .Where(x => x.Key > 8)
+ .Select(x => new { Key = (int)x.Key })
+ .Union(subQuery);
+
+ var res = query.ToList();
+
+ CollectionAssert.AreEquivalent(new[] { 4, 9 }, res.Select(x => x.Key));
+
+ StringAssert.Contains(
+ "select cast(_T0.KEY as int) " +
+ "from PUBLIC.TBL_INT8 as _T0 " +
+ "where (cast(_T0.KEY as int) > ?) " +
+ "union (select _T1.KEY from PUBLIC.TBL_INT32 as _T1 where
((_T1.KEY > ?) and (_T1.KEY < ?)))",
+ query.ToString());
+ }
+
+ [Test]
+ public void TestIntersect()
+ {
+ var subQuery = PocoView.AsQueryable()
+ .Where(x => x.Key < 5)
+ .Select(x => new { Id = x.Key });
+
+ var query = PocoLongView.AsQueryable()
+ .Where(x => x.Key > 2)
+ .Select(x => new { Id = x.Key })
+ .Intersect(subQuery);
+
+ var res = query.ToList();
+
+ CollectionAssert.AreEquivalent(new[] { 3, 4 }, res.Select(x => x.Id));
+
+ StringAssert.Contains(
+ "select _T0.KEY from PUBLIC.TBL_INT64 as _T0 where (_T0.KEY > ?) "
+
+ "intersect (select _T1.KEY from PUBLIC.TBL1 as _T1 where (_T1.KEY
< ?)",
+ query.ToString());
+ }
+
+ [Test]
+ public void TestIntersectEmpty()
+ {
+ var subQuery = PocoView.AsQueryable()
+ .Where(x => x.Key < 2)
+ .Select(x => new { Id = x.Key });
+
+ var query = PocoLongView.AsQueryable()
+ .Where(x => x.Key > 8)
+ .Select(x => new { Id = x.Key })
+ .Intersect(subQuery);
+
+ var res = query.ToList();
+
+ CollectionAssert.IsEmpty(res);
+ }
+
+ [Test]
+ public void TestExcept()
+ {
+ var subQuery = PocoView.AsQueryable()
+ .Where(x => x.Key < 7)
+ .Select(x => new { Id = x.Key });
+
+ var query = PocoLongView.AsQueryable()
+ .Where(x => x.Key > 5)
+ .Select(x => new { Id = x.Key })
+ .Except(subQuery);
+
+ var res = query.ToList();
+
+ CollectionAssert.AreEquivalent(new[] { 7, 8, 9 }, res.Select(x =>
x.Id));
+
+ StringAssert.Contains(
+ "select _T0.KEY from PUBLIC.TBL_INT64 as _T0 where (_T0.KEY > ?) "
+
+ "except (select _T1.KEY from PUBLIC.TBL1 as _T1 where (_T1.KEY <
?))",
+ query.ToString());
+ }
+}
diff --git
a/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryExecutor.cs
b/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryExecutor.cs
index f9815f3ac6..fd049619ff 100644
---
a/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryExecutor.cs
+++
b/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryExecutor.cs
@@ -28,6 +28,8 @@ using Ignite.Sql;
using Ignite.Transactions;
using Proto.BinaryTuple;
using Remotion.Linq;
+using Remotion.Linq.Clauses;
+using Remotion.Linq.Clauses.Expressions;
using Sql;
using Table.Serialization;
@@ -180,6 +182,15 @@ internal sealed class IgniteQueryExecutor : IQueryExecutor
};
}
+ if (selectorExpression is QuerySourceReferenceExpression
+ {
+ ReferencedQuerySource: IFromClause { FromExpression:
SubQueryExpression subQuery }
+ })
+ {
+ // Select everything from a sub-query - use nested selector.
+ return GetResultSelector<T>(columns,
subQuery.QueryModel.SelectClause.Selector);
+ }
+
return (IReadOnlyList<IColumnMetadata> cols, ref BinaryTupleReader
reader) =>
{
var res = (T)FormatterServices.GetUninitializedObject(typeof(T));
diff --git
a/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryModelVisitor.cs
b/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryModelVisitor.cs
index 8a5e3e3e32..c8422091f0 100644
---
a/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryModelVisitor.cs
+++
b/modules/platforms/dotnet/Apache.Ignite/Internal/Linq/IgniteQueryModelVisitor.cs
@@ -78,7 +78,7 @@ internal sealed class IgniteQueryModelVisitor :
QueryModelVisitorBase
/** <inheritdoc /> */
public override void VisitQueryModel(QueryModel queryModel)
{
- VisitQueryModel(queryModel, false);
+ VisitQueryModel(queryModel, includeAllFields:
queryModel.MainFromClause.FromExpression is SubQueryExpression);
}
/** <inheritdoc /> */
@@ -180,7 +180,21 @@ internal sealed class IgniteQueryModelVisitor :
QueryModelVisitorBase
/** <inheritdoc /> */
public override void VisitMainFromClause(MainFromClause fromClause,
QueryModel queryModel)
{
- base.VisitMainFromClause(fromClause, queryModel);
+ // Special case for UNION subquery.
+ if (fromClause.FromExpression is SubQueryExpression subQuery &&
+ subQuery.QueryModel.ResultOperators.Any(x => x is
UnionResultOperator))
+ {
+ _builder.Append("from (");
+
+ VisitQueryModel(subQuery.QueryModel);
+
+ _builder.TrimEnd()
+ .Append(") as ")
+
.Append(_aliases.GetTableAlias(subQuery.QueryModel.MainFromClause))
+ .Append(' ');
+
+ return;
+ }
// TODO: IGNITE-18137 DML: queryModel.ResultOperators.LastOrDefault()
is UpdateAllResultOperator;
var isUpdateQuery = false;
@@ -274,12 +288,9 @@ internal sealed class IgniteQueryModelVisitor :
QueryModelVisitorBase
{
var parenCount = ProcessResultOperatorsBegin(queryModel);
- if (parenCount >= 0)
- {
- // FIELD1, FIELD2
- BuildSqlExpression(queryModel.SelectClause.Selector, parenCount >
0, includeAllFields);
- _builder.TrimEnd().Append(')', parenCount).Append(' ');
- }
+ // FIELD1, FIELD2
+ BuildSqlExpression(queryModel.SelectClause.Selector, parenCount > 0,
includeAllFields);
+ _builder.TrimEnd().Append(')', parenCount).Append(' ');
}
/** <inheritdoc /> */
@@ -393,67 +404,61 @@ internal sealed class IgniteQueryModelVisitor :
QueryModelVisitorBase
{
ProcessSkipTake(queryModel);
- foreach (var op in queryModel.ResultOperators.Reverse())
+ for (var i = queryModel.ResultOperators.Count - 1; i >= 0; i--)
{
+ var op = queryModel.ResultOperators[i];
string? keyword = null;
Expression? source = null;
- var union = op as UnionResultOperator;
- if (union != null)
+ if (op is UnionResultOperator union)
{
keyword = "union";
source = union.Source2;
}
- var intersect = op as IntersectResultOperator;
- if (intersect != null)
+ if (op is IntersectResultOperator intersect)
{
keyword = "intersect";
source = intersect.Source2;
}
- var except = op as ExceptResultOperator;
- if (except != null)
+ if (op is ExceptResultOperator except)
{
keyword = "except";
source = except.Source2;
}
- if (keyword != null)
+ if (keyword == null)
{
- _builder.Append(keyword).Append(" (");
+ continue;
+ }
- var subQuery = source as SubQueryExpression;
+ _builder.Append(keyword).Append(" (");
- if (subQuery != null)
+ if (source is SubQueryExpression subQuery)
+ {
+ // Subquery union.
+ VisitQueryModel(subQuery.QueryModel);
+ }
+ else
+ {
+ // Direct union, source is IIgniteQueryableInternal
+ if (source is not ConstantExpression innerExpr)
{
- // Subquery union.
- VisitQueryModel(subQuery.QueryModel);
+ throw new NotSupportedException("Unexpected UNION inner
sequence: " + source);
}
- else
- {
- // Direct union, source is IIgniteQueryableInternal
- var innerExpr = source as ConstantExpression;
-
- if (innerExpr == null)
- {
- throw new NotSupportedException("Unexpected UNION
inner sequence: " + source);
- }
- var queryable = innerExpr.Value as
IIgniteQueryableInternal;
-
- if (queryable == null)
- {
- throw new NotSupportedException("Unexpected UNION
inner sequence " +
- "(only results of
cache.ToQueryable() are supported): " +
- innerExpr.Value);
- }
-
- VisitQueryModel(queryable.GetQueryModel());
+ if (innerExpr.Value is not IIgniteQueryableInternal queryable)
+ {
+ throw new NotSupportedException("Unexpected UNION inner
sequence " +
+ "(only results of
cache.ToQueryable() are supported): " +
+ innerExpr.Value);
}
- _builder.TrimEnd().Append(')');
+ VisitQueryModel(queryable.GetQueryModel());
}
+
+ _builder.TrimEnd().Append(')');
}
}
diff --git
a/modules/platforms/dotnet/Apache.Ignite/Internal/Table/Serialization/ReflectionUtils.cs
b/modules/platforms/dotnet/Apache.Ignite/Internal/Table/Serialization/ReflectionUtils.cs
index 8815baf650..02e0658d54 100644
---
a/modules/platforms/dotnet/Apache.Ignite/Internal/Table/Serialization/ReflectionUtils.cs
+++
b/modules/platforms/dotnet/Apache.Ignite/Internal/Table/Serialization/ReflectionUtils.cs
@@ -172,7 +172,7 @@ namespace Apache.Ignite.Internal.Table.Serialization
if (fieldInfo.GetCustomAttribute<ColumnAttribute>() is { Name: { }
columnAttributeName })
{
- return new(columnAttributeName, fieldInfo,
HasColumnNameAttribute: true);
+ return new(columnAttributeName, fieldInfo, null,
HasColumnNameAttribute: true);
}
var cleanName = fieldInfo.GetCleanName();
@@ -188,11 +188,13 @@ namespace Apache.Ignite.Internal.Table.Serialization
if (property.GetCustomAttribute<ColumnAttribute>() is { Name:
{ } columnAttributeName2 })
{
// This is a compiler-generated backing field for an
automatic property - get the attribute from the property.
- return new(columnAttributeName2, fieldInfo,
HasColumnNameAttribute: true);
+ return new(columnAttributeName2, fieldInfo, property,
HasColumnNameAttribute: true);
}
+
+ return new(cleanName, fieldInfo, property,
HasColumnNameAttribute: false);
}
- return new(cleanName, fieldInfo, HasColumnNameAttribute: false);
+ return new(cleanName, fieldInfo, null, HasColumnNameAttribute:
false);
}
/// <summary>
@@ -232,8 +234,9 @@ namespace Apache.Ignite.Internal.Table.Serialization
/// </summary>
/// <param name="Name">Column name.</param>
/// <param name="Field">Corresponding field.</param>
+ /// <param name="Property">Corresponding property (when <see
cref="Field"/> is a backing field of a auto property).</param>
/// <param name="HasColumnNameAttribute">Whether corresponding field
or property has <see cref="ColumnAttribute"/>
/// with <see cref="ColumnAttribute.Name"/> set.</param>
- internal record ColumnInfo(string Name, FieldInfo Field, bool
HasColumnNameAttribute);
+ internal record ColumnInfo(string Name, FieldInfo Field, PropertyInfo?
Property, bool HasColumnNameAttribute);
}
}
diff --git a/modules/platforms/dotnet/Apache.Ignite/Sql/IResultSet.cs
b/modules/platforms/dotnet/Apache.Ignite/Sql/IResultSet.cs
index 8a61bed18c..89a14b8949 100644
--- a/modules/platforms/dotnet/Apache.Ignite/Sql/IResultSet.cs
+++ b/modules/platforms/dotnet/Apache.Ignite/Sql/IResultSet.cs
@@ -19,6 +19,7 @@ namespace Apache.Ignite.Sql
{
using System;
using System.Collections.Generic;
+ using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
/// <summary>
@@ -42,6 +43,7 @@ namespace Apache.Ignite.Sql
/// <summary>
/// Gets a value indicating whether this result set contains a
collection of rows.
/// </summary>
+ [MemberNotNullWhen(true, nameof(Metadata))]
bool HasRowSet { get; }
/// <summary>
