[ignite-3] branch main updated: IGNITE-13816 - Add separate top level dirs for logs and configs (#26)
This is an automated email from the ASF dual-hosted git repository.
vkulichenko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 7a2e7ad IGNITE-13816 - Add separate top level dirs for logs and
configs (#26)
7a2e7ad is described below
commit 7a2e7ade35293eb62f7c79eac6ca04fa13a1b485
Author: Kirill Gusakov
AuthorDate: Wed Feb 3 07:29:30 2021 +0300
IGNITE-13816 - Add separate top level dirs for logs and configs (#26)
---
.../apache/ignite/cli/CliPathsConfigLoader.java| 11 +++--
.../java/org/apache/ignite/cli/IgnitePaths.java| 47
.../cli/builtins/init/InitIgniteCommand.java | 15 ++-
.../ignite/cli/builtins/node/NodeManager.java | 19
.../apache/ignite/cli/spec/NodeCommandSpec.java| 4 +-
.../apache/ignite/cli/IgniteCliInterfaceTest.java | 50 --
.../cli/builtins/module/ModuleMangerTest.java | 16 +--
7 files changed, 91 insertions(+), 71 deletions(-)
diff --git
a/modules/cli/src/main/java/org/apache/ignite/cli/CliPathsConfigLoader.java
b/modules/cli/src/main/java/org/apache/ignite/cli/CliPathsConfigLoader.java
index 7b0bf00..f3db1ca 100644
--- a/modules/cli/src/main/java/org/apache/ignite/cli/CliPathsConfigLoader.java
+++ b/modules/cli/src/main/java/org/apache/ignite/cli/CliPathsConfigLoader.java
@@ -56,7 +56,7 @@ public class CliPathsConfigLoader {
CliVersionInfo cliVerInfo
) {
this.pathRslvr = pathRslvr;
-this.ver = cliVerInfo.ver;
+ver = cliVerInfo.ver;
}
/**
@@ -110,13 +110,18 @@ public class CliPathsConfigLoader {
Properties props = new Properties();
props.load(inputStream);
-if ((props.getProperty("bin") == null) ||
(props.getProperty("work") == null))
+if ((props.getProperty("bin") == null) ||
+(props.getProperty("work") == null) ||
+(props.getProperty("config") == null) ||
+(props.getProperty("log") == null) )
throw new IgniteCLIException("Config file has wrong format. " +
-"It must contain correct paths to bin and work dirs");
+"It must contain correct paths to bin, work, config and
log dirs");
return new IgnitePaths(
Path.of(props.getProperty("bin")),
Path.of(props.getProperty("work")),
+Path.of(props.getProperty("config")),
+Path.of(props.getProperty("log")),
ver);
}
catch (IOException e) {
diff --git a/modules/cli/src/main/java/org/apache/ignite/cli/IgnitePaths.java
b/modules/cli/src/main/java/org/apache/ignite/cli/IgnitePaths.java
index 1f2479c..4ef1862 100644
--- a/modules/cli/src/main/java/org/apache/ignite/cli/IgnitePaths.java
+++ b/modules/cli/src/main/java/org/apache/ignite/cli/IgnitePaths.java
@@ -56,6 +56,12 @@ public class IgnitePaths {
/** Work directory for Ignite server and CLI operation. */
public final Path workDir;
+/** Directory for storing server node configs. */
+public final Path configDir;
+
+/** Directory for server nodes logs */
+public final Path logDir;
+
/**
* Ignite CLI version.
* Also, the same version will be used for addressing any binaries inside
bin dir
@@ -69,9 +75,11 @@ public class IgnitePaths {
* @param workDir Work directory.
* @param ver Ignite CLI version.
*/
-public IgnitePaths(Path binDir, Path workDir, String ver) {
+public IgnitePaths(Path binDir, Path workDir, Path configDir, Path logDir,
String ver) {
this.binDir = binDir;
this.workDir = workDir;
+this.configDir = configDir;
+this.logDir = logDir;
this.ver = ver;
}
@@ -114,28 +122,31 @@ public class IgnitePaths {
* Path to default Ignite node config.
*/
public Path serverDefaultConfigFile() {
-return serverConfigDir().resolve("default-config.xml");
+return configDir.resolve("default-config.xml");
}
/**
* Init or recovers Ignite distributive directories structure.
*/
public void initOrRecover() {
-File igniteWork = workDir.toFile();
-if (!(igniteWork.exists() || igniteWork.mkdirs()))
-throw new IgniteCLIException("Can't create working directory: " +
workDir);
-
-File igniteBin = libsDir().toFile();
-if (!(igniteBin.exists() || igniteBin.mkdirs()))
-throw new IgniteCLIException("Can't create a directory for ignite
modules: " + libsDir());
-
-File igniteBinCli = cliLibsDir().toFile();
-if (!(igniteBinCli.exists() || igniteBinCli.mkdirs()))
-throw new IgniteCLIException("Can't create a directory for cli
modules: " + cliLibsDir());
+initDirIfNeeded(workDir,"Can't create working directory: "
[ignite-3] branch main updated: IGNITE-13991 - Maven checks fixes (#32)
This is an automated email from the ASF dual-hosted git repository.
vkulichenko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 5ef4d1e IGNITE-13991 - Maven checks fixes (#32)
5ef4d1e is described below
commit 5ef4d1e60b38ec86f9ff93f925fd6364f5487a89
Author: Peter Ivanov
AuthorDate: Wed Feb 3 07:28:46 2021 +0300
IGNITE-13991 - Maven checks fixes (#32)
---
parent/pom.xml | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/parent/pom.xml b/parent/pom.xml
index ac1fc7d..4eb9707 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -57,7 +57,6 @@
1.18
3.12.0
1.13.0
-1.3.2
2.0.1.Final
20.1.0
5.7.0
@@ -252,13 +251,13 @@
${maven.compiler.plugin.version}
-
+
-
+
org.apache.maven.plugins
[ignite] branch ignite-2.10 updated (241ff5a -> 85b54da)
This is an automated email from the ASF dual-hosted git repository. mmuzaf pushed a change to branch ignite-2.10 in repository https://gitbox.apache.org/repos/asf/ignite.git. from 241ff5a IGNITE-14107 update ignite 2.10 release version (#8739) add 85b54da IGNITE-14117 Fix travis build under jdk11 due to incorrect TLS version (#8741) No new revisions were added by this update. Summary of changes: .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
[ignite] branch master updated (a7c83a5 -> d467e31)
This is an automated email from the ASF dual-hosted git repository. mmuzaf pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git. from a7c83a5 IGNITE-14093 TTL cleanup worker logic fix: it should not start on a node removed from baseline - Fixes #8721. add d467e31 IGNITE-14117 Fix travis build under jdk11 due to incorrect TLS version (#8741) No new revisions were added by this update. Summary of changes: .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
[ignite] branch master updated: IGNITE-14093 TTL cleanup worker logic fix: it should not start on a node removed from baseline - Fixes #8721.
This is an automated email from the ASF dual-hosted git repository.
sergeychugunov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new a7c83a5 IGNITE-14093 TTL cleanup worker logic fix: it should not
start on a node removed from baseline - Fixes #8721.
a7c83a5 is described below
commit a7c83a5f190a6622151950a17d7e127f9b47bb6d
Author: Mirza Aliev
AuthorDate: Tue Feb 2 16:52:08 2021 +0300
IGNITE-14093 TTL cleanup worker logic fix: it should not start on a node
removed from baseline - Fixes #8721.
Signed-off-by: Sergey Chugunov
---
.../apache/ignite/internal/processors/cache/GridCacheProcessor.java | 4
1 file changed, 4 insertions(+)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index ca5a5a2..0fbeba6 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -5399,6 +5399,10 @@ public class GridCacheProcessor extends
GridProcessorAdapter {
/** {@inheritDoc} */
@Override public void onBaselineChange() {
+for (GridCacheAdapter cache : caches.values())
+if (cache != null)
+cache.context().ttl().unregister();
+
onKernalStopCaches(true);
stopCaches(true);
[ignite-python-thin-client] branch master updated: IGNITE-13863: Fix Null reading and writing
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
The following commit(s) were added to refs/heads/master by this push:
new 2c0ecb2 IGNITE-13863: Fix Null reading and writing
2c0ecb2 is described below
commit 2c0ecb2cba76ba5c70ea062e46d04dcc164e3d03
Author: Igor Sapego
AuthorDate: Tue Feb 2 16:36:45 2021 +0300
IGNITE-13863: Fix Null reading and writing
This closes #6
---
pyignite/datatypes/complex.py | 69 -
pyignite/datatypes/internal.py | 5 ++-
pyignite/datatypes/primitive_arrays.py | 33 +---
pyignite/datatypes/primitive_objects.py | 23 +--
pyignite/datatypes/standard.py | 28 +
tests/test_binary.py| 46 +++---
6 files changed, 172 insertions(+), 32 deletions(-)
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index ad2a770..6860583 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -20,11 +20,13 @@ from typing import Iterable, Dict
from pyignite.constants import *
from pyignite.exceptions import ParseError
+
from .base import IgniteDataType
from .internal import AnyDataObject, infer_from_python
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -68,8 +70,13 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -91,7 +98,10 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
-for i in range(ctype_object.length):
+length = getattr(ctype_object, "length", None)
+if length is None:
+return None
+for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -102,6 +112,9 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+if value is None:
+return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -150,8 +163,13 @@ class WrappedDataObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
@@ -243,8 +261,13 @@ class CollectionObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -266,7 +289,10 @@ class CollectionObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
-for i in range(ctype_object.length):
+length = getattr(ctype_object, "length", None)
+if length is None:
+return None
+for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -277,6 +303,9 @@ class CollectionObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+if value is None:
+return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -330,8 +359,13 @@ class Map(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer =
[ignite] branch master updated: IGNITE-14111 Add javadoc for AbstractDataPageIO - Fixes #8742.
This is an automated email from the ASF dual-hosted git repository.
agoncharuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 02043cc IGNITE-14111 Add javadoc for AbstractDataPageIO - Fixes #8742.
02043cc is described below
commit 02043ccf0fa42ce0892649d4303d0020e7938faf
Author: Alexey Goncharuk
AuthorDate: Tue Feb 2 16:28:24 2021 +0300
IGNITE-14111 Add javadoc for AbstractDataPageIO - Fixes #8742.
Signed-off-by: Alexey Goncharuk
---
.../persistence/tree/io/AbstractDataPageIO.java| 124 +
1 file changed, 124 insertions(+)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
index 4708f423..55690b8 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
@@ -37,6 +37,130 @@ import static
org.apache.ignite.internal.util.GridUnsafe.bufferAddress;
/**
* Data pages IO.
+ *
+ * Rows in a data page are organized into two arrays growing toward each
other: items table and row data.
+ *
+ * Items table contains direct or indirect items which locate a row within the
page. Items table is stored at the
+ * beginning of a page. Each item has an item ID which serves as an external
item reference
+ * (see {@link PageIdUtils#link(long, int)}) and can be either direct or
indirect. The size of any item in the items
+ * table is 2 bytes. ID of a direct item is always the same as its index in
items table so it is not stored in the
+ * item itself. ID of an indirect item may differ from its index (see example
below) so it is stored it the item
+ * along with ID (index) of direct item.
+ *
+ * Direct and indirect items are always placed in the items table in such a
way that direct items are stored first,
+ * and indirect items are always stored after direct items. A data page
explicitly stores both direct and indirect
+ * items count (see {@link #getDirectCount(long)} and {@link
#getIndirectCount(long)}), so that the item type can be
+ * easily determined: items with indexes {@code [0, directCnt)} are always
direct and items with indexes
+ * {@code [directCnt, directCnt + indirectCnt)} are always indirect. Having
both direct and indirect items in a
+ * page allows page defragmentation without breaking external links. Total
number of rows stored in a page is equal
+ * to the number of direct items.
+ *
+ * The row data is stored at the end of the page; newer rows are stored closer
to the end of the items table.
+ * Direct Items
+ * Direct items refer a stored row directly by offset in the page:
+ *
+ *
+-+
+ * | Direct Items . (rows data)
|
+ * | (4000), (3800), (3600) . row_2_ row_1_ row_0_
|
+ * | | | |_^ ^^
|
+ * | | |_||
|
+ * | |__|
|
+ * | directCnt: 3
|
+ * | indirectCnt: 0
|
+ *
+-+
+ *
+ * Direct item ID always matches it's index in the items table. The value of a
direct item in the table is an
+ * offset of the row data within the page.
+ * Indirect Items
+ * An indirect item explicitly stores the indirect item ID (1 byte) and the
index of the direct item it refers to
+ * (1 byte). The referred direct item (referrent) stores the actual offset of
the row in the data page:
+ *
+ *
+-+
+ * | Direct Items .. Indirect items (rows data)
|
+ * |
|
+ * | ⌄|
|
+ * | (3600), (3800), (3 -> 0) row_2_ row_1_
|
+ * | | |_^___^
|
+ * | |_|
|
+ * | directCnt: 2
|
+ * | indirectCount: 1
[ignite] branch sql-calcite updated: IGNITE-13566 Calcite integration. Table size statistics for cost planer (#8678)
This is an automated email from the ASF dual-hosted git repository.
tledkov pushed a commit to branch sql-calcite
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/sql-calcite by this push:
new 563c832 IGNITE-13566 Calcite integration. Table size statistics for
cost planer (#8678)
563c832 is described below
commit 563c832eb2d5d40fcd6aa52c3696c6f854929787
Author: Stanilovsky Evgeny
AuthorDate: Tue Feb 2 14:10:33 2021 +0300
IGNITE-13566 Calcite integration. Table size statistics for cost planer
(#8678)
---
.../query/calcite/rel/AbstractIndexScan.java | 6 +-
.../query/calcite/schema/IgniteTableImpl.java | 18 +-
.../query/calcite/schema/TableDescriptor.java | 6 +
.../query/calcite/schema/TableDescriptorImpl.java | 5 +
.../CalciteBasicSecondaryIndexIntegrationTest.java | 267 +++--
.../query/calcite/CalciteQueryProcessorTest.java | 14 --
.../query/calcite/SqlFieldsQueryUsageTest.java | 103
.../calcite/planner/IndexSpoolPlannerTest.java | 1 -
.../query/calcite/rules/OrToUnionRuleTest.java | 21 +-
.../ignite/testsuites/IgniteCalciteTestSuite.java | 4 +-
.../processors/query/GridQueryProcessor.java | 5 +
.../processors/query/h2/opt/GridH2Table.java | 49
.../processors/cache/index/BasicIndexTest.java | 4 +-
.../query/h2/TableStatisticsAbstractTest.java | 2 +-
14 files changed, 464 insertions(+), 41 deletions(-)
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/AbstractIndexScan.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/AbstractIndexScan.java
index 7260e75..00e3a4c 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/AbstractIndexScan.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/AbstractIndexScan.java
@@ -132,15 +132,15 @@ public abstract class AbstractIndexScan extends
ProjectableFilterableTableScan {
cost = 0;
if (lowerCondition() != null) {
-double selectivity0 = mq.getSelectivity(this,
RexUtil.composeDisjunction(builder, lowerCondition()));
+double selectivity0 = mq.getSelectivity(this,
RexUtil.composeConjunction(builder, lowerCondition()));
selectivity -= 1 - selectivity0;
cost += Math.log(rows);
}
-if (upperCondition() != null) {
-double selectivity0 = mq.getSelectivity(this,
RexUtil.composeDisjunction(builder, upperCondition()));
+if (upperCondition() != null && lowerCondition() != null &&
!lowerCondition().equals(upperCondition())) {
+double selectivity0 = mq.getSelectivity(this,
RexUtil.composeConjunction(builder, upperCondition()));
selectivity -= 1 - selectivity0;
}
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteTableImpl.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteTableImpl.java
index 96cde58..3c874f1 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteTableImpl.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteTableImpl.java
@@ -45,6 +45,8 @@ import
org.apache.ignite.internal.processors.query.calcite.rel.logical.IgniteLog
import
org.apache.ignite.internal.processors.query.calcite.trait.IgniteDistribution;
import
org.apache.ignite.internal.processors.query.calcite.trait.RewindabilityTrait;
import
org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
import org.jetbrains.annotations.Nullable;
/**
@@ -58,6 +60,9 @@ public class IgniteTableImpl extends AbstractTable implements
IgniteTable {
private final Statistic statistic;
/** */
+private volatile GridH2Table tbl;
+
+/** */
private final Map indexes = new ConcurrentHashMap<>();
/**
@@ -75,6 +80,15 @@ public class IgniteTableImpl extends AbstractTable
implements IgniteTable {
/** {@inheritDoc} */
@Override public Statistic getStatistic() {
+if (tbl == null) {
+IgniteH2Indexing idx = (IgniteH2Indexing)
desc.cacheContext().kernalContext().query().getIndexing();
+
+final String tblName = desc.typeDescription().tableName();
+final String schemaName = desc.typeDescription().schemaName();
+
+tbl = idx.schemaManager().dataTable(schemaName, tblName);
+}
+
return statistic;
}
@@ -158,7 +172,9 @@ public class IgniteTableImpl extends
[ignite] branch master updated: IGNITE-14064 .NET: Fix SQL table name for generic query types
This is an automated email from the ASF dual-hosted git repository.
ptupitsyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 8ab3211 IGNITE-14064 .NET: Fix SQL table name for generic query types
8ab3211 is described below
commit 8ab32115ab98e1f814933fc6de2df729c1a356a6
Author: Pavel Tupitsyn
AuthorDate: Tue Feb 2 12:48:13 2021 +0300
IGNITE-14064 .NET: Fix SQL table name for generic query types
* Fix `BinaryNameMapper.simpleName` to strip namespaces from generic types
* Fix `QueryUtils.typeName` to strip generic part
---
.../ignite/binary/BinaryBasicNameMapper.java | 24
.../internal/processors/query/QueryUtils.java | 5 +
.../binary/BinaryBasicNameMapperSelfTest.java | 21 +++
.../Query/CacheQueriesCodeConfigurationTest.cs | 17 +--
.../Cache/Query/Linq/CacheLinqTest.Misc.cs | 148 +
5 files changed, 203 insertions(+), 12 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/binary/BinaryBasicNameMapper.java
b/modules/core/src/main/java/org/apache/ignite/binary/BinaryBasicNameMapper.java
index 697108f..cf99e95 100644
---
a/modules/core/src/main/java/org/apache/ignite/binary/BinaryBasicNameMapper.java
+++
b/modules/core/src/main/java/org/apache/ignite/binary/BinaryBasicNameMapper.java
@@ -87,6 +87,8 @@ public class BinaryBasicNameMapper implements
BinaryNameMapper {
private static String simpleName(String clsName) {
assert clsName != null;
+clsName = simplifyDotNetGenerics(clsName);
+
int idx = clsName.lastIndexOf('$');
if (idx == clsName.length() - 1)
@@ -119,6 +121,28 @@ public class BinaryBasicNameMapper implements
BinaryNameMapper {
return idx >= 0 ? clsName.substring(idx + 1) : clsName;
}
+/**
+ * Converts .NET generic type arguments to a simple form (without
namespaces and outer classes classes).
+ *
+ * @param clsName Class name.
+ * @return Simplified class name.
+ */
+private static String simplifyDotNetGenerics(String clsName) {
+// .NET generic part starts with [[ (not valid for Java class name).
Clean up every generic part recursively.
+// Example: Foo.Bar`1[[Baz.Qux`2[[System.String],[System.Int32
+int genericIdx = clsName.indexOf("[[");
+
+if (genericIdx > 0)
+clsName = clsName.substring(0, genericIdx + 2) +
simpleName(clsName.substring(genericIdx + 2));
+
+genericIdx = clsName.indexOf("],[", genericIdx);
+
+if (genericIdx > 0)
+clsName = clsName.substring(0, genericIdx + 3) +
simpleName(clsName.substring(genericIdx + 3));
+
+return clsName;
+}
+
/** {@inheritDoc} */
@Override public boolean equals(Object o) {
if (this == o)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
index c561a5e..bff1ae2 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
@@ -1115,6 +1115,11 @@ public class QueryUtils {
* @return Type name.
*/
public static String typeName(String clsName) {
+int genericStart = clsName.indexOf('`'); // .NET generic, not valid
for Java class name.
+
+if (genericStart >= 0)
+clsName = clsName.substring(0, genericStart);
+
int pkgEnd = clsName.lastIndexOf('.');
if (pkgEnd >= 0 && pkgEnd < clsName.length() - 1)
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryBasicNameMapperSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryBasicNameMapperSelfTest.java
index cd4cff1..7237cca 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryBasicNameMapperSelfTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryBasicNameMapperSelfTest.java
@@ -42,6 +42,27 @@ public class BinaryBasicNameMapperSelfTest extends
GridCommonAbstractTest {
* @throws Exception If failed.
*/
@Test
+public void testSimpleNameDotNet() throws Exception {
+BinaryBasicNameMapper mapper = new BinaryBasicNameMapper(true);
+
+assertEquals("Baz", mapper.typeName("Foo.Bar.Baz"));
+
+assertEquals("Bar`1[[Qux]]", mapper.typeName("Foo.Bar`1[[Baz.Qux]]"));
+
+assertEquals("List`1[[Int32[]]]",
+
mapper.typeName("System.Collections.Generic.List`1[[System.Int32[]]]"));
+
+assertEquals("Bar`1[[Qux`2[[String],[Int32",
+
mapper.typeName("Foo.Bar`1[[Baz.Qux`2[[System.String],[System.Int32"));
+
+assertEquals("Bar`1[[Qux`2[[C],[Int32",
+
[ignite-python-thin-client] 02/02: IGNITE-13863: Minor fix
This is an automated email from the ASF dual-hosted git repository. isapego pushed a commit to branch ignite-13863 in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git commit bf76b27d30c13b6e286c6f38e84973bddb6807b1 Author: Igor Sapego AuthorDate: Thu Jan 28 01:19:27 2021 +0300 IGNITE-13863: Minor fix --- pyignite/datatypes/complex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py index 2d4de53..7fa6bc2 100644 --- a/pyignite/datatypes/complex.py +++ b/pyignite/datatypes/complex.py @@ -454,10 +454,10 @@ class MapObject(Map): @classmethod def to_python(cls, ctype_object, *args, **kwargs): -type = getattr(ctype_object, "type", None) -if type is None: +typ = getattr(ctype_object, "type", None) +if typ is None: return None -return ctype_object.type, super().to_python( +return typ, super().to_python( ctype_object, *args, **kwargs )
[ignite-python-thin-client] 01/02: IGNITE-13863: Fix Null reading and writing
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch ignite-13863
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
commit 999ea3f4534c47269bd3111acc00a909641eba58
Author: Igor Sapego
AuthorDate: Wed Dec 2 18:26:44 2020 +0300
IGNITE-13863: Fix Null reading and writing
---
pyignite/datatypes/complex.py | 67 +---
pyignite/datatypes/internal.py | 5 +-
pyignite/datatypes/primitive_arrays.py | 33 --
pyignite/datatypes/primitive_objects.py | 23 +--
pyignite/datatypes/standard.py | 28 ++---
tests/test_binary.py| 106 +++-
6 files changed, 233 insertions(+), 29 deletions(-)
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index ad2a770..2d4de53 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -20,11 +20,13 @@ from typing import Iterable, Dict
from pyignite.constants import *
from pyignite.exceptions import ParseError
+
from .base import IgniteDataType
from .internal import AnyDataObject, infer_from_python
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -68,8 +70,13 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -91,7 +98,10 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
-for i in range(ctype_object.length):
+length = getattr(ctype_object, "length", None)
+if length is None:
+return None
+for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -102,6 +112,9 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+if value is None:
+return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -150,8 +163,13 @@ class WrappedDataObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
@@ -243,8 +261,13 @@ class CollectionObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -266,7 +289,10 @@ class CollectionObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
-for i in range(ctype_object.length):
+length = getattr(ctype_object, "length", None)
+if length is None:
+return None
+for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -277,6 +303,9 @@ class CollectionObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+if value is None:
+return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -330,8 +359,13 @@ class Map(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+if tc_type == TC_NULL:
+return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
-buffer = client.recv(ctypes.sizeof(header_class))
+buffer = tc_type + client.recv(ctypes.sizeof(header_class) -
len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -420,12
[ignite-python-thin-client] branch ignite-13863 created (now bf76b27)
This is an automated email from the ASF dual-hosted git repository. isapego pushed a change to branch ignite-13863 in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git. at bf76b27 IGNITE-13863: Minor fix This branch includes the following new commits: new 999ea3f IGNITE-13863: Fix Null reading and writing new bf76b27 IGNITE-13863: Minor fix The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[ignite-python-thin-client] branch master updated: IGNITE-14059: Fix hashing of complex objects
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
The following commit(s) were added to refs/heads/master by this push:
new e0c22ef IGNITE-14059: Fix hashing of complex objects
e0c22ef is described below
commit e0c22ef3aef39ea8a42ddb6b4495b7bcaa479417
Author: Igor Sapego
AuthorDate: Tue Feb 2 12:16:43 2021 +0300
IGNITE-14059: Fix hashing of complex objects
This closes #5
---
pyignite/utils.py| 35 ++-
tests/test_binary.py | 51 +++
2 files changed, 77 insertions(+), 9 deletions(-)
diff --git a/pyignite/utils.py b/pyignite/utils.py
index ebe5501..ce00d53 100644
--- a/pyignite/utils.py
+++ b/pyignite/utils.py
@@ -106,20 +106,37 @@ def unwrap_binary(client: 'Client', wrapped: tuple) ->
object:
return result
-def hashcode(string: Union[str, bytes]) -> int:
+def hashcode(data: Union[str, bytes]) -> int:
"""
Calculate hash code used for identifying objects in Ignite binary API.
-:param string: UTF-8-encoded string identifier of binary buffer,
+:param data: UTF-8-encoded string identifier of binary buffer or byte array
:return: hash code.
"""
-result = 1 if isinstance(string, (bytes, bytearray)) else 0
-for char in string:
-try:
-char = ord(char)
-except TypeError:
-pass
-result = int_overflow(31 * result + char)
+if isinstance(data, str):
+"""
+For strings we iterate over code point which are of the int type
+and can take up to 4 bytes and can only be positive.
+"""
+result = 0
+for char in data:
+try:
+char_val = ord(char)
+result = int_overflow(31 * result + char_val)
+except TypeError:
+pass
+else:
+"""
+For byte array we iterate over bytes which only take 1 byte. But
+according to protocol, bytes during hashing should be treated as signed
+integer numbers 8 bits long. On other hand elements in Python's `bytes`
+are unsigned. For this reason we use ctypes.c_byte() to make them
+signed.
+"""
+result = 1
+for byte in data:
+byte = ctypes.c_byte(byte).value
+result = int_overflow(31 * result + byte)
return result
diff --git a/tests/test_binary.py b/tests/test_binary.py
index 5190a6a..4c45afb 100644
--- a/tests/test_binary.py
+++ b/tests/test_binary.py
@@ -304,3 +304,54 @@ def test_complex_object_names(client):
obj = cache.get(key)
assert obj.type_name == type_name, 'Complex type name mismatch'
assert obj.field == data, 'Complex object data failure'
+
+
+def test_complex_object_hash(client):
+"""
+Test that Python client correctly calculates hash of the binary
+object that contains negative bytes.
+"""
+class Internal(
+metaclass=GenericObjectMeta,
+type_name='Internal',
+schema=OrderedDict([
+('id', IntObject),
+('str', String),
+])
+):
+pass
+
+class TestObject(
+metaclass=GenericObjectMeta,
+type_name='TestObject',
+schema=OrderedDict([
+('id', IntObject),
+('str', String),
+('internal', BinaryObject),
+])
+):
+pass
+
+obj_ascii = TestObject()
+obj_ascii.id = 1
+obj_ascii.str = 'test_string'
+
+obj_ascii.internal = Internal()
+obj_ascii.internal.id = 2
+obj_ascii.internal.str = 'lorem ipsum'
+
+hash_ascii = BinaryObject.hashcode(obj_ascii, client=client)
+
+assert hash_ascii == -1314567146, 'Invalid hashcode value for object with
ASCII strings'
+
+obj_utf8 = TestObject()
+obj_utf8.id = 1
+obj_utf8.str = 'юникод'
+
+obj_utf8.internal = Internal()
+obj_utf8.internal.id = 2
+obj_utf8.internal.str = 'ユニコード'
+
+hash_utf8 = BinaryObject.hashcode(obj_utf8, client=client)
+
+assert hash_utf8 == -1945378474, 'Invalid hashcode value for object with
UTF-8 strings'
