This is an automated email from the ASF dual-hosted git repository. ivandasch pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new 3c27c15 IGNITE-15922 Implement NUMA-aware allocator - Fixes #9569. 3c27c15 is described below commit 3c27c1579a11727e425e7132def72c859ec4de24 Author: Ivan Daschinsky <ivanda...@apache.org> AuthorDate: Mon Dec 13 16:09:50 2021 +0300 IGNITE-15922 Implement NUMA-aware allocator - Fixes #9569. Signed-off-by: Ivan Daschinsky <ivanda...@apache.org> --- .travis.yml | 15 +- .../configuration/DataRegionConfiguration.java | 23 +++ .../configuration/DataStorageConfiguration.java | 25 +++ .../internal/mem/unsafe/UnsafeMemoryAllocator.java | 33 ++++ .../internal/mem/unsafe/UnsafeMemoryProvider.java | 36 ++-- .../IgniteCacheDatabaseSharedManager.java | 8 +- .../org/apache/ignite/mem/MemoryAllocator.java | 36 ++++ modules/numa-allocator/README.md | 141 ++++++++++++++ modules/numa-allocator/pom.xml | 210 ++++++++++++++++++++ modules/numa-allocator/src/main/cpp/CMakeLists.txt | 45 +++++ .../src/main/cpp/include/numa/numa_alloc.h | 110 +++++++++++ .../org_apache_ignite_internal_mem_NumaAllocUtil.h | 38 ++++ .../src/main/cpp/src/numa/numa_alloc.cpp | 214 +++++++++++++++++++++ ...rg_apache_ignite_internal_mem_NumaAllocUtil.cpp | 129 +++++++++++++ .../apache/ignite/internal/mem/NumaAllocUtil.java | 151 +++++++++++++++ .../mem/InterleavedNumaAllocationStrategy.java | 78 ++++++++ .../ignite/mem/LocalNumaAllocationStrategy.java | 44 +++++ .../apache/ignite/mem/NumaAllocationStrategy.java | 30 +++ .../java/org/apache/ignite/mem/NumaAllocator.java | 53 +++++ .../ignite/mem/SimpleNumaAllocationStrategy.java | 77 ++++++++ .../internal/mem/NumaAllocatorBasicTest.java | 177 +++++++++++++++++ .../ignite/internal/mem/NumaAllocatorUnitTest.java | 121 ++++++++++++ .../ignite/testsuites/NumaAllocatorTestSuite.java | 32 +++ parent/pom.xml | 4 + pom.xml | 8 + 25 files changed, 1822 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index d073814..837f7c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,10 @@ matrix: include: - language: java os: linux - dist: xenial + dist: bionic + before_install: + - sudo apt-get update + - sudo apt-get -y install libnuma-dev install: skip jdk: openjdk8 before_script: @@ -32,7 +35,10 @@ matrix: - language: java os: linux - dist: xenial + dist: bionic + before_install: + - sudo apt-get update + - sudo apt-get -y install libnuma-dev install: skip jdk: openjdk11 before_script: @@ -58,7 +64,10 @@ matrix: - language: java name: "Check test suites" os: linux - dist: xenial + dist: bionic + before_install: + - sudo apt-get update + - sudo apt-get -y install libnuma-dev install: skip jdk: openjdk8 script: mvn test -Pcheck-test-suites,all-java,all-scala,scala -B -V diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java index 3f3b09a..2865bab4 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java @@ -20,6 +20,7 @@ import java.io.Serializable; import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.mem.MemoryAllocator; import org.apache.ignite.mxbean.DataRegionMetricsMXBean; import org.apache.ignite.mxbean.MetricsMxBean; import org.jetbrains.annotations.Nullable; @@ -152,6 +153,9 @@ public final class DataRegionConfiguration implements Serializable { /** Warm-up configuration. */ @Nullable private WarmUpConfiguration warmUpCfg; + /** Memory allocator. */ + @Nullable private MemoryAllocator memoryAllocator = null; + /** * Gets data region name. * @@ -245,6 +249,25 @@ public final class DataRegionConfiguration implements Serializable { } /** + * @return Memory allocator instance. + */ + @Nullable public MemoryAllocator getMemoryAllocator() { + return memoryAllocator; + } + + /** + * Sets memory allocator. If not specified, default, based on {@code Unsafe} allocator will be used. + * + * @param allocator Memory allocator instance. + * @return {@code this} for chaining. + */ + public DataRegionConfiguration setMemoryAllocator(MemoryAllocator allocator) { + memoryAllocator = allocator; + + return this; + } + + /** * Gets memory pages eviction mode. If {@link DataPageEvictionMode#DISABLED} is used (default) then an out of * memory exception will be thrown if the memory region usage, defined by this data region, goes beyond its * capacity which is {@link #getMaxSize()}. diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java index 7a47faa..6e4801b 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteExperimental; +import org.apache.ignite.mem.MemoryAllocator; import org.apache.ignite.mxbean.MetricsMxBean; import org.jetbrains.annotations.Nullable; @@ -337,6 +338,9 @@ public class DataStorageConfiguration implements Serializable { /** Minimum size of wal archive folder, in bytes. */ private long minWalArchiveSize = HALF_MAX_WAL_ARCHIVE_SIZE; + /** Default memory allocator for all data regions. */ + @Nullable private MemoryAllocator memoryAllocator = null; + /** * Creates valid durable memory configuration with all default values. */ @@ -1364,6 +1368,27 @@ public class DataStorageConfiguration implements Serializable { return this; } + /** + * @return Memory allocator instance. + */ + @Nullable public MemoryAllocator getMemoryAllocator() { + return memoryAllocator; + } + + /** + * Sets default memory allocator for all memory regions. If not specified, default, based on {@code Unsafe} + * allocator will be used. Allocator can be overrided for data region using + * {@link DataRegionConfiguration#setMemoryAllocator(MemoryAllocator)} + * + * @param allocator Memory allocator instance. + * @return {@code this} for chaining. + */ + public DataStorageConfiguration setMemoryAllocator(MemoryAllocator allocator) { + memoryAllocator = allocator; + + return this; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(DataStorageConfiguration.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryAllocator.java new file mode 100644 index 0000000..2d80390 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryAllocator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.mem.unsafe; + +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.mem.MemoryAllocator; + +/** */ +public class UnsafeMemoryAllocator implements MemoryAllocator { + /** {@inheritDoc} */ + @Override public long allocateMemory(long size) { + return GridUnsafe.allocateMemory(size); + } + + /** {@inheritDoc} */ + @Override public void freeMemory(long addr) { + GridUnsafe.freeMemory(addr); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java index 8cb8119..d281912 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java @@ -25,8 +25,9 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.mem.DirectMemoryProvider; import org.apache.ignite.internal.mem.DirectMemoryRegion; import org.apache.ignite.internal.mem.UnsafeChunk; -import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.mem.MemoryAllocator; +import org.jetbrains.annotations.Nullable; /** * Memory provider implementation based on unsafe memory access. @@ -49,10 +50,22 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { /** */ private int used = 0; + /** */ + private final MemoryAllocator allocator; + + /** + * @param log Ignite logger to use. + */ + public UnsafeMemoryProvider(@Nullable IgniteLogger log) { + this(log, null); + } + /** * @param log Ignite logger to use. + * @param allocator Memory allocator. If {@code null}, default {@link UnsafeMemoryAllocator} will be used. */ - public UnsafeMemoryProvider(IgniteLogger log) { + public UnsafeMemoryProvider(@Nullable IgniteLogger log, @Nullable MemoryAllocator allocator) { + this.allocator = allocator == null ? new UnsafeMemoryAllocator() : allocator; this.log = log; } @@ -70,20 +83,21 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { /** {@inheritDoc} */ @Override public void shutdown(boolean deallocate) { + if (!deallocate) { + used = 0; + + return; + } + if (regions != null) { for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) { DirectMemoryRegion chunk = it.next(); - if (deallocate) { - GridUnsafe.freeMemory(chunk.address()); + allocator.freeMemory(chunk.address()); - // Safety. - it.remove(); - } + // Safety. + it.remove(); } - - if (!deallocate) - used = 0; } } @@ -100,7 +114,7 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider { long ptr; try { - ptr = GridUnsafe.allocateMemory(chunkSize); + ptr = allocator.allocateMemory(chunkSize); } catch (IllegalArgumentException e) { String msg = "Failed to allocate next memory chunk: " + U.readableSize(chunkSize, true) + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index e8a6da0..0742afd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -1276,7 +1276,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap boolean trackable, PageReadWriteManager pmPageMgr ) throws IgniteCheckedException { - PageMemory pageMem = createPageMemory(createOrReuseMemoryProvider(plcCfg), memCfg, plcCfg, memMetrics, trackable, pmPageMgr); + if (plcCfg.getMemoryAllocator() == null) + plcCfg.setMemoryAllocator(memCfg.getMemoryAllocator()); + + PageMemory pageMem = createPageMemory(createOrReuseMemoryProvider(plcCfg), memCfg, plcCfg, memMetrics, + trackable, pmPageMgr); return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem)); } @@ -1315,7 +1319,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap File allocPath = buildAllocPath(plcCfg); return allocPath == null ? - new UnsafeMemoryProvider(log) : + new UnsafeMemoryProvider(log, plcCfg.getMemoryAllocator()) : new MappedFileMemoryProvider( log, allocPath); diff --git a/modules/core/src/main/java/org/apache/ignite/mem/MemoryAllocator.java b/modules/core/src/main/java/org/apache/ignite/mem/MemoryAllocator.java new file mode 100644 index 0000000..0d68739 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/mem/MemoryAllocator.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.mem; + +/** + * Base interface for offheap memory allocator. + */ +public interface MemoryAllocator { + /** + * @param size Size of allocated memory. + * + * @return Pointer to memory or {@code 0} if failed. + */ + public long allocateMemory(long size); + + /** + * Deallocates memory. + * + * @param addr Address of memory. + */ + public void freeMemory(long addr); +} diff --git a/modules/numa-allocator/README.md b/modules/numa-allocator/README.md new file mode 100644 index 0000000..89abbc4 --- /dev/null +++ b/modules/numa-allocator/README.md @@ -0,0 +1,141 @@ +# NUMA aware memory allocator for Apache Ignite +Allocates memory on linux using `libnuma` under the hood. + +## Requirements +`libnuma 2.0.x` must be installed on your Linux machine. It is recommended to install `numactl` utility also. +1. `RHEL` or `Cent OS` +```bash +$ sudo yum install numactl +``` +2. `Ubuntu` or `Debian` +```bash +$ sudo apt install numactl +``` + +## Building from source +`JDK`, `C++ 11` compatible compiler (`gcc >= 4.8.5`) and `libnuma` headers +1. `RHEL` or `Cent OS` +```bash +$ sudo yum groupinstall 'Development Tools' +$ sudo yum install java-1.8.0-openjdk numactl-devel libstdc++-static +``` +2. `Ubuntu` or `Debian` +```bash +$ sudo apt install build-essential libnuma-dev openjdk-8-jdk +``` +## Usage +### Simple allocation strategy +1. Allocation with default NUMA policy for thread/process, uses `void *numa_alloc(size_t)` under the hood: +```xml +<property name="dataStorageConfiguration"> + <bean class="org.apache.ignite.configuration.DataStorageConfiguration"> + <property name="defaultDataRegionConfiguration"> + <bean class="org.apache.ignite.configuration.DataRegionConfiguration"> + <property name="name" value="Default_Region"/> + .... + <property name="memoryAllocator"> + <bean class="org.apache.ignite.mem.NumaAllocator"> + <constructor-arg> + <bean class="org.apache.ignite.mem.SimpleNumaAllocationStrategy"/> + </constructor-arg> + </bean> + </property> + </bean> + </property> + </bean> +</property> +``` +2. Allocation on specific NUMA node, uses `void *numa_alloc_onnode(size_t, int)` under the hood: +```xml +<property name="dataStorageConfiguration"> + <bean class="org.apache.ignite.configuration.DataStorageConfiguration"> + <property name="defaultDataRegionConfiguration"> + <bean class="org.apache.ignite.configuration.DataRegionConfiguration"> + <property name="name" value="Default_Region"/> + .... + <property name="memoryAllocator"> + <bean class="org.apache.ignite.mem.NumaAllocator"> + <constructor-arg> + <bean class="org.apache.ignite.mem.SimpleNumaAllocationStrategy"> + <constructor-arg name="node" value="0"/> + </bean> + </constructor-arg> + </bean> + </property> + </bean> + </property> + </bean> +</property> +``` +### Interleaved allocation strategy. +1. Interleaved allocation on all NUMA nodes, uses `void *numa_alloc_interleaved(size_t)` under the hood: +```xml +<property name="dataStorageConfiguration"> + <bean class="org.apache.ignite.configuration.DataStorageConfiguration"> + <property name="defaultDataRegionConfiguration"> + <bean class="org.apache.ignite.configuration.DataRegionConfiguration"> + <property name="name" value="Default_Region"/> + .... + <property name="memoryAllocator"> + <bean class="org.apache.ignite.mem.NumaAllocator"> + <constructor-arg> + <bean class="org.apache.ignite.mem.InterleavedNumaAllocationStrategy"/> + </constructor-arg> + </bean> + </property> + </bean> + </property> + </bean> +</property> +``` +2. Interleaved allocation on specified NUMA nodes, uses `void *numa_alloc_interleaved_subset(size_t, struct bitmask*)` +under the hood: +```xml +<property name="dataStorageConfiguration"> + <bean class="org.apache.ignite.configuration.DataStorageConfiguration"> + <property name="defaultDataRegionConfiguration"> + <bean class="org.apache.ignite.configuration.DataRegionConfiguration"> + <property name="name" value="Default_Region"/> + .... + <property name="memoryAllocator"> + <bean class="org.apache.ignite.mem.NumaAllocator"> + <constructor-arg> + <bean class="org.apache.ignite.mem.InterleavedNumaAllocationStrategy"> + <constructor-arg name="nodes"> + <array> + <value>0</value> + <value>1</value> + </array> + </constructor-arg> + </bean> + </constructor-arg> + </bean> + </property> + </bean> + </property> + </bean> +</property> +``` +## Local node allocation strategy +Allocation on local for process NUMA node, uses `void* numa_alloc_onnode(size_t)` under the hood. +```xml +<property name="dataStorageConfiguration"> + <bean class="org.apache.ignite.configuration.DataStorageConfiguration"> + <property name="defaultDataRegionConfiguration"> + <bean class="org.apache.ignite.configuration.DataRegionConfiguration"> + <property name="name" value="Default_Region"/> + .... + <property name="memoryAllocator"> + <bean class="org.apache.ignite.mem.NumaAllocator"> + <constructor-arg> + <constructor-arg> + <bean class="org.apache.ignite.mem.LocalNumaAllocationStrategy"/> + </constructor-arg> + </constructor-arg> + </bean> + </property> + </bean> + </property> + </bean> +</property> +``` diff --git a/modules/numa-allocator/pom.xml b/modules/numa-allocator/pom.xml new file mode 100644 index 0000000..830330a --- /dev/null +++ b/modules/numa-allocator/pom.xml @@ -0,0 +1,210 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + POM file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-parent</artifactId> + <version>1</version> + <relativePath>../../parent</relativePath> + </parent> + + <artifactId>ignite-numa-allocator</artifactId> + <version>${revision}</version> + <url>http://ignite.apache.org</url> + + <dependencies> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>ignite-tools</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-beans</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.thoughtworks.xstream</groupId> + <artifactId>xstream</artifactId> + <version>1.4.8</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>linux-build</id> + <activation> + <os> + <name>Linux</name> + <arch>amd64</arch> + </os> + </activation> + <properties> + <cmake.classifier>linux-x86_64</cmake.classifier> + </properties> + <build> + <plugins> + <plugin> + <groupId>com.googlecode.cmake-maven-project</groupId> + <artifactId>cmake-maven-plugin</artifactId> + <version>3.7.2-b1</version> + <executions> + <execution> + <id>cmake-generate</id> + <phase>generate-resources</phase> + <goals> + <goal>generate</goal> + </goals> + <configuration> + <classifier>${cmake.classifier}</classifier> + <sourcePath>${basedir}/src/main/cpp</sourcePath> + <targetPath>${project.build.directory}/cppbuild</targetPath> + <generator>Unix Makefiles</generator> + <options> + <option>-DCMAKE_BUILD_TYPE=Release</option> + </options> + </configuration> + </execution> + <execution> + <id>cmake-build</id> + <phase>generate-resources</phase> + <goals> + <goal>compile</goal> + </goals> + <configuration> + <classifier>${cmake.classifier}</classifier> + <projectDirectory>${project.build.directory}/cppbuild</projectDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <executions> + <execution> + <id>copy-sources</id> + <phase>generate-resources</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/classes/org/apache/ignite/internal/mem/linux/${os.arch}/</outputDirectory> + <resources> + <resource> + <directory>${project.build.directory}/cppbuild</directory> + <includes> + <include>*.so</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <build> + <plugins> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-libs</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <excludeTransitive>false</excludeTransitive> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8.2</version> + <configuration> + <skip>false</skip> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/modules/numa-allocator/src/main/cpp/CMakeLists.txt b/modules/numa-allocator/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..48b42fa --- /dev/null +++ b/modules/numa-allocator/src/main/cpp/CMakeLists.txt @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 3.6) +project(numa_alloc) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_BUILD_TYPE Release) +set(CMAKE_CXX_FLAGS "-shared -D_FORTIFY_SOURCE=2 -z noexecstack -z,relro -z,now -fPIC -Wformat -Wformat-security -Werror=format-security") +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pie") + +find_package(JNI REQUIRED) +find_package(Threads REQUIRED) +include_directories(${CMAKE_THREAD_LIBS_INIT}) + +find_library(NUMA_LIBRARIES NAMES libnuma.so) +if (NOT NUMA_LIBRARIES) + message(FATAL_ERROR "not found numa library") +endif (NOT NUMA_LIBRARIES) + +include_directories(${JAVA_INCLUDE_PATH}) +include_directories(${JAVA_INCLUDE_PATH2}) +include_directories(include) + +file(GLOB SOURCES + src/numa/numa_alloc.cpp + src/org_apache_ignite_internal_mem_NumaAllocUtil.cpp +) +add_library(numa_alloc SHARED ${SOURCES}) +message(STATUS "NUMA_LIBRARIES: ${NUMA_LIBRARIES}") +target_link_libraries(numa_alloc PRIVATE -static-libgcc -static-libstdc++ ${NUMA_LIBRARIES}) diff --git a/modules/numa-allocator/src/main/cpp/include/numa/numa_alloc.h b/modules/numa-allocator/src/main/cpp/include/numa/numa_alloc.h new file mode 100644 index 0000000..b116a40 --- /dev/null +++ b/modules/numa-allocator/src/main/cpp/include/numa/numa_alloc.h @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _NUMA_ALLOC_H +#define _NUMA_ALLOC_H + +#include <ostream> + +namespace numa { + class BitSet { + public: + BitSet(); + + BitSet(const BitSet &other); + + BitSet &operator=(const BitSet &other); + + ~BitSet(); + + class Reference { + public: + explicit Reference(BitSet &bitset, size_t pos) : bitset(&bitset), pos(pos) {} + + Reference &operator=(bool value) { + bitset->Set(pos, value); + return *this; + } + + Reference &operator=(const Reference &other) { + if (&other != this) { + bitset->Set(pos, other.bitset->Get(other.pos)); + } + return *this; + } + + explicit operator bool() const { + return bitset->Get(pos); + } + + private: + BitSet *bitset; + size_t pos; + }; + + Reference operator[](size_t pos) { + return Reference(*this, pos); + } + + bool operator[](size_t pos) const { + return this->Get(pos); + } + + bool operator==(const BitSet &other); + + bool operator!=(const BitSet &other); + + friend void *AllocInterleaved(size_t size, const BitSet &node_set); + + void Set(size_t pos, bool value = true); + + void Set(); + + void Reset(size_t pos); + + void Reset(); + + size_t Size() const; + + friend std::ostream &operator<<(std::ostream &os, const BitSet &set); + + private: + bool Get(size_t pos) const; + + class BitSetImpl; + + BitSetImpl *p_impl_; + }; + + int NumaNodesCount(); + + void *Alloc(size_t size); + + void *Alloc(size_t size, int node); + + void *AllocLocal(size_t size); + + void *AllocInterleaved(size_t size); + + void *AllocInterleaved(size_t size, const BitSet &node_set); + + size_t Size(void *ptr); + + void Free(void *ptr); +} + +#endif //_NUMA_ALLOC_H diff --git a/modules/numa-allocator/src/main/cpp/include/org_apache_ignite_internal_mem_NumaAllocUtil.h b/modules/numa-allocator/src/main/cpp/include/org_apache_ignite_internal_mem_NumaAllocUtil.h new file mode 100644 index 0000000..3a7bbb5 --- /dev/null +++ b/modules/numa-allocator/src/main/cpp/include/org_apache_ignite_internal_mem_NumaAllocUtil.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _ORG_APACHE_IGNITE_INTERNAL_MEM_NUMAALLOCUTIL_H +#define _ORG_APACHE_IGNITE_INTERNAL_MEM_NUMAALLOCUTIL_H + +#include <jni.h> + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocate(JNIEnv *, jclass, jlong); +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocateOnNode(JNIEnv *, jclass, jlong, jint); +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocateLocal(JNIEnv *, jclass, jlong); +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocateInterleaved(JNIEnv *, jclass, jlong, jintArray); +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_chunkSize(JNIEnv *, jclass, jlong); +JNIEXPORT void JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_free(JNIEnv *, jclass, jlong); +JNIEXPORT jint JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_nodesCount(JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif // _ORG_APACHE_IGNITE_INTERNAL_MEM_NUMAALLOCUTIL_H diff --git a/modules/numa-allocator/src/main/cpp/src/numa/numa_alloc.cpp b/modules/numa-allocator/src/main/cpp/src/numa/numa_alloc.cpp new file mode 100644 index 0000000..4194978 --- /dev/null +++ b/modules/numa-allocator/src/main/cpp/src/numa/numa_alloc.cpp @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <numa.h> +#include <numa/numa_alloc.h> + +namespace numa { + class BitSet::BitSetImpl { + public: + BitSetImpl() : size_(numa_max_node() + 1) { + mask_ = numa_bitmask_alloc(size_); + } + + BitSetImpl(const BitSetImpl &other) : size_(other.size_) { + mask_ = numa_bitmask_alloc(size_); + copy_bitmask_to_bitmask(other.mask_, mask_); + } + + void SetBit(size_t idx, bool val) { + if (idx < size_) { + if (val) + numa_bitmask_setbit(mask_, idx); + else + numa_bitmask_clearbit(mask_, idx); + } + } + + void SetAll(bool val) { + if (val) + numa_bitmask_setall(mask_); + else + numa_bitmask_clearall(mask_); + } + + bool GetBit(size_t idx) const { + if (idx < size_) + return numa_bitmask_isbitset(mask_, idx); + else + return false; + } + + bool Equals(const BitSetImpl &other) const { + return numa_bitmask_equal(mask_, other.mask_); + } + + size_t Size() const { + return size_; + } + + bitmask *Raw() { + return mask_; + } + + ~BitSetImpl() { + numa_bitmask_free(mask_); + } + + private: + bitmask *mask_; + size_t size_; + }; + + BitSet::BitSet() { + p_impl_ = new BitSetImpl(); + } + + BitSet::BitSet(const BitSet &other) { + p_impl_ = new BitSetImpl(*other.p_impl_); + } + + BitSet &BitSet::operator=(const BitSet &other) { + if (this != &other) { + BitSet tmp(other); + std::swap(this->p_impl_, tmp.p_impl_); + } + return *this; + } + + bool BitSet::Get(size_t pos) const { + return p_impl_->GetBit(pos); + } + + void BitSet::Set(size_t pos, bool value) { + p_impl_->SetBit(pos, value); + } + + void BitSet::Set() { + p_impl_->SetAll(true); + } + + void BitSet::Reset(size_t pos) { + p_impl_->SetBit(pos, false); + } + + void BitSet::Reset() { + p_impl_->SetAll(false); + } + + size_t BitSet::Size() const { + return p_impl_->Size(); + } + + bool BitSet::operator==(const BitSet &other) { + return this->p_impl_->Equals(*other.p_impl_); + } + + bool BitSet::operator!=(const BitSet &other) { + return !(*this == other); + } + + BitSet::~BitSet() { + delete p_impl_; + } + + std::ostream &operator<<(std::ostream &os, const BitSet &set) { + os << '{'; + for (size_t i = 0; i < set.Size(); ++i) { + os << set[i]; + if (i < set.Size() - 1) { + os << ", "; + } + } + os << '}'; + return os; + } + + int NumaNodesCount() { + return numa_max_node() + 1; + } + + /** + * Memory layout: + * +-------------------------------+------------------------+ + * | Header (sizeof(max_align_t)) | Application memory | + * +------------------------------ +------------------------+ + * ^ + * | + * Result pointer + * Size of application memory chunk is written to header. + * Total allocated size equals to size of application memory chunk plus sizeof(max_align_t). + */ + union region_size { + size_t size; + max_align_t a; + }; + + template<typename Func, typename ...Args> + inline void *NumaAllocHelper(Func f, size_t size, Args ...args) { + auto ptr = static_cast<region_size *>(f(size + sizeof(region_size), args...)); + if (ptr) { + ptr->size = size; + ptr++; + } + return ptr; + } + + inline region_size* ConvertPointer(void* buf) { + if (buf) { + auto *ptr = static_cast<region_size *>(buf); + ptr--; + return ptr; + } + return nullptr; + } + + void *Alloc(size_t size) { + return NumaAllocHelper(numa_alloc, size); + } + + void *Alloc(size_t size, int node) { + return NumaAllocHelper(numa_alloc_onnode, size, node); + } + + void *AllocLocal(size_t size) { + return NumaAllocHelper(numa_alloc_local, size); + } + + void *AllocInterleaved(size_t size) { + return NumaAllocHelper(numa_alloc_interleaved, size); + } + + void *AllocInterleaved(size_t size, const BitSet &node_set) { + return NumaAllocHelper(numa_alloc_interleaved_subset, size, node_set.p_impl_->Raw()); + } + + size_t Size(void *buf) { + auto ptr = ConvertPointer(buf); + if (ptr) { + return ptr->size; + } + return 0; + } + + void Free(void *buf) { + auto ptr = ConvertPointer(buf); + if (ptr) { + numa_free(ptr, ptr->size + sizeof(region_size)); + } + } +} diff --git a/modules/numa-allocator/src/main/cpp/src/org_apache_ignite_internal_mem_NumaAllocUtil.cpp b/modules/numa-allocator/src/main/cpp/src/org_apache_ignite_internal_mem_NumaAllocUtil.cpp new file mode 100644 index 0000000..38ad95b --- /dev/null +++ b/modules/numa-allocator/src/main/cpp/src/org_apache_ignite_internal_mem_NumaAllocUtil.cpp @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <numa/numa_alloc.h> +#include "org_apache_ignite_internal_mem_NumaAllocUtil.h" + +class JIntArrayGuard { +public: + JIntArrayGuard(JNIEnv* env, jintArray arr): env_(env), java_arr_(arr) { + arr_ = env->GetIntArrayElements(java_arr_, nullptr); + length_ = static_cast<size_t>(env->GetArrayLength(java_arr_)); + } + + JIntArrayGuard() = delete; + JIntArrayGuard(const JIntArrayGuard&) = delete; + JIntArrayGuard& operator=(const JIntArrayGuard&) = delete; + + ~JIntArrayGuard() { + env_->ReleaseIntArrayElements(java_arr_, arr_, 0); + } + + const int& operator[](size_t pos) const { + return arr_[pos]; + } + + int& operator[](size_t pos) { + return arr_[pos]; + } + + size_t Size() const { + return length_; + } +private: + JNIEnv* env_; + jintArray java_arr_; + jint* arr_; + size_t length_; +}; + +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocate( + JNIEnv*, + jclass, + jlong size +) { + void* ptr = numa::Alloc(static_cast<size_t>(size)); + return reinterpret_cast<jlong>(ptr); +} + +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocateOnNode( + JNIEnv*, + jclass, + jlong size, + jint node +) { + void* ptr; + auto size_ = static_cast<size_t>(size); + if (node >= 0 && node < numa::NumaNodesCount()) { + ptr = numa::Alloc(size_, static_cast<int>(node)); + } + else { + ptr = numa::Alloc(size_); + } + return reinterpret_cast<jlong>(ptr); +} + +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocateLocal( + JNIEnv*, + jclass, + jlong size +) { + void* ptr = numa::AllocLocal(static_cast<size_t>(size)); + return reinterpret_cast<jlong>(ptr); +} + +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_allocateInterleaved( + JNIEnv *jniEnv, + jclass, + jlong size, + jintArray arr +) { + void* ptr; + auto size_ = static_cast<size_t>(size); + if (arr != nullptr) { + JIntArrayGuard nodes(jniEnv, arr); + + if (nodes.Size() > 0) { + numa::BitSet bs; + for (size_t i = 0; i < nodes.Size(); ++i) { + bs.Set(nodes[i]); + } + ptr = numa::AllocInterleaved(size_, bs); + } + else { + ptr = numa::AllocInterleaved(size_); + } + } + else { + ptr = numa::AllocInterleaved(size_); + } + return reinterpret_cast<jlong>(ptr); +} + +JNIEXPORT jlong JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_chunkSize(JNIEnv *, jclass, jlong addr) { + void* ptr = reinterpret_cast<void*>(addr); + return static_cast<jlong>(numa::Size(ptr)); +} + +JNIEXPORT void JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_free(JNIEnv *, jclass, jlong addr) { + void* ptr = reinterpret_cast<void*>(addr); + numa::Free(ptr); +} + +JNIEXPORT jint JNICALL Java_org_apache_ignite_internal_mem_NumaAllocUtil_nodesCount(JNIEnv *, jclass) { + return static_cast<jint>(numa::NumaNodesCount()); +} diff --git a/modules/numa-allocator/src/main/java/org/apache/ignite/internal/mem/NumaAllocUtil.java b/modules/numa-allocator/src/main/java/org/apache/ignite/internal/mem/NumaAllocUtil.java new file mode 100644 index 0000000..18f753f --- /dev/null +++ b/modules/numa-allocator/src/main/java/org/apache/ignite/internal/mem/NumaAllocUtil.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.mem; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; + +/** */ +public class NumaAllocUtil { + /** */ + private static final String extension = ".so"; + + /** */ + private static final String libraryName = "libnuma_alloc"; + + /** */ + private NumaAllocUtil() { + // No-op. + } + + /** */ + private static String getLibName() { + return Paths.get("/org/apache/ignite/internal/mem", getOSName(), getOSArch(), libraryName + extension) + .toString(); + } + + /** */ + private static String getOSArch() { + return System.getProperty("os.arch", ""); + } + + /** */ + private static String getOSName() { + if (System.getProperty("os.name", "").contains("Linux")) + return "linux"; + else + throw new UnsupportedOperationException("Operating System is not supported"); + } + + static { + String libName = getLibName(); + File nativeLib = null; + + try (InputStream in = NumaAllocUtil.class.getResourceAsStream(libName)) { + if (in == null) { + throw new ExceptionInInitializerError("Failed to load native numa_alloc library, " + + libName + " not found"); + } + + nativeLib = File.createTempFile(libraryName, extension); + + try (FileOutputStream out = new FileOutputStream(nativeLib)) { + byte[] buf = new byte[4096]; + + int bytesRead; + while ((bytesRead = in.read(buf)) > 0) + out.write(buf, 0, bytesRead); + } + + System.load(nativeLib.getAbsolutePath()); + + NUMA_NODES_CNT = nodesCount(); + } + catch (IOException | UnsatisfiedLinkError e) { + throw new ExceptionInInitializerError(e); + } + finally { + if (nativeLib != null) + nativeLib.deleteOnExit(); + } + } + + /** */ + public static final int NUMA_NODES_CNT; + + /** + * Allocate memory using using default NUMA memory policy of current thread. + * Uses {@code void *numa_alloc(size_t)} under the hood. + * <p> + * @param size Size of buffer. + * @return Address of buffer. + */ + public static native long allocate(long size); + + /** + * Allocate memory on specific NUMA node. Uses {@code void *numa_alloc_onnode(size_t)} under the hood. + * <p> + * @param size Size of buffer. + * @param node NUMA node. + * @return Address of buffer. + */ + public static native long allocateOnNode(long size, int node); + + /** + * Allocate memory on local NUMA node. Uses {@code void *numa_alloc_local(size_t)} under the hood. + * <p> + * @param size Size of buffer. + * @return Address of buffer. + */ + public static native long allocateLocal(long size); + + /** + * Allocate memory interleaved on NUMA nodes. Uses + * {@code void *numa_alloc_interleaved_subset(size_t, struct bitmask*)} under the hood. + * <p> + * @param size Size of buffer. + * @param nodes Array of nodes allocate on. If {@code null} or empty, allocate on all NODES. + * @return Address of buffer. + */ + public static native long allocateInterleaved(long size, int[] nodes); + + /** + * Get allocated buffer size. + * + * @param addr Address of buffer. + * @return Size of buffer. + */ + public static native long chunkSize(long addr); + + /** + * Free allocated memory. + * + * @param addr Address of buffer. + */ + public static native void free(long addr); + + /** + * Get NUMA nodes count. + * + * @return NUMA nodes count available on system. + */ + private static native int nodesCount(); +} diff --git a/modules/numa-allocator/src/main/java/org/apache/ignite/mem/InterleavedNumaAllocationStrategy.java b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/InterleavedNumaAllocationStrategy.java new file mode 100644 index 0000000..573078c --- /dev/null +++ b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/InterleavedNumaAllocationStrategy.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.mem; + +import java.io.Serializable; +import java.util.Arrays; +import org.apache.ignite.internal.mem.NumaAllocUtil; +import org.apache.ignite.internal.util.tostring.GridToStringBuilder; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.internal.A; + +/** + * Interleaved NUMA allocation strategy. + * <p> + * Use {@link InterleavedNumaAllocationStrategy#InterleavedNumaAllocationStrategy()} to allocate memory interleaved + * on all available NUMA nodes. Memory will be allocated using {@code void *numa_alloc_interleaved(size_t)} + * of {@code libnuma}. + * <p> + * Use {@link InterleavedNumaAllocationStrategy#InterleavedNumaAllocationStrategy(int[])} to allocate memory interleaved + * on specified nodes. + * {@code void *numa_alloc_interleaved_subset(size_t, struct bitmask*)} of {@code lubnuma}. + */ +public class InterleavedNumaAllocationStrategy implements NumaAllocationStrategy, Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + @GridToStringInclude + private final int[] nodes; + + /** */ + public InterleavedNumaAllocationStrategy() { + this(null); + } + + /** + * @param nodes Array of NUMA nodes to allocate on. + */ + public InterleavedNumaAllocationStrategy(int[] nodes) { + if (nodes != null && nodes.length > 0) { + this.nodes = Arrays.copyOf(nodes, nodes.length); + + Arrays.sort(this.nodes); + A.ensure(this.nodes[0] >= 0, "NUMA node number must be positive, passed instead " + + Arrays.toString(this.nodes)); + A.ensure(this.nodes[this.nodes.length - 1] < NumaAllocUtil.NUMA_NODES_CNT, + "NUMA node number must be less than NUMA_NODES_CNT=" + NumaAllocUtil.NUMA_NODES_CNT + + ", passed instead " + Arrays.toString(this.nodes)); + } + else + this.nodes = null; + } + + /** {@inheritDoc}*/ + @Override public long allocateMemory(long size) { + return NumaAllocUtil.allocateInterleaved(size, nodes); + } + + /** {@inheritDoc}*/ + @Override public String toString() { + return GridToStringBuilder.toString(InterleavedNumaAllocationStrategy.class, this); + } +} diff --git a/modules/numa-allocator/src/main/java/org/apache/ignite/mem/LocalNumaAllocationStrategy.java b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/LocalNumaAllocationStrategy.java new file mode 100644 index 0000000..e4e3387 --- /dev/null +++ b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/LocalNumaAllocationStrategy.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.mem; + +import java.io.Serializable; +import org.apache.ignite.internal.mem.NumaAllocUtil; +import org.apache.ignite.internal.util.tostring.GridToStringBuilder; + +/** + * Local NUMA allocation strategy. + * <p> + * Memory will be allocated using {@code void* numa_alloc_onnode(size_t)} of {@code lubnuma}. + */ +public class LocalNumaAllocationStrategy implements NumaAllocationStrategy, Serializable { + /** + * + */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override public long allocateMemory(long size) { + return NumaAllocUtil.allocateLocal(size); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return GridToStringBuilder.toString(LocalNumaAllocationStrategy.class, this); + } +} diff --git a/modules/numa-allocator/src/main/java/org/apache/ignite/mem/NumaAllocationStrategy.java b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/NumaAllocationStrategy.java new file mode 100644 index 0000000..781bf71 --- /dev/null +++ b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/NumaAllocationStrategy.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.mem; + +/** + * Basic interface for allocating memory on NUMA nodes with different policies using {@code libnuma}. + */ +public interface NumaAllocationStrategy { + /** + * @param size Size of allocated memory. + * + * @return Pointer to memory. + */ + public long allocateMemory(long size); +} diff --git a/modules/numa-allocator/src/main/java/org/apache/ignite/mem/NumaAllocator.java b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/NumaAllocator.java new file mode 100644 index 0000000..8b90862 --- /dev/null +++ b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/NumaAllocator.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.mem; + +import java.io.Serializable; +import org.apache.ignite.internal.mem.NumaAllocUtil; + +/** + * NUMA aware memory allocator. Uses {@code libnuma} under the hood. Only Linux distros with {@code libnuma >= 2.0.x} + * are supported. + * <p> + * Allocation strategy can be defined by setting {@code allocStrategy} to + * {@link NumaAllocator#NumaAllocator(NumaAllocationStrategy)}. + */ +public class NumaAllocator implements MemoryAllocator, Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private final NumaAllocationStrategy allocStrategy; + + /** + * @param allocStrategy Allocation strategy. + */ + public NumaAllocator(NumaAllocationStrategy allocStrategy) { + this.allocStrategy = allocStrategy; + } + + /** {@inheritDoc}*/ + @Override public long allocateMemory(long size) { + return allocStrategy.allocateMemory(size); + } + + /** {@inheritDoc}*/ + @Override public void freeMemory(long addr) { + NumaAllocUtil.free(addr); + } +} diff --git a/modules/numa-allocator/src/main/java/org/apache/ignite/mem/SimpleNumaAllocationStrategy.java b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/SimpleNumaAllocationStrategy.java new file mode 100644 index 0000000..d74fb7a --- /dev/null +++ b/modules/numa-allocator/src/main/java/org/apache/ignite/mem/SimpleNumaAllocationStrategy.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.mem; + +import java.io.Serializable; +import org.apache.ignite.internal.mem.NumaAllocUtil; +import org.apache.ignite.internal.util.tostring.GridToStringBuilder; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.internal.A; + +/** + * Simple NUMA allocation strategy. + * <p> + * Use {@link SimpleNumaAllocationStrategy#SimpleNumaAllocationStrategy(int)} to allocate memory on specific NUMA node + * with number equals to {@code node}. Memory will be allocated using {@code void *numa_alloc_onnode(size_t, int)} + * of {@code libnuma}. + * <p> + * Use {@link SimpleNumaAllocationStrategy#SimpleNumaAllocationStrategy()} to allocate memory using default NUMA + * memory policy of current thread. Memory will be allocated using {@code void *numa_alloc(size_t)} of {@code lubnuma}. + * Memory policy could be set by running application with {@code numactl} + */ +public class SimpleNumaAllocationStrategy implements NumaAllocationStrategy, Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private static final int NODE_NOT_SET = Integer.MAX_VALUE; + + /** */ + @GridToStringInclude + private final int node; + + /** */ + public SimpleNumaAllocationStrategy() { + node = NODE_NOT_SET; + } + + /** + * @param node Numa node to allocate on. + */ + public SimpleNumaAllocationStrategy(int node) { + A.ensure(node >= 0, "NUMA node number must be positive, passed instead " + node); + A.ensure(node < NumaAllocUtil.NUMA_NODES_CNT, + "NUMA node number must be less than NUMA_NODES_CNT=" + NumaAllocUtil.NUMA_NODES_CNT + + ", passed instead " + node); + + this.node = node; + } + + /** {@inheritDoc}*/ + @Override public long allocateMemory(long size) { + if (node == NODE_NOT_SET) + return NumaAllocUtil.allocate(size); + + return NumaAllocUtil.allocateOnNode(size, node); + } + + /** {@inheritDoc}*/ + @Override public String toString() { + return GridToStringBuilder.toString(SimpleNumaAllocationStrategy.class, this); + } +} diff --git a/modules/numa-allocator/src/test/java/org/apache/ignite/internal/mem/NumaAllocatorBasicTest.java b/modules/numa-allocator/src/test/java/org/apache/ignite/internal/mem/NumaAllocatorBasicTest.java new file mode 100644 index 0000000..db959d9 --- /dev/null +++ b/modules/numa-allocator/src/test/java/org/apache/ignite/internal/mem/NumaAllocatorBasicTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.mem; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.DataRegion; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.mem.InterleavedNumaAllocationStrategy; +import org.apache.ignite.mem.LocalNumaAllocationStrategy; +import org.apache.ignite.mem.NumaAllocationStrategy; +import org.apache.ignite.mem.NumaAllocator; +import org.apache.ignite.mem.SimpleNumaAllocationStrategy; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** */ +@RunWith(Parameterized.class) +public class NumaAllocatorBasicTest extends GridCommonAbstractTest { + /** */ + private static final long INITIAL_SIZE = 30L * 1024 * 1024; + + /** */ + private static final long MAX_SIZE = 100L * 1024 * 1024; + + /** */ + private static final String TEST_CACHE = "test"; + + /** */ + private static final byte[] BUF = new byte[4096]; + + /** */ + private static final int NUM_NODES = 3; + + static { + ThreadLocalRandom.current().nextBytes(BUF); + } + + /** */ + @Parameterized.Parameters(name = "allocationStrategy={0}, defaultConfig={1}") + public static Iterable<Object[]> data() { + return Stream.of( + new LocalNumaAllocationStrategy(), + new InterleavedNumaAllocationStrategy(), + new InterleavedNumaAllocationStrategy(IntStream.range(0, NumaAllocUtil.NUMA_NODES_CNT).toArray()), + new SimpleNumaAllocationStrategy(), + new SimpleNumaAllocationStrategy(NumaAllocUtil.NUMA_NODES_CNT - 1) + ) + .flatMap(strategy -> Stream.of(new Object[]{strategy, true}, new Object[]{strategy, false})) + .collect(Collectors.toList()); + } + + /** */ + @Parameterized.Parameter(0) + public NumaAllocationStrategy strategy; + + /** */ + @Parameterized.Parameter(1) + public boolean defaultConfig; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + DataStorageConfiguration memCfg = new DataStorageConfiguration(); + + DataRegionConfiguration dfltReg = new DataRegionConfiguration() + .setInitialSize(INITIAL_SIZE) + .setMaxSize(MAX_SIZE) + .setMetricsEnabled(true); + + NumaAllocator memAlloc = new NumaAllocator(strategy); + + if (defaultConfig) + memCfg.setMemoryAllocator(memAlloc); + else + dfltReg.setMemoryAllocator(memAlloc); + + memCfg.setDefaultDataRegionConfiguration(dfltReg); + + cfg.setDataStorageConfiguration(memCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + startGrids(NUM_NODES); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(true); + } + + /** */ + @Test + public void testLoadData() throws Exception { + IgniteEx client = startClientGrid("client"); + + client.getOrCreateCache(TEST_CACHE); + + try (IgniteDataStreamer<Integer, byte[]> ds = client.dataStreamer(TEST_CACHE)) { + int cnt = 0; + while (hasFreeSpace()) { + ds.addData(++cnt, BUF); + + if (cnt % 100 == 0) + ds.flush(); + } + } + + assertEquals(NUM_NODES, serverGrids().count()); + + serverGrids().forEach(g -> { + assertTrue(getDefaultRegion(g).config().getMemoryAllocator() instanceof NumaAllocator); + }); + } + + /** */ + private boolean hasFreeSpace() { + return serverGrids().allMatch(g -> { + DataRegion dr = getDefaultRegion(g); + + return dr.metrics().getTotalAllocatedSize() < 0.9 * MAX_SIZE; + }); + } + + /** */ + private static Stream<IgniteEx> serverGrids() { + return G.allGrids().stream().filter(g -> !g.cluster().localNode().isClient()).map(g -> (IgniteEx)g); + } + + /** */ + private static DataRegion getDefaultRegion(IgniteEx g) { + assertFalse(g.cluster().localNode().isClient()); + + String dataRegionName = g.configuration().getDataStorageConfiguration() + .getDefaultDataRegionConfiguration().getName(); + + try { + return g.context().cache().context().database().dataRegion(dataRegionName); + } + catch (IgniteCheckedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/modules/numa-allocator/src/test/java/org/apache/ignite/internal/mem/NumaAllocatorUnitTest.java b/modules/numa-allocator/src/test/java/org/apache/ignite/internal/mem/NumaAllocatorUnitTest.java new file mode 100644 index 0000000..7044ea8 --- /dev/null +++ b/modules/numa-allocator/src/test/java/org/apache/ignite/internal/mem/NumaAllocatorUnitTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.mem; + +import java.util.Arrays; +import java.util.stream.IntStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.mem.InterleavedNumaAllocationStrategy; +import org.apache.ignite.mem.LocalNumaAllocationStrategy; +import org.apache.ignite.mem.NumaAllocationStrategy; +import org.apache.ignite.mem.NumaAllocator; +import org.apache.ignite.mem.SimpleNumaAllocationStrategy; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** */ +@RunWith(Enclosed.class) +public class NumaAllocatorUnitTest { + /** */ + @RunWith(Parameterized.class) + public static class PositiveScenarioTest extends GridCommonAbstractTest { + /** */ + private static final long BUF_SZ = 32 * 1024 * 1024; + + /** */ + private static final int[] EVEN_NODES = IntStream.range(0, NumaAllocUtil.NUMA_NODES_CNT) + .filter(x -> x % 2 == 0).toArray(); + + /** */ + private static final int[] ALL_NODES = IntStream.range(0, NumaAllocUtil.NUMA_NODES_CNT).toArray(); + + /** + * + */ + @Parameterized.Parameters(name = "allocationStrategy={0}") + public static Iterable<Object[]> data() { + return Arrays.asList( + new Object[] {new LocalNumaAllocationStrategy()}, + new Object[] {new InterleavedNumaAllocationStrategy()}, + new Object[] {new InterleavedNumaAllocationStrategy(new int[0])}, + new Object[] {new InterleavedNumaAllocationStrategy(EVEN_NODES)}, + new Object[] {new InterleavedNumaAllocationStrategy(ALL_NODES)}, + new Object[] {new SimpleNumaAllocationStrategy()}, + new Object[] {new SimpleNumaAllocationStrategy(NumaAllocUtil.NUMA_NODES_CNT - 1)} + ); + } + + /** */ + @Parameterized.Parameter() + public NumaAllocationStrategy strategy; + + /** */ + @Test + public void test() { + NumaAllocator allocator = new NumaAllocator(strategy); + + long ptr = 0; + try { + ptr = allocator.allocateMemory(BUF_SZ); + + assertEquals(BUF_SZ, NumaAllocUtil.chunkSize(ptr)); + + GridUnsafe.setMemory(ptr, BUF_SZ, (byte)1); + + for (long i = 0; i < BUF_SZ; i++) + assertEquals((byte)1, GridUnsafe.getByte(ptr + i)); + } + finally { + if (ptr != 0) + allocator.freeMemory(ptr); + } + } + } + + /** */ + public static class ErrorScenarioTest extends GridCommonAbstractTest { + /** */ + @Test + public void testInvalidInterleavedStrategyParams() { + int[][] invalidNodes = { + {-3, -4, 0}, + IntStream.range(0, NumaAllocUtil.NUMA_NODES_CNT + 1).toArray() + }; + + for (int[] nodeSet: invalidNodes) { + GridTestUtils.assertThrows(log(), () -> new InterleavedNumaAllocationStrategy(nodeSet), + IllegalArgumentException.class, null); + } + } + + /** */ + @Test + public void testInvalidSimpleStrategyParams() { + int[] invalidNodes = {-3, NumaAllocUtil.NUMA_NODES_CNT}; + + for (int node: invalidNodes) { + GridTestUtils.assertThrows(log(), () -> new SimpleNumaAllocationStrategy(node), + IllegalArgumentException.class, null); + } + } + } +} diff --git a/modules/numa-allocator/src/test/java/org/apache/ignite/testsuites/NumaAllocatorTestSuite.java b/modules/numa-allocator/src/test/java/org/apache/ignite/testsuites/NumaAllocatorTestSuite.java new file mode 100644 index 0000000..9fc6ab6 --- /dev/null +++ b/modules/numa-allocator/src/test/java/org/apache/ignite/testsuites/NumaAllocatorTestSuite.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import org.apache.ignite.internal.mem.NumaAllocatorBasicTest; +import org.apache.ignite.internal.mem.NumaAllocatorUnitTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + NumaAllocatorUnitTest.class, + NumaAllocatorBasicTest.class +}) +public class NumaAllocatorTestSuite { +} diff --git a/parent/pom.xml b/parent/pom.xml index a57e612..38e1dfb 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -366,6 +366,10 @@ <packages>org.apache.ignite.mbean:org.apache.ignite.mxbean</packages> </group> <group> + <title>Memory allocator APIs</title> + <packages>org.apache.ignite.mem</packages> + </group> + <group> <title>SPI: CheckPoint</title> <packages>org.apache.ignite.spi.checkpoint*</packages> </group> diff --git a/pom.xml b/pom.xml index f93a4c3..715cf1c 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,7 @@ <module>modules/hibernate-4.2</module> <module>modules/hibernate-5.1</module> <module>modules/hibernate-5.3</module> + <module>modules/numa-allocator</module> <module>modules/schedule</module> <module>modules/yardstick</module> </modules> @@ -131,6 +132,13 @@ </profile> <profile> + <id>numa-allocator</id> + <modules> + <module>modules/numa-allocator</module> + </modules> + </profile> + + <profile> <id>test</id> <build> <plugins>