Setup hugepages on nodes. This is useful not only on SUT nodes, but
also on TG nodes which use TGs that utilize hugepages.
Signed-off-by: Juraj Linkeš <juraj.lin...@pantheon.tech>
---
dts/framework/remote_session/__init__.py | 1 +
dts/framework/remote_session/arch/__init__.py | 20 +++++
dts/framework/remote_session/arch/arch.py | 57 +++++++++++++
.../remote_session/os/linux_session.py | 85 +++++++++++++++++++
dts/framework/remote_session/os/os_session.py | 10 +++
dts/framework/testbed_model/node/node.py | 15 +++-
6 files changed, 187 insertions(+), 1 deletion(-)
create mode 100644 dts/framework/remote_session/arch/__init__.py
create mode 100644 dts/framework/remote_session/arch/arch.py
diff --git a/dts/framework/remote_session/__init__.py
b/dts/framework/remote_session/__init__.py
index f2339b20bd..f0deeadac6 100644
--- a/dts/framework/remote_session/__init__.py
+++ b/dts/framework/remote_session/__init__.py
@@ -11,4 +11,5 @@
# pylama:ignore=W0611
+from .arch import Arch, create_arch
from .os import OSSession, create_session
diff --git a/dts/framework/remote_session/arch/__init__.py
b/dts/framework/remote_session/arch/__init__.py
new file mode 100644
index 0000000000..d78ad42ac5
--- /dev/null
+++ b/dts/framework/remote_session/arch/__init__.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+from framework.config import Architecture, NodeConfiguration
+
+from .arch import PPC64, Arch, Arm64, i686, x86_32, x86_64
+
+
+def create_arch(node_config: NodeConfiguration) -> Arch:
+ match node_config.arch:
+ case Architecture.x86_64:
+ return x86_64()
+ case Architecture.x86_32:
+ return x86_32()
+ case Architecture.i686:
+ return i686()
+ case Architecture.ppc64le:
+ return PPC64()
+ case Architecture.arm64:
+ return Arm64()
diff --git a/dts/framework/remote_session/arch/arch.py
b/dts/framework/remote_session/arch/arch.py
new file mode 100644
index 0000000000..05c7602def
--- /dev/null
+++ b/dts/framework/remote_session/arch/arch.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+
+class Arch(object):
+ """
+ Stores architecture-specific information.
+ """
+
+ @property
+ def default_hugepage_memory(self) -> int:
+ """
+ Return the default amount of memory allocated for hugepages DPDK will
use.
+ The default is an amount equal to 256 2MB hugepages (512MB memory).
+ """
+ return 256 * 2048
+
+ @property
+ def hugepage_force_first_numa(self) -> bool:
+ """
+ An architecture may need to force configuration of hugepages to first
socket.
+ """
+ return False
+
+
+class x86_64(Arch):
+ @property
+ def default_hugepage_memory(self) -> int:
+ return 4096 * 2048
+
+
+class x86_32(Arch):
+ @property
+ def hugepage_force_first_numa(self) -> bool:
+ return True
+
+
+class i686(Arch):
+ @property
+ def default_hugepage_memory(self) -> int:
+ return 512 * 2048
+
+ @property
+ def hugepage_force_first_numa(self) -> bool:
+ return True
+
+
+class PPC64(Arch):
+ @property
+ def default_hugepage_memory(self) -> int:
+ return 512 * 2048
+
+
+class Arm64(Arch):
+ @property
+ def default_hugepage_memory(self) -> int:
+ return 2048 * 2048
diff --git a/dts/framework/remote_session/os/linux_session.py
b/dts/framework/remote_session/os/linux_session.py
index 21f117b714..fad33d7613 100644
--- a/dts/framework/remote_session/os/linux_session.py
+++ b/dts/framework/remote_session/os/linux_session.py
@@ -3,6 +3,8 @@
# Copyright(c) 2022 University of New Hampshire
from framework.config import CPU
+from framework.exception import RemoteCommandExecutionError
+from framework.utils import expand_range
from .posix_session import PosixSession
@@ -24,3 +26,86 @@ def get_remote_cpus(self, bypass_core0: bool) -> list[CPU]:
continue
cpus.append(CPU(int(cpu), int(core), int(socket), int(node)))
return cpus
+
+ def setup_hugepages(
+ self, hugepage_amount: int = -1, force_first_numa: bool = False
+ ) -> None:
+ self.logger.info("Getting Hugepage information.")
+ hugepage_size = self._get_hugepage_size()
+ hugepages_total = self._get_hugepages_total()
+ self._numa_nodes = self._get_numa_nodes()
+
+ target_hugepages_total = int(hugepage_amount / hugepage_size)
+ if hugepage_amount % hugepage_size:
+ target_hugepages_total += 1
+ if force_first_numa or hugepages_total != target_hugepages_total:
+ # when forcing numa, we need to clear existing hugepages regardless
+ # of size, so they can be moved to the first numa node
+ self._configure_huge_pages(
+ target_hugepages_total, hugepage_size, force_first_numa
+ )
+ else:
+ self.logger.info("Hugepages already configured.")
+ self._mount_huge_pages()
+
+ def _get_hugepage_size(self) -> int:
+ hugepage_size = self.remote_session.send_command(
+ "awk '/Hugepagesize/ {print $2}' /proc/meminfo"
+ ).stdout
+ return int(hugepage_size)
+
+ def _get_hugepages_total(self) -> int:
+ hugepages_total = self.remote_session.send_command(
+ "awk '/HugePages_Total/ { print $2 }' /proc/meminfo"
+ ).stdout
+ return int(hugepages_total)
+
+ def _get_numa_nodes(self) -> list[int]:
+ try:
+ numa_range = self.remote_session.send_command(
+ "cat /sys/devices/system/node/online", verify=True
+ ).stdout
+ numa_range = expand_range(numa_range)
+ except RemoteCommandExecutionError:
+ # the file doesn't exist, meaning the node doesn't support numa
+ numa_range = []
+ return numa_range
+
+ def _mount_huge_pages(self) -> None:
+ self.logger.info("Re-mounting Hugepages.")
+ hugapge_fs_cmd = "awk '/hugetlbfs/ { print $2 }' /proc/mounts"
+ self.remote_session.send_command(f"umount $({hugapge_fs_cmd})")
+ result = self.remote_session.send_command(hugapge_fs_cmd)
+ if result.stdout == "":
+ remote_mount_path = "/mnt/huge"
+ self.remote_session.send_command(f"mkdir -p {remote_mount_path}")
+ self.remote_session.send_command(
+ f"mount -t hugetlbfs nodev {remote_mount_path}"
+ )
+
+ def _supports_numa(self) -> bool:
+ # the system supports numa if self._numa_nodes is non-empty and there
are more
+ # than one numa node (in the latter case it may actually support numa,
but
+ # there's no reason to do any numa specific configuration)
+ return len(self._numa_nodes) > 1
+
+ def _configure_huge_pages(
+ self, amount: int, size: int, force_first_numa: bool
+ ) -> None:
+ self.logger.info("Configuring Hugepages.")
+ hugepage_config_path = (
+ f"/sys/kernel/mm/hugepages/hugepages-{size}kB/nr_hugepages"
+ )
+ if force_first_numa and self._supports_numa():
+ # clear non-numa hugepages
+ self.remote_session.send_command(
+ f"echo 0 | sudo tee {hugepage_config_path}"
+ )
+ hugepage_config_path = (
+ f"/sys/devices/system/node/node{self._numa_nodes[0]}/hugepages"
+ f"/hugepages-{size}kB/nr_hugepages"
+ )
+
+ self.remote_session.send_command(
+ f"echo {amount} | sudo tee {hugepage_config_path}"
+ )
diff --git a/dts/framework/remote_session/os/os_session.py
b/dts/framework/remote_session/os/os_session.py
index 6f6b6a979e..f84f3ce63c 100644
--- a/dts/framework/remote_session/os/os_session.py
+++ b/dts/framework/remote_session/os/os_session.py
@@ -144,3 +144,13 @@ def kill_cleanup_dpdk_apps(self, dpdk_prefix_list:
Iterable[str]) -> None:
Kill and cleanup all DPDK apps identified by dpdk_prefix_list. If
dpdk_prefix_list is empty, attempt to find running DPDK apps to kill
and clean.
"""
+
+ @abstractmethod
+ def setup_hugepages(
+ self, hugepage_amount: int = -1, force_first_numa: bool = False
+ ) -> None:
+ """
+ Get the node's Hugepage Size, configure the specified amount of
hugepages
+ if needed and mount the hugepages if needed.
+ If force_first_numa is True, configure hugepages just on the first
socket.
+ """
diff --git a/dts/framework/testbed_model/node/node.py
b/dts/framework/testbed_model/node/node.py
index 5ee7023335..96a1724f4c 100644
--- a/dts/framework/testbed_model/node/node.py
+++ b/dts/framework/testbed_model/node/node.py
@@ -16,7 +16,7 @@
)
from framework.exception import NodeCleanupError, NodeSetupError,
convert_exception
from framework.logger import DTSLOG, getLogger
-from framework.remote_session import OSSession, create_session
+from framework.remote_session import Arch, OSSession, create_arch,
create_session
from framework.testbed_model.hw import CPUAmount, cpu_filter
@@ -33,6 +33,7 @@ class Node(object):
config: NodeConfiguration
cpus: list[CPU]
_other_sessions: list[OSSession]
+ _arch: Arch
def __init__(self, node_config: NodeConfiguration):
self.config = node_config
@@ -42,6 +43,7 @@ def __init__(self, node_config: NodeConfiguration):
self.logger = getLogger(self.name)
self.logger.info(f"Created node: {self.name}")
self.main_session = create_session(self.config, self.name, self.logger)
+ self._arch = create_arch(self.config)
self._get_remote_cpus()
@convert_exception(NodeSetupError)
@@ -50,6 +52,7 @@ def setup_execution(self, execution_config:
ExecutionConfiguration) -> None:
Perform the execution setup that will be done for each execution
this node is part of.
"""
+ self._setup_hugepages()
self._setup_execution(execution_config)
def _setup_execution(self, execution_config: ExecutionConfiguration) ->
None:
@@ -145,6 +148,16 @@ def _get_remote_cpus(self) -> None:
self.logger.info("Getting CPU information.")
self.cpus = self.main_session.get_remote_cpus(self.config.bypass_core0)
+ def _setup_hugepages(self):
+ """
+ Setup hugepages on the Node. Different architectures can supply
different
+ amounts of memory for hugepages and numa-based hugepage allocation may
need
+ to be considered.
+ """
+ self.main_session.setup_hugepages(
+ self._arch.default_hugepage_memory,
self._arch.hugepage_force_first_numa
+ )
+
def close(self) -> None:
"""
Close all connections and free other resources.
--
2.30.2
