[PATCH v4 8/8] scripts/gen_compile_commands: fix usage message
Replace mentions to 'kernel' by 'U-Boot' to avoid confusion.
Signed-off-by: Joao Marcos Costa
---
scripts/gen_compile_commands.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
index aa52e88e18..cdca85e6b0 100755
--- a/scripts/gen_compile_commands.py
+++ b/scripts/gen_compile_commands.py
@@ -37,10 +37,10 @@ def parse_arguments():
output: Where to write the compile-commands JSON file.
paths: The list of files/directories to handle to find .cmd files.
"""
-usage = 'Creates a compile_commands.json database from kernel .cmd files'
+usage = 'Creates a compile_commands.json database from U-Boot .cmd files'
parser = argparse.ArgumentParser(description=usage)
-directory_help = ('specify the output directory used for the kernel build '
+directory_help = ('specify the output directory used for the U-Boot build '
'(defaults to the working directory)')
parser.add_argument('-d', '--directory', type=str, default='.',
help=directory_help)
--
2.41.0
[PATCH v4 7/8] doc: add ide_integration.rst to doc/develop
Add 'Integration with IDEs' chapter. For now, this chapter is mostly a reference to the documentation of gen_compile_commands, in doc/build, but it can be futurely used as a guide for other IDE-friendly features. Signed-off-by: Joao Marcos Costa --- doc/develop/ide_integration.rst | 12 doc/develop/index.rst | 1 + 2 files changed, 13 insertions(+) create mode 100644 doc/develop/ide_integration.rst diff --git a/doc/develop/ide_integration.rst b/doc/develop/ide_integration.rst new file mode 100644 index 00..455e09959c --- /dev/null +++ b/doc/develop/ide_integration.rst @@ -0,0 +1,12 @@ +Integration with IDEs += + +IDEs and text editors (e.g., VSCode, Emacs, Vim, Neovim) typically offer +plugins to enhance the development experience, such as Clangd LSP. These +plugins provide features like code navigation (i.e., jumping to definitions +and declarations), code completion, and code formatting. + +U-Boot provides a script (i.e., scripts/gen_compile_commands.py) that +generates a compilation database to be utilized by Clangd LSP for code +navigation. For detailed usage instructions, please refer to the script's +documentation: :doc:`../build/gen_compile_commands`. diff --git a/doc/develop/index.rst b/doc/develop/index.rst index 5b230d0321..272bdef84d 100644 --- a/doc/develop/index.rst +++ b/doc/develop/index.rst @@ -19,6 +19,7 @@ General security sending_patches system_configuration + ide_integration Implementation -- -- 2.41.0
[PATCH v4 6/8] doc: add documentation for gen_compile_commands.py
This documentation briefly explains what is a compilation database,
and how to use the script to generate one.
This is not a portage, as there was no original documentation in the
Linux sources.
Acknowledge the documentation in the script's header and in doc/build
index.
Signed-off-by: Joao Marcos Costa
---
doc/build/gen_compile_commands.rst | 83 ++
doc/build/index.rst| 1 +
scripts/gen_compile_commands.py| 1 +
3 files changed, 85 insertions(+)
create mode 100644 doc/build/gen_compile_commands.rst
diff --git a/doc/build/gen_compile_commands.rst
b/doc/build/gen_compile_commands.rst
new file mode 100644
index 00..50305cec4a
--- /dev/null
+++ b/doc/build/gen_compile_commands.rst
@@ -0,0 +1,83 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+Create build database for IDEs
+==
+
+gen_compile_commands (scripts/gen_compile_commands.py) is a script used to
+generate a compilation database (compile_commands.json). This database consists
+of an array of "command objects" describing how each translation unit was
+compiled.
+
+Example::
+
+ {
+ "command": "gcc -Wp,-MD,arch/x86/cpu/.lapic.o.d -nostdinc -isystem (...)"
+ "directory": "/home/jmcosta/u-boot",
+ "file": "/home/jmcosta/u-boot/arch/x86/cpu/lapic.c"
+ }
+
+Such information comes from parsing the respective .cmd file of each
translation
+unit. In the previous example, that would be `arch/x86/cpu/.lapic.o.cmd`.
+
+For more details on the database format, please refer to the official
+documentation at https://clang.llvm.org/docs/JSONCompilationDatabase.html.
+
+The compilation database is quite useful for text editors (and IDEs) that use
+Clangd LSP. It allows jumping to definitions and declarations. Since it relies
+on parsing .cmd files, one needs to have a target (e.g. configs/xxx_defconfig)
+built before running the script.
+
+Example::
+
+ make sandbox_defconfig
+ make
+ ./scripts/gen_compile_commands.py
+
+Beware that depending on the changes you made to the project's source code, you
+may need to run the script again (presuming you recompiled your target, of
+course) to have an up-to-date database.
+
+The database will be in the root of the repository. No further modifications
are
+needed for it to be usable by the LSP, unless you set a name for the database
+other than it's default one (compile_commands.json).
+
+Compatible IDEs
+===
+
+Several popular integrated development environments (IDEs) support the use
+of JSON compilation databases for C/C++ development, making it easier to
+manage build configurations and code analysis. Some of these IDEs include:
+
+1. **Visual Studio Code (VS Code)**: IntelliSense in VS Code can be set up to
+ use compile_commands.json by following the instructions in
+
https://code.visualstudio.com/docs/cpp/faq-cpp#_how-do-i-get-intellisense-to-work-correctly.
+
+2. **CLion**: JetBrains' CLion IDE supports JSON compilation databases out
+ of the box. You can configure your project to use a compile_commands.json
+ file via the project settings. Details on setting up CLion with JSON
+ compilation databases can be found at
+ https://www.jetbrains.com/help/clion/compilation-database.html.
+
+3. **Qt Creator**: Qt Creator, a popular IDE for Qt development, also
+ supports compile_commands.json for C/C++ projects. Instructions on how to
+ use this feature can be found at
+
https://doc.qt.io/qtcreator/creator-clang-codemodel.html#using-compilation-databases.
+
+4. **Eclipse CDT**: Eclipse's C/C++ Development Tools (CDT) can be
+ configured to use JSON compilation databases for better project management.
+ You can find guidance on setting up JSON compilation database support at the
+ wiki: https://wiki.eclipse.org/CDT/User/NewIn910#Build.
+
+For Vim, Neovim, and Emacs, if you are using Clangd as your LSP, placing the
+compile_commands.json in the root of the repository should suffice to enable
+code navigation.
+
+Usage
+=
+
+For further details on the script's options, please refer to its help message,
+as in the example below.
+
+Help::
+
+ ./scripts/gen_compile_commands.py --help
diff --git a/doc/build/index.rst b/doc/build/index.rst
index 64e66491bd..7a4507b574 100644
--- a/doc/build/index.rst
+++ b/doc/build/index.rst
@@ -14,3 +14,4 @@ Build U-Boot
tools
buildman
documentation
+ gen_compile_commands
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
index 1a9c49b34a..aa52e88e18 100755
--- a/scripts/gen_compile_commands.py
+++ b/scripts/gen_compile_commands.py
@@ -5,6 +5,7 @@
#
# Author: Tom Roeder
# Ported and modified for U-Boot by Joao Marcos Costa
+# Briefly documented at doc/build/gen_compile_commands.rst
#
"""A tool for generating compile_commands.json in U-Boot."""
--
2.41.0
[PATCH v4 5/8] .gitignore: add compile_commands.json
Add Clang's compilation database file (i.e. compile_commands.json) to .gitignore, at the root of the repository. Signed-off-by: Joao Marcos Costa --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 002f95de4f..261a1d6754 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ __pycache__ # moveconfig database /moveconfig.db + +# Clang's compilation database file +/compile_commands.json -- 2.41.0
[PATCH v4 4/8] scripts/gen_compile_commands.py: add acknowledgments
Add acknowledgments for porting and modifying the script. Of course, the license, author, and copyright notice remain the same as in the original script. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 63d036a773..1a9c49b34a 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -4,6 +4,7 @@ # Copyright (C) Google LLC, 2018 # # Author: Tom Roeder +# Ported and modified for U-Boot by Joao Marcos Costa # """A tool for generating compile_commands.json in U-Boot.""" -- 2.41.0
[PATCH v4 3/8] scripts/gen_compile_commands.py: fix docstring
The referred tool is now in U-Boot. Replace "the Linux kernel" by "U-Boot" to make the docstring coherent. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 0227522959..63d036a773 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -5,7 +5,7 @@ # # Author: Tom Roeder # -"""A tool for generating compile_commands.json in the Linux kernel.""" +"""A tool for generating compile_commands.json in U-Boot.""" import argparse import json -- 2.41.0
[PATCH v4 2/8] scripts/gen_compile_commands.py: adapt _LINE_PATTERN
For U-Boot's context, the regular expression defined by _LINE_PATTERN should be adapted. Replace 'savedcmd' by 'cmd'. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 15ba56527a..0227522959 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. -- 2.41.0
[PATCH v4 1/8] scripts: Port Linux's gen_compile_commands.py to U-Boot
This script generates a database of compiler flags, namely
compile_commands.json. It is quite useful for text editors that use
clangd LSP (e.g. Vim, Neovim).
It was ported from Linux's sources:
- tag: v6.4
- revision 6995e2de6891c724bfeb2db33d7b87775f913ad1
Modifications for U-Boot compatibility will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
scripts/gen_compile_commands.py | 228
1 file changed, 228 insertions(+)
create mode 100755 scripts/gen_compile_commands.py
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
new file mode 100755
index 00..15ba56527a
--- /dev/null
+++ b/scripts/gen_compile_commands.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) Google LLC, 2018
+#
+# Author: Tom Roeder
+#
+"""A tool for generating compile_commands.json in the Linux kernel."""
+
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+
+_DEFAULT_OUTPUT = 'compile_commands.json'
+_DEFAULT_LOG_LEVEL = 'WARNING'
+
+_FILENAME_PATTERN = r'^\..*\.cmd$'
+_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)'
+_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+# The tools/ directory adopts a different build system, and produces .cmd
+# files in a different format. Do not support it.
+_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
+
+def parse_arguments():
+"""Sets up and parses command-line arguments.
+
+Returns:
+log_level: A logging level to filter log output.
+directory: The work directory where the objects were built.
+ar: Command used for parsing .a archives.
+output: Where to write the compile-commands JSON file.
+paths: The list of files/directories to handle to find .cmd files.
+"""
+usage = 'Creates a compile_commands.json database from kernel .cmd files'
+parser = argparse.ArgumentParser(description=usage)
+
+directory_help = ('specify the output directory used for the kernel build '
+ '(defaults to the working directory)')
+parser.add_argument('-d', '--directory', type=str, default='.',
+help=directory_help)
+
+output_help = ('path to the output command database (defaults to ' +
+ _DEFAULT_OUTPUT + ')')
+parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
+help=output_help)
+
+log_level_help = ('the level of log messages to produce (defaults to ' +
+ _DEFAULT_LOG_LEVEL + ')')
+parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
+default=_DEFAULT_LOG_LEVEL, help=log_level_help)
+
+ar_help = 'command used for parsing .a archives'
+parser.add_argument('-a', '--ar', type=str, default='llvm-ar',
help=ar_help)
+
+paths_help = ('directories to search or files to parse '
+ '(files should be *.o, *.a, or modules.order). '
+ 'If nothing is specified, the current directory is searched')
+parser.add_argument('paths', type=str, nargs='*', help=paths_help)
+
+args = parser.parse_args()
+
+return (args.log_level,
+os.path.abspath(args.directory),
+args.output,
+args.ar,
+args.paths if len(args.paths) > 0 else [args.directory])
+
+
+def cmdfiles_in_dir(directory):
+"""Generate the iterator of .cmd files found under the directory.
+
+Walk under the given directory, and yield every .cmd file found.
+
+Args:
+directory: The directory to search for .cmd files.
+
+Yields:
+The path to a .cmd file.
+"""
+
+filename_matcher = re.compile(_FILENAME_PATTERN)
+exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
+
+for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+# Prune unwanted directories.
+if dirpath in exclude_dirs:
+dirnames[:] = []
+continue
+
+for filename in filenames:
+if filename_matcher.match(filename):
+yield os.path.join(dirpath, filename)
+
+
+def to_cmdfile(path):
+"""Return the path of .cmd file used for the given build artifact
+
+Args:
+Path: file path
+
+Returns:
+The path to .cmd file
+"""
+dir, base = os.path.split(path)
+return os.path.join(dir, '.' + base + '.cmd')
+
+
+def cmdfiles_for_a(archive, ar):
+"""Generate the iterator of .cmd files associated with the archive.
+
+Parse the given archive, and yield every .cmd file used to build it.
+
+Args:
+archive: The archive to parse
+
+Yields:
+The path to every .cmd file found
+"""
+for obj in
[PATCH v4 0/8] Port gen_compile_commands.py from Linux to U-Boot
Hello U-Boot community, I'm submitting a patch series that ports the gen_compile_commands.py script from the Linux kernel's sources to U-Boot. This script, originally located in scripts/clang-tools/gen_compile_commands.py, enables the generation of compile_commands.json file for improved code navigation and analysis. The series consists of the initial script import, the necessary modifications for U-Boot compatibility, and finally some documentation. Your feedback on these contributions would be greatly appreciated. Best regards, Changes in v4: - Replace "Options" section in the doc by "Usage", as it is simply a reference to the script's usage message - Replace 'kernel' by 'U-Boot' in the usage message - Replace label by :doc: link - Instead of adding a chapter about integration with IDEs into doc/build/tools.rst, add a new file (i.e., doc/develop/ide_integration.rst) - Replace the doc's heading (gen_compile_commands) by 'Create build database for IDEs' - Add a section listing some of the compatible IDEs and how to set them up Changes in v3: - Add documentation to index and fix syntax issues - Add reference to documentation in doc/build/tools Changes in v2: - Add compile_commands.json to gitignore - Add documentation Joao Marcos Costa (8): scripts: Port Linux's gen_compile_commands.py to U-Boot scripts/gen_compile_commands.py: adapt _LINE_PATTERN scripts/gen_compile_commands.py: fix docstring scripts/gen_compile_commands.py: add acknowledgments .gitignore: add compile_commands.json doc: add documentation for gen_compile_commands.py doc: add ide_integration.rst to doc/develop scripts/gen_compile_commands: fix usage message .gitignore | 3 + doc/build/gen_compile_commands.rst | 84 +++ doc/build/index.rst| 1 + doc/develop/ide_integration.rst| 13 ++ doc/develop/index.rst | 1 + scripts/gen_compile_commands.py| 230 + 6 files changed, 332 insertions(+) create mode 100644 doc/build/gen_compile_commands.rst create mode 100644 doc/develop/ide_integration.rst create mode 100755 scripts/gen_compile_commands.py -- 2.41.0
[PATCH v1] doc: Update path to source_file_format.rst
Previously, the file extension was .txt, and it referenced the uImage.FIT directory, which no longer exists. This commit updates the path accordingly. Signed-off-by: Joao Marcos Costa --- doc/usage/fit/howto.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage/fit/howto.rst b/doc/usage/fit/howto.rst index c933703d1d..8b38cd534d 100644 --- a/doc/usage/fit/howto.rst +++ b/doc/usage/fit/howto.rst @@ -22,7 +22,7 @@ for its latest version. mkimage (together with dtc) takes as input an image source file, which describes the contents of the image and defines its various properties used during booting. By convention, image source file has the ".its" extension, also, the details of its format are given in -doc/uImage.FIT/source_file_format.txt. The actual data that is to be included in +doc/usage/fit/source_file_format.rst. The actual data that is to be included in the uImage (kernel, ramdisk, etc.) is specified in the image source file in the form of paths to appropriate data files. The outcome of the image creation process is a binary file (by convention with the ".itb" extension) that -- 2.41.0
[PATCH v3 6/7] doc: add documentation for gen_compile_commands.py
This documentation briefly explains what is a compilation database,
and how to use the script to generate one.
This is not a portage, as there was no original documentation in the
Linux sources.
Acknowledge the documentation in the script's header and in doc/build
index.
Signed-off-by: Joao Marcos Costa
---
doc/build/gen_compile_commands.rst | 46 ++
doc/build/index.rst| 1 +
scripts/gen_compile_commands.py| 1 +
3 files changed, 48 insertions(+)
create mode 100644 doc/build/gen_compile_commands.rst
diff --git a/doc/build/gen_compile_commands.rst
b/doc/build/gen_compile_commands.rst
new file mode 100644
index 00..6b32eb678a
--- /dev/null
+++ b/doc/build/gen_compile_commands.rst
@@ -0,0 +1,46 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+
+gen_compile_commands
+
+
+gen_compile_commands (scripts/gen_compile_commands.py) is a script used to
+generate a compilation database (compile_commands.json). This database consists
+of an array of "command objects" describing how each translation unit was
+compiled.
+
+Example::
+
+ {
+ "command": "gcc -Wp,-MD,arch/x86/cpu/.lapic.o.d -nostdinc -isystem (...)"
+ "directory": "/home/jmcosta/u-boot",
+ "file": "/home/jmcosta/u-boot/arch/x86/cpu/lapic.c"
+ }
+
+Such information comes from parsing the respective .cmd file of each
translation
+unit. In the previous example, that would be `arch/x86/cpu/.lapic.o.cmd`.
+
+The compilation database is quite useful for text editors (and IDEs) that use
+Clangd LSP. It allows jumping to definitions and declarations. Since it relies
+on parsing .cmd files, one needs to have a target (e.g. configs/xxx_defconfig)
+built before running the script.
+
+Example::
+
+ make sandbox_defconfig
+ make
+ ./scripts/gen_compile_commands.py
+
+The database will be in the root of the repository. No further modifications
are
+needed for it to be usable by the LSP, unless you set a name for the database
+other than it's default one (compile_commands.json).
+
+Options
+===
+
+For further details on how to use the script and its options, please refer to
+its help message, as in the example below.
+
+Help::
+
+ ./scripts/gen_compile_commands.py --help
diff --git a/doc/build/index.rst b/doc/build/index.rst
index 64e66491bd..7a4507b574 100644
--- a/doc/build/index.rst
+++ b/doc/build/index.rst
@@ -14,3 +14,4 @@ Build U-Boot
tools
buildman
documentation
+ gen_compile_commands
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
index 1a9c49b34a..aa52e88e18 100755
--- a/scripts/gen_compile_commands.py
+++ b/scripts/gen_compile_commands.py
@@ -5,6 +5,7 @@
#
# Author: Tom Roeder
# Ported and modified for U-Boot by Joao Marcos Costa
+# Briefly documented at doc/build/gen_compile_commands.rst
#
"""A tool for generating compile_commands.json in U-Boot."""
--
2.41.0
[PATCH v3 7/7] doc: add new section to build/tools
Add 'Integration with IDEs' section. For now, this section is mostly a reference to the documentation of gen_compile_commands, also in doc/build, but it can be futurely used as a guide for other IDE-friendly features. Signed-off-by: Joao Marcos Costa --- doc/build/gen_compile_commands.rst | 1 + doc/build/tools.rst| 13 + 2 files changed, 14 insertions(+) diff --git a/doc/build/gen_compile_commands.rst b/doc/build/gen_compile_commands.rst index 6b32eb678a..da812525fe 100644 --- a/doc/build/gen_compile_commands.rst +++ b/doc/build/gen_compile_commands.rst @@ -1,4 +1,5 @@ .. SPDX-License-Identifier: GPL-2.0-only +.. _gen-compile-commands-label: gen_compile_commands diff --git a/doc/build/tools.rst b/doc/build/tools.rst index ec01722925..5cdbf27d83 100644 --- a/doc/build/tools.rst +++ b/doc/build/tools.rst @@ -45,3 +45,16 @@ Launch the MSYS2 shell of the MSYS2 environment, and do the following:: $ make tools-only_defconfig $ make tools-only + +Integration with IDEs +- + +IDEs and text editors (e.g., VSCode, Emacs, Vim, Neovim) typically offer +plugins to enhance the development experience, such as Clangd LSP. These +plugins provide features like code navigation (i.e., jumping to definitions +and declarations), code completion, and code formatting. + +U-Boot provides a script (i.e., scripts/gen_compile_commands.py) that +generates a compilation database to be utilized by Clangd LSP for code +navigation. For detailed usage instructions, please refer to the script's +documentation: :ref:`gen-compile-commands-label`. -- 2.41.0
[PATCH v3 4/7] scripts/gen_compile_commands.py: add acknowledgments
Add acknowledgments for porting and modifying the script. Of course, the license, author, and copyright notice remain the same as in the original script. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 63d036a773..1a9c49b34a 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -4,6 +4,7 @@ # Copyright (C) Google LLC, 2018 # # Author: Tom Roeder +# Ported and modified for U-Boot by Joao Marcos Costa # """A tool for generating compile_commands.json in U-Boot.""" -- 2.41.0
[PATCH v3 3/7] scripts/gen_compile_commands.py: fix docstring
The referred tool is now in U-Boot. Replace "the Linux kernel" by "U-Boot" to make the docstring coherent. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 0227522959..63d036a773 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -5,7 +5,7 @@ # # Author: Tom Roeder # -"""A tool for generating compile_commands.json in the Linux kernel.""" +"""A tool for generating compile_commands.json in U-Boot.""" import argparse import json -- 2.41.0
[PATCH v3 5/7] .gitignore: add compile_commands.json
Add Clang's compilation database file (i.e. compile_commands.json) to .gitignore, at the root of the repository. Signed-off-by: Joao Marcos Costa --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 002f95de4f..261a1d6754 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ __pycache__ # moveconfig database /moveconfig.db + +# Clang's compilation database file +/compile_commands.json -- 2.41.0
[PATCH v3 2/7] scripts/gen_compile_commands.py: adapt _LINE_PATTERN
For U-Boot's context, the regular expression defined by _LINE_PATTERN should be adapted. Replace 'savedcmd' by 'cmd'. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 15ba56527a..0227522959 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. -- 2.41.0
[PATCH v3 1/7] scripts: Port Linux's gen_compile_commands.py to U-Boot
This script generates a database of compiler flags, namely
compile_commands.json. It is quite useful for text editors that use
clangd LSP (e.g. Vim, Neovim).
It was ported from Linux's sources:
- tag: v6.4
- revision 6995e2de6891c724bfeb2db33d7b87775f913ad1
Modifications for U-Boot compatibility will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
scripts/gen_compile_commands.py | 228
1 file changed, 228 insertions(+)
create mode 100755 scripts/gen_compile_commands.py
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
new file mode 100755
index 00..15ba56527a
--- /dev/null
+++ b/scripts/gen_compile_commands.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) Google LLC, 2018
+#
+# Author: Tom Roeder
+#
+"""A tool for generating compile_commands.json in the Linux kernel."""
+
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+
+_DEFAULT_OUTPUT = 'compile_commands.json'
+_DEFAULT_LOG_LEVEL = 'WARNING'
+
+_FILENAME_PATTERN = r'^\..*\.cmd$'
+_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)'
+_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+# The tools/ directory adopts a different build system, and produces .cmd
+# files in a different format. Do not support it.
+_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
+
+def parse_arguments():
+"""Sets up and parses command-line arguments.
+
+Returns:
+log_level: A logging level to filter log output.
+directory: The work directory where the objects were built.
+ar: Command used for parsing .a archives.
+output: Where to write the compile-commands JSON file.
+paths: The list of files/directories to handle to find .cmd files.
+"""
+usage = 'Creates a compile_commands.json database from kernel .cmd files'
+parser = argparse.ArgumentParser(description=usage)
+
+directory_help = ('specify the output directory used for the kernel build '
+ '(defaults to the working directory)')
+parser.add_argument('-d', '--directory', type=str, default='.',
+help=directory_help)
+
+output_help = ('path to the output command database (defaults to ' +
+ _DEFAULT_OUTPUT + ')')
+parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
+help=output_help)
+
+log_level_help = ('the level of log messages to produce (defaults to ' +
+ _DEFAULT_LOG_LEVEL + ')')
+parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
+default=_DEFAULT_LOG_LEVEL, help=log_level_help)
+
+ar_help = 'command used for parsing .a archives'
+parser.add_argument('-a', '--ar', type=str, default='llvm-ar',
help=ar_help)
+
+paths_help = ('directories to search or files to parse '
+ '(files should be *.o, *.a, or modules.order). '
+ 'If nothing is specified, the current directory is searched')
+parser.add_argument('paths', type=str, nargs='*', help=paths_help)
+
+args = parser.parse_args()
+
+return (args.log_level,
+os.path.abspath(args.directory),
+args.output,
+args.ar,
+args.paths if len(args.paths) > 0 else [args.directory])
+
+
+def cmdfiles_in_dir(directory):
+"""Generate the iterator of .cmd files found under the directory.
+
+Walk under the given directory, and yield every .cmd file found.
+
+Args:
+directory: The directory to search for .cmd files.
+
+Yields:
+The path to a .cmd file.
+"""
+
+filename_matcher = re.compile(_FILENAME_PATTERN)
+exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
+
+for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+# Prune unwanted directories.
+if dirpath in exclude_dirs:
+dirnames[:] = []
+continue
+
+for filename in filenames:
+if filename_matcher.match(filename):
+yield os.path.join(dirpath, filename)
+
+
+def to_cmdfile(path):
+"""Return the path of .cmd file used for the given build artifact
+
+Args:
+Path: file path
+
+Returns:
+The path to .cmd file
+"""
+dir, base = os.path.split(path)
+return os.path.join(dir, '.' + base + '.cmd')
+
+
+def cmdfiles_for_a(archive, ar):
+"""Generate the iterator of .cmd files associated with the archive.
+
+Parse the given archive, and yield every .cmd file used to build it.
+
+Args:
+archive: The archive to parse
+
+Yields:
+The path to every .cmd file found
+"""
+for obj in
[PATCH v3 0/7] Port gen_compile_commands.py from Linux to U-Boot
Hello U-Boot community, I'm submitting a patch series that ports the gen_compile_commands.py script from the Linux kernel's sources to U-Boot. This script, originally located in scripts/clang-tools/gen_compile_commands.py, enables the generation of compile_commands.json file for improved code navigation and analysis. The series consists of the initial script import, the necessary modifications for U-Boot compatibility, and finally some documentation. Your feedback on these contributions would be greatly appreciated. Best regards, Changes in v3: - Add documentation to index and fix syntax issues - Add reference to documentation in doc/build/tools Changes in v2: - Add compile_commands.json to gitignore - Add documentation Joao Marcos Costa (7): scripts: Port Linux's gen_compile_commands.py to U-Boot scripts/gen_compile_commands.py: adapt _LINE_PATTERN scripts/gen_compile_commands.py: fix docstring scripts/gen_compile_commands.py: add acknowledgments .gitignore: add compile_commands.json doc: add documentation for gen_compile_commands.py doc: add new section to build/tools .gitignore | 3 + doc/build/gen_compile_commands.rst | 47 ++ doc/build/index.rst| 1 + doc/build/tools.rst| 13 ++ scripts/gen_compile_commands.py| 230 + 5 files changed, 294 insertions(+) create mode 100644 doc/build/gen_compile_commands.rst create mode 100755 scripts/gen_compile_commands.py -- 2.41.0
[PATCH v2 6/6] doc: add documentation for gen_compile_commands.py
This documentation briefly explains what is a compilation database,
and how to use the script to generate one.
This is not a portage, as there was no original documentation in the
Linux sources.
Acknowledge the documentation in the script's header.
Signed-off-by: Joao Marcos Costa
---
doc/develop/gen_compile_commands.rst | 46
scripts/gen_compile_commands.py | 1 +
2 files changed, 47 insertions(+)
create mode 100644 doc/develop/gen_compile_commands.rst
diff --git a/doc/develop/gen_compile_commands.rst
b/doc/develop/gen_compile_commands.rst
new file mode 100644
index 00..09466938fe
--- /dev/null
+++ b/doc/develop/gen_compile_commands.rst
@@ -0,0 +1,46 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+==
+gen_compile_commands
+==
+
+gen_compile_commands (scripts/gen_compile_commands.py) is a script used to
+generate a compilation database (compile_commands.json). This database consists
+of an array of "command objects" describing how each translation unit was
+compiled.
+
+Example::
+
+ {
+ "command": "gcc -Wp,-MD,arch/x86/cpu/.lapic.o.d -nostdinc -isystem (...)"
+ "directory": "/home/jmcosta/u-boot",
+ "file": "/home/jmcosta/u-boot/arch/x86/cpu/lapic.c"
+ }
+
+Such information comes from parsing the respective .cmd file of each
translation
+unit. In the previous example, that would be `arch/x86/cpu/.lapic.o.cmd`.
+
+The compilation database is quite useful for text editors (and IDEs) that use
+Clangd LSP. It allows jumping to definitions and declarations. Since it relies
+on parsing .cmd files, one needs to have a target (e.g. configs/*_defconfig)
+built before running the script.
+
+Example::
+
+ make sandbox_defconfig
+ make
+ ./scripts/gen_compile_commands.py
+
+The database will be in the root of the repository. No further modifications
are
+needed for it to be usable by the LSP, unless you set a name for the database
+other than it's default one (compile_commands.json).
+
+Options
+===
+
+For further details on how to use the script and its options, please refer to
+its help message, as in the example below.
+
+Help::
+
+ ./scripts/gen_compile_commands.py --help
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
index 1a9c49b34a..f0c6bafdc5 100755
--- a/scripts/gen_compile_commands.py
+++ b/scripts/gen_compile_commands.py
@@ -5,6 +5,7 @@
#
# Author: Tom Roeder
# Ported and modified for U-Boot by Joao Marcos Costa
+# Briefly documented at doc/develop/gen_compile_commands.rst
#
"""A tool for generating compile_commands.json in U-Boot."""
--
2.41.0
[PATCH v2 5/6] .gitignore: add compile_commands.json
Add Clang's compilation database file (i.e. compile_commands.json) to .gitignore, at the root of the repository. Signed-off-by: Joao Marcos Costa --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 002f95de4f..261a1d6754 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ __pycache__ # moveconfig database /moveconfig.db + +# Clang's compilation database file +/compile_commands.json -- 2.41.0
[PATCH v2 4/6] scripts/gen_compile_commands.py: add acknowledgments
Add acknowledgments for porting and modifying the script. Of course, the license, author, and copyright notice remain the same as in the original script. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 63d036a773..1a9c49b34a 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -4,6 +4,7 @@ # Copyright (C) Google LLC, 2018 # # Author: Tom Roeder +# Ported and modified for U-Boot by Joao Marcos Costa # """A tool for generating compile_commands.json in U-Boot.""" -- 2.41.0
[PATCH v2 3/6] scripts/gen_compile_commands.py: fix docstring
The referred tool is now in U-Boot. Replace "the Linux kernel" by "U-Boot" to make the docstring coherent. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 0227522959..63d036a773 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -5,7 +5,7 @@ # # Author: Tom Roeder # -"""A tool for generating compile_commands.json in the Linux kernel.""" +"""A tool for generating compile_commands.json in U-Boot.""" import argparse import json -- 2.41.0
[PATCH v2 2/6] scripts/gen_compile_commands.py: adapt _LINE_PATTERN
For U-Boot's context, the regular expression defined by _LINE_PATTERN should be adapted. Replace 'savedcmd' by 'cmd'. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 15ba56527a..0227522959 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. -- 2.41.0
[PATCH v2 1/6] scripts: Port Linux's gen_compile_commands.py to U-Boot
This script generates a database of compiler flags, namely
compile_commands.json. It is quite useful for text editors that use
clangd LSP (e.g. Vim, Neovim).
It was ported from Linux's sources:
- tag: v6.4
- revision 6995e2de6891c724bfeb2db33d7b87775f913ad1
Modifications for U-Boot compatibility will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
scripts/gen_compile_commands.py | 228
1 file changed, 228 insertions(+)
create mode 100755 scripts/gen_compile_commands.py
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
new file mode 100755
index 00..15ba56527a
--- /dev/null
+++ b/scripts/gen_compile_commands.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) Google LLC, 2018
+#
+# Author: Tom Roeder
+#
+"""A tool for generating compile_commands.json in the Linux kernel."""
+
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+
+_DEFAULT_OUTPUT = 'compile_commands.json'
+_DEFAULT_LOG_LEVEL = 'WARNING'
+
+_FILENAME_PATTERN = r'^\..*\.cmd$'
+_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)'
+_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+# The tools/ directory adopts a different build system, and produces .cmd
+# files in a different format. Do not support it.
+_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
+
+def parse_arguments():
+"""Sets up and parses command-line arguments.
+
+Returns:
+log_level: A logging level to filter log output.
+directory: The work directory where the objects were built.
+ar: Command used for parsing .a archives.
+output: Where to write the compile-commands JSON file.
+paths: The list of files/directories to handle to find .cmd files.
+"""
+usage = 'Creates a compile_commands.json database from kernel .cmd files'
+parser = argparse.ArgumentParser(description=usage)
+
+directory_help = ('specify the output directory used for the kernel build '
+ '(defaults to the working directory)')
+parser.add_argument('-d', '--directory', type=str, default='.',
+help=directory_help)
+
+output_help = ('path to the output command database (defaults to ' +
+ _DEFAULT_OUTPUT + ')')
+parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
+help=output_help)
+
+log_level_help = ('the level of log messages to produce (defaults to ' +
+ _DEFAULT_LOG_LEVEL + ')')
+parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
+default=_DEFAULT_LOG_LEVEL, help=log_level_help)
+
+ar_help = 'command used for parsing .a archives'
+parser.add_argument('-a', '--ar', type=str, default='llvm-ar',
help=ar_help)
+
+paths_help = ('directories to search or files to parse '
+ '(files should be *.o, *.a, or modules.order). '
+ 'If nothing is specified, the current directory is searched')
+parser.add_argument('paths', type=str, nargs='*', help=paths_help)
+
+args = parser.parse_args()
+
+return (args.log_level,
+os.path.abspath(args.directory),
+args.output,
+args.ar,
+args.paths if len(args.paths) > 0 else [args.directory])
+
+
+def cmdfiles_in_dir(directory):
+"""Generate the iterator of .cmd files found under the directory.
+
+Walk under the given directory, and yield every .cmd file found.
+
+Args:
+directory: The directory to search for .cmd files.
+
+Yields:
+The path to a .cmd file.
+"""
+
+filename_matcher = re.compile(_FILENAME_PATTERN)
+exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
+
+for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+# Prune unwanted directories.
+if dirpath in exclude_dirs:
+dirnames[:] = []
+continue
+
+for filename in filenames:
+if filename_matcher.match(filename):
+yield os.path.join(dirpath, filename)
+
+
+def to_cmdfile(path):
+"""Return the path of .cmd file used for the given build artifact
+
+Args:
+Path: file path
+
+Returns:
+The path to .cmd file
+"""
+dir, base = os.path.split(path)
+return os.path.join(dir, '.' + base + '.cmd')
+
+
+def cmdfiles_for_a(archive, ar):
+"""Generate the iterator of .cmd files associated with the archive.
+
+Parse the given archive, and yield every .cmd file used to build it.
+
+Args:
+archive: The archive to parse
+
+Yields:
+The path to every .cmd file found
+"""
+for obj in
[PATCH v2 0/6] Port gen_compile_commands.py from Linux to U-Boot
Hello U-Boot community, I'm submitting a patch series that ports the gen_compile_commands.py script from the Linux kernel's sources to U-Boot. This script, originally located in scripts/clang-tools/gen_compile_commands.py, enables the generation of compile_commands.json file for improved code navigation and analysis. The series consists of six patches: the initial script import and the necessary modifications for U-Boot compatibility. Your feedback on these contributions would be greatly appreciated. Best regards, Changes in v2: - Add compile_commands.json to gitignore - Add documentation Joao Marcos Costa (6): scripts: Port Linux's gen_compile_commands.py to U-Boot scripts/gen_compile_commands.py: adapt _LINE_PATTERN scripts/gen_compile_commands.py: fix docstring scripts/gen_compile_commands.py: add acknowledgments .gitignore: add compile_commands.json doc: add documentation for gen_compile_commands.py .gitignore | 3 + doc/develop/gen_compile_commands.rst | 46 ++ scripts/gen_compile_commands.py | 230 +++ 3 files changed, 279 insertions(+) create mode 100644 doc/develop/gen_compile_commands.rst create mode 100755 scripts/gen_compile_commands.py -- 2.41.0
[PATCH v1 4/4] scripts/gen_compile_commands.py: add acknowledgments
Add acknowledgments for porting and modifying the script. Of course, the license, author, and copyright notice remain the same as in the original script. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 63d036a773..1a9c49b34a 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -4,6 +4,7 @@ # Copyright (C) Google LLC, 2018 # # Author: Tom Roeder +# Ported and modified for U-Boot by Joao Marcos Costa # """A tool for generating compile_commands.json in U-Boot.""" -- 2.41.0
[PATCH v1 3/4] scripts/gen_compile_commands.py: fix docstring
The referred tool is now in U-Boot. Replace "the Linux kernel" by "U-Boot" to make the docstring coherent. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 0227522959..63d036a773 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -5,7 +5,7 @@ # # Author: Tom Roeder # -"""A tool for generating compile_commands.json in the Linux kernel.""" +"""A tool for generating compile_commands.json in U-Boot.""" import argparse import json -- 2.41.0
[PATCH v1 2/4] scripts/gen_compile_commands.py: adapt _LINE_PATTERN
For U-Boot's context, the regular expression defined by _LINE_PATTERN should be adapted. Replace 'savedcmd' by 'cmd'. Signed-off-by: Joao Marcos Costa --- scripts/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py index 15ba56527a..0227522959 100755 --- a/scripts/gen_compile_commands.py +++ b/scripts/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. -- 2.41.0
[PATCH v1 1/4] scripts: Port Linux's gen_compile_commands.py to U-Boot
This script generates a database of compiler flags, namely
compile_commands.json. It is quite useful for text editors that use
clangd LSP (e.g. Vim, Neovim).
It was ported from Linux's sources:
- tag: v6.4
- revision 6995e2de6891c724bfeb2db33d7b87775f913ad1
Modifications for U-Boot compatibility will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
scripts/gen_compile_commands.py | 228
1 file changed, 228 insertions(+)
create mode 100755 scripts/gen_compile_commands.py
diff --git a/scripts/gen_compile_commands.py b/scripts/gen_compile_commands.py
new file mode 100755
index 00..15ba56527a
--- /dev/null
+++ b/scripts/gen_compile_commands.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) Google LLC, 2018
+#
+# Author: Tom Roeder
+#
+"""A tool for generating compile_commands.json in the Linux kernel."""
+
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+
+_DEFAULT_OUTPUT = 'compile_commands.json'
+_DEFAULT_LOG_LEVEL = 'WARNING'
+
+_FILENAME_PATTERN = r'^\..*\.cmd$'
+_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)'
+_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+# The tools/ directory adopts a different build system, and produces .cmd
+# files in a different format. Do not support it.
+_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
+
+def parse_arguments():
+"""Sets up and parses command-line arguments.
+
+Returns:
+log_level: A logging level to filter log output.
+directory: The work directory where the objects were built.
+ar: Command used for parsing .a archives.
+output: Where to write the compile-commands JSON file.
+paths: The list of files/directories to handle to find .cmd files.
+"""
+usage = 'Creates a compile_commands.json database from kernel .cmd files'
+parser = argparse.ArgumentParser(description=usage)
+
+directory_help = ('specify the output directory used for the kernel build '
+ '(defaults to the working directory)')
+parser.add_argument('-d', '--directory', type=str, default='.',
+help=directory_help)
+
+output_help = ('path to the output command database (defaults to ' +
+ _DEFAULT_OUTPUT + ')')
+parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
+help=output_help)
+
+log_level_help = ('the level of log messages to produce (defaults to ' +
+ _DEFAULT_LOG_LEVEL + ')')
+parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
+default=_DEFAULT_LOG_LEVEL, help=log_level_help)
+
+ar_help = 'command used for parsing .a archives'
+parser.add_argument('-a', '--ar', type=str, default='llvm-ar',
help=ar_help)
+
+paths_help = ('directories to search or files to parse '
+ '(files should be *.o, *.a, or modules.order). '
+ 'If nothing is specified, the current directory is searched')
+parser.add_argument('paths', type=str, nargs='*', help=paths_help)
+
+args = parser.parse_args()
+
+return (args.log_level,
+os.path.abspath(args.directory),
+args.output,
+args.ar,
+args.paths if len(args.paths) > 0 else [args.directory])
+
+
+def cmdfiles_in_dir(directory):
+"""Generate the iterator of .cmd files found under the directory.
+
+Walk under the given directory, and yield every .cmd file found.
+
+Args:
+directory: The directory to search for .cmd files.
+
+Yields:
+The path to a .cmd file.
+"""
+
+filename_matcher = re.compile(_FILENAME_PATTERN)
+exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
+
+for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+# Prune unwanted directories.
+if dirpath in exclude_dirs:
+dirnames[:] = []
+continue
+
+for filename in filenames:
+if filename_matcher.match(filename):
+yield os.path.join(dirpath, filename)
+
+
+def to_cmdfile(path):
+"""Return the path of .cmd file used for the given build artifact
+
+Args:
+Path: file path
+
+Returns:
+The path to .cmd file
+"""
+dir, base = os.path.split(path)
+return os.path.join(dir, '.' + base + '.cmd')
+
+
+def cmdfiles_for_a(archive, ar):
+"""Generate the iterator of .cmd files associated with the archive.
+
+Parse the given archive, and yield every .cmd file used to build it.
+
+Args:
+archive: The archive to parse
+
+Yields:
+The path to every .cmd file found
+"""
+for obj in
[PATCH v1 0/4] Port gen_compile_commands.py from Linux to U-Boot
Hello U-Boot community, I'm submitting a patch series that ports the gen_compile_commands.py script from the Linux kernel's sources to U-Boot. This script, originally located in scripts/clang-tools/gen_compile_commands.py, enables the generation of compile_commands.json files for improved code navigation and analysis. The series consists of four patches: the initial script import and the necessary modifications for U-Boot compatibility. Your feedback on these contributions would be greatly appreciated. Best regards, Joao Marcos Costa (4): scripts: Port Linux's gen_compile_commands.py to U-Boot scripts/gen_compile_commands.py: adapt _LINE_PATTERN scripts/gen_compile_commands.py: fix docstring scripts/gen_compile_commands.py: add acknowledgments scripts/gen_compile_commands.py | 229 1 file changed, 229 insertions(+) create mode 100755 scripts/gen_compile_commands.py -- 2.41.0
[PATCH] MAINTAINERS: Update e-mail address
Replace former professional address by my personal e-mail. Signed-off-by: Joao Marcos Costa --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 7f27ff4c20..1b859194d6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1267,7 +1267,7 @@ F:drivers/spmi/ F: include/spmi/ SQUASHFS -M: Joao Marcos Costa +M: Joao Marcos Costa R: Thomas Petazzoni R: Miquel Raynal S: Maintained -- 2.25.1
[PATCH v3 3/3] test/py: rewrite sqfsls command test suite
Add more details to test cases by comparing each expected line with the
command's output. Add new test cases:
- sqfsls at an empty directory
- sqfsls at a sub-directory
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/test_sqfs_ls.py | 140 +++---
1 file changed, 121 insertions(+), 19 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
index a0dca2e2fc..9eb00d6888 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
@@ -4,7 +4,101 @@
import os
import pytest
-from sqfs_common import *
+
+from sqfs_common import STANDARD_TABLE
+from sqfs_common import generate_sqfs_src_dir, make_all_images
+from sqfs_common import clean_sqfs_src_dir, clean_all_images
+from sqfs_common import check_mksquashfs_version
+
+def sqfs_ls_at_root(u_boot_console):
+""" Runs sqfsls at image's root.
+
+This test checks if all the present files and directories were listed.
Also,
+it checks if passing the slash or not changes the output, which it
shouldn't.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+
+no_slash = u_boot_console.run_command('sqfsls host 0')
+slash = u_boot_console.run_command('sqfsls host 0 /')
+assert no_slash == slash
+
+expected_lines = ['empty-dir/', '1000 f1000', '4096 f4096', '5096
f5096',
+ 'subdir/', ' sym', '4 file(s), 2 dir(s)']
+
+output = u_boot_console.run_command('sqfsls host 0')
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_empty_dir(u_boot_console):
+""" Runs sqfsls at an empty directory.
+
+This tests checks if sqfsls will print anything other than the 'Empty
directory'
+message.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+assert u_boot_console.run_command('sqfsls host 0 empty-dir') == 'Empty
directory.'
+
+def sqfs_ls_at_subdir(u_boot_console):
+""" Runs sqfsls at the SquashFS image's subdir.
+
+This test checks if the path resolution works, since the directory is not
the
+root.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)']
+output = u_boot_console.run_command('sqfsls host 0 subdir')
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_symlink(u_boot_console):
+""" Runs sqfsls at a SquashFS image's symbolic link.
+
+This test checks if the symbolic link's target resolution works.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+# since sym -> subdir, the following outputs must be equal
+output = u_boot_console.run_command('sqfsls host 0 sym')
+output_subdir = u_boot_console.run_command('sqfsls host 0 subdir')
+assert output == output_subdir
+
+expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)']
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_non_existent_dir(u_boot_console):
+""" Runs sqfsls at a file and at a non-existent directory.
+
+This test checks if the SquashFS support won't crash if it doesn't find the
+specified directory or if it takes a file as an input instead of an actual
+directory. In both cases, the output should be the same.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+out_non_existent = u_boot_console.run_command('sqfsls host 0 fff')
+out_not_dir = u_boot_console.run_command('sqfsls host 0 f1000')
+assert out_non_existent == out_not_dir
+assert '** Cannot find directory. **' in out_non_existent
+
+def sqfs_run_all_ls_tests(u_boot_console):
+""" Runs all the previously defined test cases.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+sqfs_ls_at_root(u_boot_console)
+sqfs_ls_at_empty_dir(u_boot_console)
+sqfs_ls_at_subdir(u_boot_console)
+sqfs_ls_at_symlink(u_boot_console)
+sqfs_ls_at_non_existent_dir(u_boot_console)
@pytest.mark.boardspec('sandbox')
@pytest.mark.buildconfigspec('cmd_fs_generic')
@@ -12,25 +106,33 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_ls(u_boot_console):
+""" Executes the sqfsls test suite.
+
+First, it generates the SquashFS images, then it runs the test cases and
+finally cleans the workspace. If an exception is raised, the workspace is
+cleaned before exiting.
+
+Args:
+u_boot_
[PATCH v3 2/3] test/py: rewrite sqfsload command test suite
The previous strategy to know if a file was correctly loaded was to
check for how many bytes were read and compare it against the file's
original size. Since this is not a good solution, replace it by
comparing the checksum of the loaded bytes against the original file's
checksum. Add more test cases: files at a sub-directory and non-existent
file.
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/test_sqfs_load.py | 168 ++
1 file changed, 138 insertions(+), 30 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index 9e90062384..6ec6ccec6c 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -3,8 +3,118 @@
# Author: Joao Marcos Costa
import os
+import subprocess
import pytest
-from sqfs_common import *
+
+from sqfs_common import SQFS_SRC_DIR, STANDARD_TABLE
+from sqfs_common import generate_sqfs_src_dir, make_all_images
+from sqfs_common import clean_sqfs_src_dir, clean_all_images
+from sqfs_common import check_mksquashfs_version
+
+@pytest.mark.requiredtool('md5sum')
+def original_md5sum(path):
+""" Runs md5sum command.
+
+Args:
+path: path to original file.
+Returns:
+The original file's checksum as a string.
+"""
+
+out = subprocess.run(['md5sum ' + path], shell=True, check=True,
+ capture_output=True, text=True)
+checksum = out.stdout.split()[0]
+
+return checksum
+
+def uboot_md5sum(u_boot_console, address, count):
+""" Runs U-Boot's md5sum command.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+address: address where the file was loaded (e.g.: $kernel_addr_r).
+count: file's size. It was named 'count' to match md5sum's respective
+argument name.
+Returns:
+The checksum of the file loaded with sqfsload as a string.
+"""
+
+out = u_boot_console.run_command('md5sum {} {}'.format(address, count))
+checksum = out.split()[-1]
+
+return checksum
+
+def sqfs_load_files(u_boot_console, files, sizes, address):
+""" Loads files and asserts their checksums.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+files: list of files to be loaded.
+sizes: the sizes of each file.
+address: the address where the files should be loaded.
+"""
+build_dir = u_boot_console.config.build_dir
+for (file, size) in zip(files, sizes):
+out = u_boot_console.run_command('sqfsload host 0 {}
{}'.format(address, file))
+
+# check if the right amount of bytes was read
+assert size in out
+
+# compare original file's checksum against u-boot's
+u_boot_checksum = uboot_md5sum(u_boot_console, address, hex(int(size)))
+original_file_path = os.path.join(build_dir, SQFS_SRC_DIR + '/' + file)
+original_checksum = original_md5sum(original_file_path)
+assert u_boot_checksum == original_checksum
+
+def sqfs_load_files_at_root(u_boot_console):
+""" Calls sqfs_load_files passing the files at the SquashFS image's root.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+
+files = ['f4096', 'f5096', 'f1000']
+sizes = ['4096', '5096', '1000']
+address = '$kernel_addr_r'
+sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_files_at_subdir(u_boot_console):
+""" Calls sqfs_load_files passing the files at the SquashFS image's subdir.
+
+This test checks if the path resolution works, since the file is not at the
+root directory.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+files = ['subdir/subdir-file']
+sizes = ['100']
+address = '$kernel_addr_r'
+sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_non_existent_file(u_boot_console):
+""" Calls sqfs_load_files passing an non-existent file to raise an error.
+
+This test checks if the SquashFS support won't crash if it doesn't find the
+specified file.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+address = '$kernel_addr_r'
+file = 'non-existent'
+out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address,
file))
+assert 'Failed to load' in out
+
+def sqfs_run_all_load_tests(u_boot_console):
+""" Runs all the previously defined test cases.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+sqfs_load_files_at_root(u_boot_console)
+sqfs_load_
[PATCH v3 1/3] test/py: rewrite common tools for SquashFS tests
Remove the previous OOP approach, which was confusing and incomplete.
Add more test cases by making SquashFS images with various options,
concerning file fragmentation and its compression. Add comments to
properly document the code.
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/sqfs_common.py | 257 +-
1 file changed, 193 insertions(+), 64 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index c96f92c1d8..267c4b57d1 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -3,74 +3,203 @@
# Author: Joao Marcos Costa
import os
-import random
-import string
+import shutil
import subprocess
-def sqfs_get_random_letters(size):
-letters = []
-for i in range(0, size):
-letters.append(random.choice(string.ascii_letters))
+""" standard test images table: Each table item is a key:value pair
+representing the output image name and its respective mksquashfs options.
+This table should be modified only when adding support for new compression
+algorithms. The 'default' case takes no options but the input and output
+names, so it must be assigned with an empty string.
+"""
+STANDARD_TABLE = {
+'default' : '',
+'lzo_comp_frag' : '',
+'lzo_frag' : '',
+'lzo_no_frag' : '',
+'zstd_comp_frag' : '',
+'zstd_frag' : '',
+'zstd_no_frag' : '',
+'gzip_comp_frag' : '',
+'gzip_frag' : '',
+'gzip_no_frag' : ''
+}
-return ''.join(letters)
+""" EXTRA_TABLE: Set this table's keys and values if you want to make squashfs
+images with your own customized options.
+"""
+EXTRA_TABLE = {}
-def sqfs_generate_file(path, size):
-content = sqfs_get_random_letters(size)
-file = open(path, "w")
+# path to source directory used to make squashfs test images
+SQFS_SRC_DIR = 'sqfs_src_dir'
+
+def get_opts_list():
+""" Combines fragmentation and compression options into a list of strings.
+
+opts_list's firts item is an empty string as STANDARD_TABLE's first item is
+the 'default' case.
+
+Returns:
+A list of strings whose items are formed by a compression and a
+fragmentation option joined by a whitespace.
+"""
+# supported compression options only
+comp_opts = ['-comp lzo', '-comp zstd', '-comp gzip']
+# file fragmentation options
+frag_opts = ['-always-use-fragments', '-always-use-fragments -noF',
'-no-fragments']
+
+opts_list = [' ']
+for comp_opt in comp_opts:
+for frag_opt in frag_opts:
+opts_list.append(' '.join([comp_opt, frag_opt]))
+
+return opts_list
+
+def init_standard_table():
+""" Initializes STANDARD_TABLE values.
+
+STANDARD_TABLE's keys are pre-defined, and init_standard_table() assigns
+the right value for each one of them.
+"""
+opts_list = get_opts_list()
+
+for key, value in zip(STANDARD_TABLE.keys(), opts_list):
+STANDARD_TABLE[key] = value
+
+def generate_file(file_name, file_size):
+""" Generates a file filled with 'x'.
+
+Args:
+file_name: the file's name.
+file_size: the content's length and therefore the file size.
+"""
+content = 'x' * file_size
+
+file = open(file_name, 'w')
file.write(content)
file.close()
-class Compression:
-def __init__(self, name, files, sizes, block_size = 4096):
-self.name = name
-self.files = files
-self.sizes = sizes
-self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
-
-def add_opt(self, opt):
-self.mksquashfs_opts += " " + opt
-
-def gen_image(self, build_dir):
-src = os.path.join(build_dir, "sqfs_src/")
-os.mkdir(src)
-for (f, s) in zip(self.files, self.sizes):
-sqfs_generate_file(src + f, s)
-
-# the symbolic link always targets the first file
-os.symlink(self.files[0], src + "sym")
-
-sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
-i_o = src + " " + sqfs_img
-opts = self.mksquashfs_opts
-try:
-subprocess.run(["mksquashfs " + i_o + opts], shell = True, check =
True)
-except:
-print("mksquashfs error. Compression type: " + self.name)
-raise RuntimeError
-
-def clean_source(self, build_dir):
-src = os.path.join(build_dir, "sqfs_src/")
-for f in self.files:
-os.remove(src + f)
-os.remove(src + "sym")
-os.rmdir(src)
-
-def cleanup(self, build_dir):
-self.clean_source(build_dir)
-sqfs_img
[PATCH v3 0/3] test/py: Rewrite SquashFS commands test suite
Hello, This patch series fixes the following issues: - poor strategy to check if files were properly loaded - wrong quoting style for strings - tests failing at the second run because of a wrong clean-up strategy Finally, it improves: - code overall documentation level, with more comments and better naming for functions and variables - code readability by adding more helper functions - completeness: more test cases were added for both sqfsls and sqfsload commands The sqfsload new test suite may fail when testing images with fragmented files if the patch I previously sent (fs/squashfs: fix reading of fragmented files) is not applied, so this patch series depends on it. Changes since V2: - Add check_mksquashfs_version function to raise an exception if mksquashfs version is too old. I chose to set the required version as 4.4, since it seems to be the most recent one. Changes since V1: - Leave the copyright year as it was (2020) instead of changing it to 2021 - Remove spurious comments and print statements - Fix the style issues pointed by pylint3 Best regards, Joao Joao Marcos Costa (3): test/py: rewrite common tools for SquashFS tests test/py: rewrite sqfsload command test suite test/py: rewrite sqfsls command test suite .../test_fs/test_squashfs/sqfs_common.py | 257 +- .../test_fs/test_squashfs/test_sqfs_load.py | 168 ++-- .../test_fs/test_squashfs/test_sqfs_ls.py | 140 -- 3 files changed, 452 insertions(+), 113 deletions(-) -- 2.25.1
[PATCH v2 3/3] test/py: rewrite sqfsls command test suite
Add more details to test cases by comparing each expected line with the
command's output. Add new test cases:
- sqfsls at an empty directory
- sqfsls at a sub-directory
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/test_sqfs_ls.py | 138 +++---
1 file changed, 119 insertions(+), 19 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
index a0dca2e2fc..02d353f8ac 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
@@ -4,7 +4,100 @@
import os
import pytest
-from sqfs_common import *
+
+from sqfs_common import STANDARD_TABLE
+from sqfs_common import generate_sqfs_src_dir, make_all_images
+from sqfs_common import clean_sqfs_src_dir, clean_all_images
+
+def sqfs_ls_at_root(u_boot_console):
+""" Runs sqfsls at image's root.
+
+This test checks if all the present files and directories were listed.
Also,
+it checks if passing the slash or not changes the output, which it
shouldn't.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+
+no_slash = u_boot_console.run_command('sqfsls host 0')
+slash = u_boot_console.run_command('sqfsls host 0 /')
+assert no_slash == slash
+
+expected_lines = ['empty-dir/', '1000 f1000', '4096 f4096', '5096
f5096',
+ 'subdir/', ' sym', '4 file(s), 2 dir(s)']
+
+output = u_boot_console.run_command('sqfsls host 0')
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_empty_dir(u_boot_console):
+""" Runs sqfsls at an empty directory.
+
+This tests checks if sqfsls will print anything other than the 'Empty
directory'
+message.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+assert u_boot_console.run_command('sqfsls host 0 empty-dir') == 'Empty
directory.'
+
+def sqfs_ls_at_subdir(u_boot_console):
+""" Runs sqfsls at the SquashFS image's subdir.
+
+This test checks if the path resolution works, since the directory is not
the
+root.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)']
+output = u_boot_console.run_command('sqfsls host 0 subdir')
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_symlink(u_boot_console):
+""" Runs sqfsls at a SquashFS image's symbolic link.
+
+This test checks if the symbolic link's target resolution works.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+# since sym -> subdir, the following outputs must be equal
+output = u_boot_console.run_command('sqfsls host 0 sym')
+output_subdir = u_boot_console.run_command('sqfsls host 0 subdir')
+assert output == output_subdir
+
+expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)']
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_non_existent_dir(u_boot_console):
+""" Runs sqfsls at a file and at a non-existent directory.
+
+This test checks if the SquashFS support won't crash if it doesn't find the
+specified directory or if it takes a file as an input instead of an actual
+directory. In both cases, the output should be the same.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+out_non_existent = u_boot_console.run_command('sqfsls host 0 fff')
+out_not_dir = u_boot_console.run_command('sqfsls host 0 f1000')
+assert out_non_existent == out_not_dir
+assert '** Cannot find directory. **' in out_non_existent
+
+def sqfs_run_all_ls_tests(u_boot_console):
+""" Runs all the previously defined test cases.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+sqfs_ls_at_root(u_boot_console)
+sqfs_ls_at_empty_dir(u_boot_console)
+sqfs_ls_at_subdir(u_boot_console)
+sqfs_ls_at_symlink(u_boot_console)
+sqfs_ls_at_non_existent_dir(u_boot_console)
@pytest.mark.boardspec('sandbox')
@pytest.mark.buildconfigspec('cmd_fs_generic')
@@ -12,25 +105,32 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_ls(u_boot_console):
+""" Executes the sqfsls test suite.
+
+First, it generates the SquashFS images, then it runs the test cases and
+finally cleans the workspace. If an exception is raised, the workspace is
+cleaned before exiting.
+
+Args:
+u_boot_console: provides the means to
[PATCH v2 2/3] test/py: rewrite sqfsload command test suite
The previous strategy to know if a file was correctly loaded was to
check for how many bytes were read and compare it against the file's
original size. Since this is not a good solution, replace it by
comparing the checksum of the loaded bytes against the original file's
checksum. Add more test cases: files at a sub-directory and non-existent
file.
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/test_sqfs_load.py | 166 ++
1 file changed, 136 insertions(+), 30 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index 9e90062384..ef4be1339c 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -3,8 +3,117 @@
# Author: Joao Marcos Costa
import os
+import subprocess
import pytest
-from sqfs_common import *
+
+from sqfs_common import SQFS_SRC_DIR, STANDARD_TABLE
+from sqfs_common import generate_sqfs_src_dir, make_all_images
+from sqfs_common import clean_sqfs_src_dir, clean_all_images
+
+@pytest.mark.requiredtool('md5sum')
+def original_md5sum(path):
+""" Runs md5sum command.
+
+Args:
+path: path to original file.
+Returns:
+The original file's checksum as a string.
+"""
+
+out = subprocess.run(['md5sum ' + path], shell=True, check=True,
+ capture_output=True, text=True)
+checksum = out.stdout.split()[0]
+
+return checksum
+
+def uboot_md5sum(u_boot_console, address, count):
+""" Runs U-Boot's md5sum command.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+address: address where the file was loaded (e.g.: $kernel_addr_r).
+count: file's size. It was named 'count' to match md5sum's respective
+argument name.
+Returns:
+The checksum of the file loaded with sqfsload as a string.
+"""
+
+out = u_boot_console.run_command('md5sum {} {}'.format(address, count))
+checksum = out.split()[-1]
+
+return checksum
+
+def sqfs_load_files(u_boot_console, files, sizes, address):
+""" Loads files and asserts their checksums.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+files: list of files to be loaded.
+sizes: the sizes of each file.
+address: the address where the files should be loaded.
+"""
+build_dir = u_boot_console.config.build_dir
+for (file, size) in zip(files, sizes):
+out = u_boot_console.run_command('sqfsload host 0 {}
{}'.format(address, file))
+
+# check if the right amount of bytes was read
+assert size in out
+
+# compare original file's checksum against u-boot's
+u_boot_checksum = uboot_md5sum(u_boot_console, address, hex(int(size)))
+original_file_path = os.path.join(build_dir, SQFS_SRC_DIR + '/' + file)
+original_checksum = original_md5sum(original_file_path)
+assert u_boot_checksum == original_checksum
+
+def sqfs_load_files_at_root(u_boot_console):
+""" Calls sqfs_load_files passing the files at the SquashFS image's root.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+
+files = ['f4096', 'f5096', 'f1000']
+sizes = ['4096', '5096', '1000']
+address = '$kernel_addr_r'
+sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_files_at_subdir(u_boot_console):
+""" Calls sqfs_load_files passing the files at the SquashFS image's subdir.
+
+This test checks if the path resolution works, since the file is not at the
+root directory.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+files = ['subdir/subdir-file']
+sizes = ['100']
+address = '$kernel_addr_r'
+sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_non_existent_file(u_boot_console):
+""" Calls sqfs_load_files passing an non-existent file to raise an error.
+
+This test checks if the SquashFS support won't crash if it doesn't find the
+specified file.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+address = '$kernel_addr_r'
+file = 'non-existent'
+out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address,
file))
+assert 'Failed to load' in out
+
+def sqfs_run_all_load_tests(u_boot_console):
+""" Runs all the previously defined test cases.
+
+Args:
+u_boot_console: provides the means to interact with U-Boot's console.
+"""
+sqfs_load_files_at_root(u_boot_console)
+sqfs_load_files_at_subdir(u_boot_console)
+sqfs_loa
[PATCH v2 1/3] test/py: rewrite common tools for SquashFS tests
Remove the previous OOP approach, which was confusing and incomplete.
Add more test cases by making SquashFS images with various options,
concerning file fragmentation and its compression. Add comments to
properly document the code.
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/sqfs_common.py | 237 +-
1 file changed, 173 insertions(+), 64 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index c96f92c1d8..5469b1c017 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -3,74 +3,183 @@
# Author: Joao Marcos Costa
import os
-import random
-import string
+import shutil
import subprocess
-def sqfs_get_random_letters(size):
-letters = []
-for i in range(0, size):
-letters.append(random.choice(string.ascii_letters))
+""" standard test images table: Each table item is a key:value pair
+representing the output image name and its respective mksquashfs options.
+This table should be modified only when adding support for new compression
+algorithms. The 'default' case takes no options but the input and output
+names, so it must be assigned with an empty string.
+"""
+STANDARD_TABLE = {
+'default' : '',
+'lzo_comp_frag' : '',
+'lzo_frag' : '',
+'lzo_no_frag' : '',
+'zstd_comp_frag' : '',
+'zstd_frag' : '',
+'zstd_no_frag' : '',
+'gzip_comp_frag' : '',
+'gzip_frag' : '',
+'gzip_no_frag' : ''
+}
-return ''.join(letters)
+""" EXTRA_TABLE: Set this table's keys and values if you want to make squashfs
+images with your own customized options.
+"""
+EXTRA_TABLE = {}
-def sqfs_generate_file(path, size):
-content = sqfs_get_random_letters(size)
-file = open(path, "w")
+# path to source directory used to make squashfs test images
+SQFS_SRC_DIR = 'sqfs_src_dir'
+
+def get_opts_list():
+""" Combines fragmentation and compression options into a list of strings.
+
+opts_list's firts item is an empty string as STANDARD_TABLE's first item is
+the 'default' case.
+
+Returns:
+A list of strings whose items are formed by a compression and a
+fragmentation option joined by a whitespace.
+"""
+# supported compression options only
+comp_opts = ['-comp lzo', '-comp zstd', '-comp gzip']
+# file fragmentation options
+frag_opts = ['-always-use-fragments', '-always-use-fragments -noF',
'-no-fragments']
+
+opts_list = [' ']
+for comp_opt in comp_opts:
+for frag_opt in frag_opts:
+opts_list.append(' '.join([comp_opt, frag_opt]))
+
+return opts_list
+
+def init_standard_table():
+""" Initializes STANDARD_TABLE values.
+
+STANDARD_TABLE's keys are pre-defined, and init_standard_table() assigns
+the right value for each one of them.
+"""
+opts_list = get_opts_list()
+
+for key, value in zip(STANDARD_TABLE.keys(), opts_list):
+STANDARD_TABLE[key] = value
+
+def generate_file(file_name, file_size):
+""" Generates a file filled with 'x'.
+
+Args:
+file_name: the file's name.
+file_size: the content's length and therefore the file size.
+"""
+content = 'x' * file_size
+
+file = open(file_name, 'w')
file.write(content)
file.close()
-class Compression:
-def __init__(self, name, files, sizes, block_size = 4096):
-self.name = name
-self.files = files
-self.sizes = sizes
-self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
-
-def add_opt(self, opt):
-self.mksquashfs_opts += " " + opt
-
-def gen_image(self, build_dir):
-src = os.path.join(build_dir, "sqfs_src/")
-os.mkdir(src)
-for (f, s) in zip(self.files, self.sizes):
-sqfs_generate_file(src + f, s)
-
-# the symbolic link always targets the first file
-os.symlink(self.files[0], src + "sym")
-
-sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
-i_o = src + " " + sqfs_img
-opts = self.mksquashfs_opts
-try:
-subprocess.run(["mksquashfs " + i_o + opts], shell = True, check =
True)
-except:
-print("mksquashfs error. Compression type: " + self.name)
-raise RuntimeError
-
-def clean_source(self, build_dir):
-src = os.path.join(build_dir, "sqfs_src/")
-for f in self.files:
-os.remove(src + f)
-os.remove(src + "sym")
-os.rmdir(src)
-
-def cleanup(self, build_dir):
-self.clean_source(build_dir)
-sqfs_img
[PATCH v2 0/3] test/py: Rewrite SquashFS commands test suite
Hello, This patch series fixes the following issues: - poor strategy to check if files were properly loaded - wrong quoting style for strings - tests failing at the second run because of a wrong clean-up strategy Finally, it improves: - code overall documentation level, with more comments and better naming for functions and variables - code readability by adding more helper functions - completeness: more test cases were added for both sqfsls and sqfsload commands The sqfsload new test suite may fail when testing images with fragmented files if the patch I previously sent (fs/squashfs: fix reading of fragmented files) is not applied, so this patch series depends on it. Changes since V1: - Leave the copyright year as it was (2020) instead of changing it to 2021 - Remove spurious comments and print statements - Fix the style issues pointed by pylint3 Best regards, Joao Joao Marcos Costa (3): test/py: rewrite common tools for SquashFS tests test/py: rewrite sqfsload command test suite test/py: rewrite sqfsls command test suite .../test_fs/test_squashfs/sqfs_common.py | 237 +- .../test_fs/test_squashfs/test_sqfs_load.py | 166 +--- .../test_fs/test_squashfs/test_sqfs_ls.py | 138 -- 3 files changed, 428 insertions(+), 113 deletions(-) -- 2.25.1
[PATCH 3/3] test/py: rewrite sqfsls command test suite
Add more details to test cases by comparing each expected line with the
command's output. Add new test cases:
- sqfsls at an empty directory
- sqfsls at a sub-directory
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/test_sqfs_ls.py | 80 ++-
1 file changed, 62 insertions(+), 18 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
index a0dca2e2fc..2895aefc3b 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
@@ -6,6 +6,52 @@ import os
import pytest
from sqfs_common import *
+def sqfs_ls_at_root(u_boot_console):
+# first and most basic test is to check if these two give the same output
+no_slash = u_boot_console.run_command('sqfsls host 0')
+slash = u_boot_console.run_command('sqfsls host 0 /')
+assert no_slash == slash
+
+expected_lines = ['empty-dir/', '1000 f1000', '4096 f4096', '5096
f5096',
+'subdir/', ' sym', '4 file(s), 2 dir(s)']
+
+output = u_boot_console.run_command('sqfsls host 0')
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_empty_dir(u_boot_console):
+assert u_boot_console.run_command('sqfsls host 0 empty-dir') == 'Empty
directory.'
+
+def sqfs_ls_at_subdir(u_boot_console):
+expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)']
+output = u_boot_console.run_command('sqfsls host 0 subdir')
+for line in expected_lines:
+assert line in output
+
+def sqfs_ls_at_symlink(u_boot_console):
+# since sym -> subdir, the following outputs must be equal
+output = u_boot_console.run_command('sqfsls host 0 sym')
+output_subdir= u_boot_console.run_command('sqfsls host 0 subdir')
+assert output == output_subdir
+
+expected_lines = ['100 subdir-file', '1 file(s), 0 dir(s)']
+for line in expected_lines:
+assert line in output
+
+# output should be the same for non-existent directories and files
+def sqfs_ls_at_non_existent_dir(u_boot_console):
+out_non_existent = u_boot_console.run_command('sqfsls host 0 fff')
+out_not_dir = u_boot_console.run_command('sqfsls host 0 f1000')
+assert out_non_existent == out_not_dir
+assert '** Cannot find directory. **' in out_non_existent
+
+def sqfs_run_all_ls_tests(u_boot_console):
+sqfs_ls_at_root(u_boot_console)
+sqfs_ls_at_empty_dir(u_boot_console)
+sqfs_ls_at_subdir(u_boot_console)
+sqfs_ls_at_symlink(u_boot_console)
+sqfs_ls_at_non_existent_dir(u_boot_console)
+
@pytest.mark.boardspec('sandbox')
@pytest.mark.buildconfigspec('cmd_fs_generic')
@pytest.mark.buildconfigspec('cmd_squashfs')
@@ -13,24 +59,22 @@ from sqfs_common import *
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_ls(u_boot_console):
build_dir = u_boot_console.config.build_dir
-for opt in comp_opts:
-try:
-opt.gen_image(build_dir)
-except RuntimeError:
-opt.clean_source(build_dir)
-# skip unsupported compression types
-continue
-path = os.path.join(build_dir, "sqfs-" + opt.name)
-output = u_boot_console.run_command("host bind 0 " + path)
+# setup test environment
+generate_sqfs_src_dir(build_dir)
+make_all_images(build_dir)
+
+# run all tests for each image
+for image in standard_table.keys():
try:
-# list files in root directory
-output = u_boot_console.run_command("sqfsls host 0")
-assert str(len(opt.files) + 1) + " file(s), 0 dir(s)" in output
-assert " sym" in output
-output = u_boot_console.run_command("sqfsls host 0 xxx")
-assert "** Cannot find directory. **" in output
+image_path = os.path.join(build_dir, image)
+u_boot_console.run_command('host bind 0 {}'.format(image_path))
+sqfs_run_all_ls_tests(u_boot_console)
except:
-opt.cleanup(build_dir)
-assert False
-opt.cleanup(build_dir)
+clean_all_images(build_dir)
+clean_sqfs_src_dir(build_dir)
+raise AssertionError
+
+# clean test environment
+clean_all_images(build_dir)
+clean_sqfs_src_dir(build_dir)
--
2.25.1
[PATCH 2/3] test/py: rewrite sqfsload command test suite
The previous strategy to know if a file was correctly loaded was to
check for how many bytes were read and compare it against the file's
original size. Since this is not a good solution, replace it by
comparing the checksum of the loaded bytes against the original file's
checksum. Add more test cases: files at a sub-directory and non-existent
file.
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/test_sqfs_load.py | 99 +--
1 file changed, 69 insertions(+), 30 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index 9e90062384..0b416eb4c3 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -1,11 +1,62 @@
# SPDX-License-Identifier: GPL-2.0
-# Copyright (C) 2020 Bootlin
+# Copyright (C) 2021 Bootlin
# Author: Joao Marcos Costa
import os
import pytest
from sqfs_common import *
+@pytest.mark.requiredtool('md5sum')
+def original_md5sum(path):
+out = subprocess.run(['md5sum ' + path], shell = True, check = True,
+capture_output = True, text = True)
+checksum = out.stdout.split()[0]
+
+return checksum
+
+def uboot_md5sum(u_boot_console, address, count):
+out = u_boot_console.run_command('md5sum {} {}'.format(address, count))
+checksum = out.split()[-1]
+
+return checksum
+
+# loads files and asserts checksums
+def sqfs_load_files(u_boot_console, files, sizes, address):
+build_dir = u_boot_console.config.build_dir
+for (f, size) in zip(files, sizes):
+out = u_boot_console.run_command('sqfsload host 0 {}
{}'.format(address, f))
+
+# check if the right amount of bytes was read
+assert size in out
+
+# compare original file's checksum against u-boot's
+u_boot_checksum = uboot_md5sum(u_boot_console, address, hex(int(size)))
+original_checksum = original_md5sum(os.path.join(build_dir,
sqfs_src_dir + '/' + f))
+assert u_boot_checksum == original_checksum
+
+def sqfs_load_files_at_root(u_boot_console):
+files = ['f4096', 'f5096', 'f1000']
+sizes = ['4096', '5096', '1000']
+address = '$kernel_addr_r'
+sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_files_at_subdir(u_boot_console):
+files = ['subdir/subdir-file']
+sizes = ['100']
+address = '$kernel_addr_r'
+sqfs_load_files(u_boot_console, files, sizes, address)
+
+def sqfs_load_non_existent_file(u_boot_console):
+address = '$kernel_addr_r'
+f = 'non-existent'
+out = u_boot_console.run_command('sqfsload host 0 {} {}'.format(address,
f))
+assert 'Failed to load' in out
+
+def sqfs_run_all_load_tests(u_boot_console):
+sqfs_load_files_at_root(u_boot_console)
+sqfs_load_files_at_subdir(u_boot_console)
+sqfs_load_non_existent_file(u_boot_console)
+
@pytest.mark.boardspec('sandbox')
@pytest.mark.buildconfigspec('cmd_fs_generic')
@pytest.mark.buildconfigspec('cmd_squashfs')
@@ -13,34 +64,22 @@ from sqfs_common import *
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_load(u_boot_console):
build_dir = u_boot_console.config.build_dir
-command = "sqfsload host 0 $kernel_addr_r "
-for opt in comp_opts:
-# generate and load the squashfs image
+# setup test environment
+generate_sqfs_src_dir(build_dir)
+make_all_images(build_dir)
+
+# run all tests for each image
+for image in standard_table.keys():
try:
-opt.gen_image(build_dir)
-except RuntimeError:
-opt.clean_source(build_dir)
-# skip unsupported compression types
-continue
-
-path = os.path.join(build_dir, "sqfs-" + opt.name)
-output = u_boot_console.run_command("host bind 0 " + path)
-
-output = u_boot_console.run_command(command + "xxx")
-assert "File not found." in output
-
-for (f, s) in zip(opt.files, opt.sizes):
-try:
-output = u_boot_console.run_command(command + f)
-assert str(s) in output
-except:
-assert False
-opt.cleanup(build_dir)
-
-# test symbolic link
-output = u_boot_console.run_command(command + "sym")
-assert str(opt.sizes[0]) in output
-
-# remove generated files
-opt.cleanup(build_dir)
+image_path = os.path.join(build_dir, image)
+u_boot_console.run_command('host bind 0 {}'.format(image_path))
+sqfs_run_all_load_tests(u_boot_console)
+except:
+clean_all_images(build_dir)
+clean_sqfs_src_dir(build_dir)
+raise AssertionError
+
+# clean test environment
+clean_all_images(build_dir)
+clean_sqfs_src_dir(build_dir)
--
2.25.1
[PATCH 1/3] test/py: rewrite common tools for SquashFS tests
Remove the previous OOP approach, which was confusing and incomplete.
Add more test cases by making SquashFS images with various options,
concerning file fragmentation and its compression. Add comments to
properly document the code.
Signed-off-by: Joao Marcos Costa
---
.../test_fs/test_squashfs/sqfs_common.py | 198 --
1 file changed, 133 insertions(+), 65 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index c96f92c1d8..81a378a9f9 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -1,76 +1,144 @@
# SPDX-License-Identifier: GPL-2.0
-# Copyright (C) 2020 Bootlin
+# Copyright (C) 2021 Bootlin
# Author: Joao Marcos Costa
import os
-import random
-import string
+import shutil
import subprocess
-def sqfs_get_random_letters(size):
-letters = []
-for i in range(0, size):
-letters.append(random.choice(string.ascii_letters))
+"""
+standard test images table: Each table item is a key:value pair representing
the
+output image name and its respective mksquashfs options. This table should be
+modified only when adding support for new compression algorithms.
+The 'default' case takes no options but the input and output names, so it must
+be assigned with an empty string.
+"""
+standard_table = {
+'default' : '',
+'lzo_comp_frag' : '',
+'lzo_frag' : '',
+'lzo_no_frag' : '',
+'zstd_comp_frag' : '',
+'zstd_frag' : '',
+'zstd_no_frag' : '',
+'gzip_comp_frag' : '',
+'gzip_frag' : '',
+'gzip_no_frag' : ''
+}
-return ''.join(letters)
+"""
+extra_table: Set this table's keys and values if you want to make squashfs
images with
+your own customized options.
+"""
+extra_table = {}
-def sqfs_generate_file(path, size):
-content = sqfs_get_random_letters(size)
-file = open(path, "w")
+# path to source directory used to make squashfs test images
+sqfs_src_dir = 'sqfs_src_dir'
+
+"""
+Combines fragmentation and compression options into a list of strings.
+opts_list's firts item is an empty string as standard_table's first item is
+the 'default' case.
+"""
+def get_opts_list():
+# supported compression options only
+comp_opts = ['-comp lzo', '-comp zstd', '-comp gzip']
+# file fragmentation options
+frag_opts = ['-always-use-fragments', '-always-use-fragments -noF',
'-no-fragments']
+
+opts_list = [' ']
+for c in comp_opts:
+for f in frag_opts:
+opts_list.append(' '.join([c, f]))
+
+return opts_list
+
+def init_standard_table():
+opts_list = get_opts_list()
+
+for key, value in zip(standard_table.keys(), opts_list):
+standard_table[key] = value
+
+def generate_file(file_name, file_size):
+# file is filled with file_size * 'x'
+content = ''
+for i in range(file_size):
+content += 'x'
+
+file = open(file_name, 'w')
file.write(content)
file.close()
-class Compression:
-def __init__(self, name, files, sizes, block_size = 4096):
-self.name = name
-self.files = files
-self.sizes = sizes
-self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
-
-def add_opt(self, opt):
-self.mksquashfs_opts += " " + opt
-
-def gen_image(self, build_dir):
-src = os.path.join(build_dir, "sqfs_src/")
-os.mkdir(src)
-for (f, s) in zip(self.files, self.sizes):
-sqfs_generate_file(src + f, s)
-
-# the symbolic link always targets the first file
-os.symlink(self.files[0], src + "sym")
-
-sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
-i_o = src + " " + sqfs_img
-opts = self.mksquashfs_opts
-try:
-subprocess.run(["mksquashfs " + i_o + opts], shell = True, check =
True)
-except:
-print("mksquashfs error. Compression type: " + self.name)
-raise RuntimeError
-
-def clean_source(self, build_dir):
-src = os.path.join(build_dir, "sqfs_src/")
-for f in self.files:
-os.remove(src + f)
-os.remove(src + "sym")
-os.rmdir(src)
-
-def cleanup(self, build_dir):
-self.clean_source(build_dir)
-sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
-os.remove(sqfs_img)
-
-files = ["blks_only", "blks_frag", "frag_only"]
-sizes = [4096, 5100, 100]
-gzip = Compression("gzip", files, sizes)
-zstd = Compression("zstd", files, sizes)
-lzo = Compression("lzo", files, sizes)
-
-# use fragment blocks for files larger than block_s
[PATCH 0/3] test/py: Rewrite SquashFS commands test suite
Hello, This patch series fixes the following issues: - poor strategy to check if files were properly loaded - wrong quoting style for strings - tests failing at the second run because of a wrong clean-up strategy Finally, it improves: - code overall documentation level, with more comments and better naming for functions and variables - code readability by adding more helper functions - completeness: more test cases were added for both sqfsls and sqfsload commands The sqfsload new test suite may fail when testing images with fragmented files if the patch I previously sent (fs/squashfs: fix reading of fragmented files) is not applied, so this patch series depends on it. Best regards, Joao Joao Marcos Costa (3): test/py: rewrite common tools for SquashFS tests test/py: rewrite sqfsload command test suite test/py: rewrite sqfsls command test suite .../test_fs/test_squashfs/sqfs_common.py | 198 -- .../test_fs/test_squashfs/test_sqfs_load.py | 99 ++--- .../test_fs/test_squashfs/test_sqfs_ls.py | 80 +-- 3 files changed, 264 insertions(+), 113 deletions(-) -- 2.25.1
[PATCH] fs/squashfs: fix reading of fragmented files
The fragmented files were not correctly read because of two issues:
- The squashfs_file_info struct has a field named 'comp', which tells if
the file's fragment is compressed or not. This field was always set to
'true' in sqfs_get_regfile_info and sqfs_get_lregfile_info. It should
actually take sqfs_frag_lookup's return value. This patch addresses
these two assignments.
- In sqfs_read, the fragments (compressed or not) were copied to the
output buffer through a for loop which was reading data at the wrong
offset. Replace these loops by equivalent calls to memcpy, with the
right parameters.
I tested this patch by comparing the MD5 checksum of a few fragmented
files with the respective md5sum output in sandbox, considering both
compressed and uncompressed fragments.
Signed-off-by: Joao Marcos Costa
---
fs/squashfs/sqfs.c | 16 ++--
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 29805c3c6f..22ef4f2691 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -1253,7 +1253,7 @@ static int sqfs_get_regfile_info(struct
squashfs_reg_inode *reg,
fentry);
if (ret < 0)
return -EINVAL;
- finfo->comp = true;
+ finfo->comp = ret;
if (fentry->size < 1 || fentry->start == 0x7FFF)
return -EINVAL;
} else {
@@ -1291,7 +1291,7 @@ static int sqfs_get_lregfile_info(struct
squashfs_lreg_inode *lreg,
fentry);
if (ret < 0)
return -EINVAL;
- finfo->comp = true;
+ finfo->comp = ret;
if (fentry->size < 1 || fentry->start == 0x7FFF)
return -EINVAL;
} else {
@@ -1547,20 +1547,16 @@ int sqfs_read(const char *filename, void *buf, loff_t
offset, loff_t len,
goto out;
}
- for (j = *actread; j < finfo.size; j++) {
- memcpy(buf + j, _block[finfo.offset + j], 1);
- (*actread)++;
- }
+ memcpy(buf + *actread, _block[finfo.offset],
finfo.size - *actread);
+ *actread = finfo.size;
free(fragment_block);
} else if (finfo.frag && !finfo.comp) {
fragment_block = (void *)fragment + table_offset;
- for (j = *actread; j < finfo.size; j++) {
- memcpy(buf + j, _block[finfo.offset + j], 1);
- (*actread)++;
- }
+ memcpy(buf + *actread, _block[finfo.offset],
finfo.size - *actread);
+ *actread = finfo.size;
}
out:
--
2.25.1
[PATCH] fs/squashfs: Fix Coverity Scan defects
Fix control flow issues and null pointer dereferences.
Signed-off-by: Joao Marcos Costa
---
fs/squashfs/sqfs.c | 20 +---
fs/squashfs/sqfs_dir.c | 3 +--
fs/squashfs/sqfs_inode.c | 5 -
3 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index f67f7c4a40..15208b4dab 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -154,7 +154,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
header = get_unaligned_le16(metadata_buffer + table_offset);
metadata = metadata_buffer + table_offset + SQFS_HEADER_SIZE;
- if (!metadata) {
+ if (!metadata || !header) {
ret = -ENOMEM;
goto free_buffer;
}
@@ -434,9 +434,9 @@ static int sqfs_search_dir(struct squashfs_dir_stream
*dirs, char **token_list,
{
struct squashfs_super_block *sblk = ctxt.sblk;
char *path, *target, **sym_tokens, *res, *rem;
- struct squashfs_ldir_inode *ldir = NULL;
int j, ret, new_inode_number, offset;
struct squashfs_symlink_inode *sym;
+ struct squashfs_ldir_inode *ldir;
struct squashfs_dir_inode *dir;
struct fs_dir_stream *dirsp;
struct fs_dirent *dent;
@@ -448,8 +448,8 @@ static int sqfs_search_dir(struct squashfs_dir_stream
*dirs, char **token_list,
table = sqfs_find_inode(dirs->inode_table, le32_to_cpu(sblk->inodes),
sblk->inodes, sblk->block_size);
- /* root is a regular directory, not an extended one */
dir = (struct squashfs_dir_inode *)table;
+ ldir = (struct squashfs_ldir_inode *)table;
/* get directory offset in directory table */
offset = sqfs_dir_offset(table, m_list, m_count);
@@ -1146,7 +1146,10 @@ static int sqfs_get_regfile_info(struct
squashfs_reg_inode *reg,
finfo->start = get_unaligned_le32(>start_block);
finfo->frag = SQFS_IS_FRAGMENTED(get_unaligned_le32(>fragment));
- if (finfo->size < 1 || finfo->offset < 0 || finfo->start < 0)
+ if (finfo->frag && finfo->offset == 0x)
+ return -EINVAL;
+
+ if (finfo->size < 1 || finfo->start == 0x)
return -EINVAL;
if (finfo->frag) {
@@ -1156,7 +1159,7 @@ static int sqfs_get_regfile_info(struct
squashfs_reg_inode *reg,
if (ret < 0)
return -EINVAL;
finfo->comp = true;
- if (fentry->size < 1 || fentry->start < 0)
+ if (fentry->size < 1 || fentry->start == 0x7FFF)
return -EINVAL;
} else {
datablk_count = DIV_ROUND_UP(finfo->size, le32_to_cpu(blksz));
@@ -1181,7 +1184,10 @@ static int sqfs_get_lregfile_info(struct
squashfs_lreg_inode *lreg,
finfo->start = get_unaligned_le64(>start_block);
finfo->frag = SQFS_IS_FRAGMENTED(get_unaligned_le32(>fragment));
- if (finfo->size < 1 || finfo->offset < 0 || finfo->start < 0)
+ if (finfo->frag && finfo->offset == 0x)
+ return -EINVAL;
+
+ if (finfo->size < 1 || finfo->start == 0x7FFF)
return -EINVAL;
if (finfo->frag) {
@@ -1191,7 +1197,7 @@ static int sqfs_get_lregfile_info(struct
squashfs_lreg_inode *lreg,
if (ret < 0)
return -EINVAL;
finfo->comp = true;
- if (fentry->size < 1 || fentry->start < 0)
+ if (fentry->size < 1 || fentry->start == 0x7FFF)
return -EINVAL;
} else {
datablk_count = DIV_ROUND_UP(finfo->size, le32_to_cpu(blksz));
diff --git a/fs/squashfs/sqfs_dir.c b/fs/squashfs/sqfs_dir.c
index 00d2891e7d..a265b98fe6 100644
--- a/fs/squashfs/sqfs_dir.c
+++ b/fs/squashfs/sqfs_dir.c
@@ -34,8 +34,7 @@ int sqfs_dir_offset(void *dir_i, u32 *m_list, int m_count)
struct squashfs_ldir_inode *ldir;
struct squashfs_dir_inode *dir;
u32 start_block;
- u16 offset;
- int j;
+ int j, offset;
switch (get_unaligned_le16(>inode_type)) {
case SQFS_DIR_TYPE:
diff --git a/fs/squashfs/sqfs_inode.c b/fs/squashfs/sqfs_inode.c
index 1387779a85..1368f3063c 100644
--- a/fs/squashfs/sqfs_inode.c
+++ b/fs/squashfs/sqfs_inode.c
@@ -142,8 +142,11 @@ int sqfs_read_metablock(unsigned char *file_mapping, int
offset,
u16 header;
data = file_mapping + offset;
+ if (!data)
+ return -EFAULT;
+
header = get_unaligned((u16 *)data);
- if (!header || !data)
+ if (!header)
return -EINVAL;
*compressed = SQFS_COMPRESSED_METADATA(header);
--
2.17.1
[PATCH 1/1] fs/squashfs: Fix Coverity Scan defects
Fix defects such as uninitialized variables and untrusted pointer
operations. Most part of the tainted variables and the related defects
actually comes from Linux's macro get_unaligned_le**, extensively used
in SquashFS code. Add sanity checks for those variables.
Signed-off-by: Joao Marcos Costa
---
fs/squashfs/sqfs.c | 40 +---
fs/squashfs/sqfs_dir.c | 3 +++
fs/squashfs/sqfs_inode.c | 5 -
3 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 9bd7b98d88..f67f7c4a40 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -154,6 +154,11 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
header = get_unaligned_le16(metadata_buffer + table_offset);
metadata = metadata_buffer + table_offset + SQFS_HEADER_SIZE;
+ if (!metadata) {
+ ret = -ENOMEM;
+ goto free_buffer;
+ }
+
entries = malloc(SQFS_METADATA_BLOCK_SIZE);
if (!entries) {
ret = -ENOMEM;
@@ -272,8 +277,8 @@ static int sqfs_join(char **strings, char *dest, int start,
int end,
*/
static int sqfs_tokenize(char **tokens, int count, const char *str)
{
+ int i, j, ret = 0;
char *aux, *strc;
- int i, j;
strc = strdup(str);
if (!strc)
@@ -282,8 +287,8 @@ static int sqfs_tokenize(char **tokens, int count, const
char *str)
if (!strcmp(strc, "/")) {
tokens[0] = strdup(strc);
if (!tokens[0]) {
- free(strc);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto free_strc;
}
} else {
for (j = 0; j < count; j++) {
@@ -292,15 +297,16 @@ static int sqfs_tokenize(char **tokens, int count, const
char *str)
if (!tokens[j]) {
for (i = 0; i < j; i++)
free(tokens[i]);
- free(strc);
- return -ENOMEM;
+ ret = -ENOMEM;
+ goto free_strc;
}
}
}
+free_strc:
free(strc);
- return 0;
+ return ret;
}
/*
@@ -428,9 +434,9 @@ static int sqfs_search_dir(struct squashfs_dir_stream
*dirs, char **token_list,
{
struct squashfs_super_block *sblk = ctxt.sblk;
char *path, *target, **sym_tokens, *res, *rem;
+ struct squashfs_ldir_inode *ldir = NULL;
int j, ret, new_inode_number, offset;
struct squashfs_symlink_inode *sym;
- struct squashfs_ldir_inode *ldir;
struct squashfs_dir_inode *dir;
struct fs_dir_stream *dirsp;
struct fs_dirent *dent;
@@ -630,7 +636,7 @@ static int sqfs_read_inode_table(unsigned char
**inode_table)
int j, ret = 0, metablks_count;
unsigned char *src_table, *itb;
u32 src_len, dest_offset = 0;
- unsigned long dest_len;
+ unsigned long dest_len = 0;
bool compressed;
table_size = get_unaligned_le64(>directory_table_start) -
@@ -685,6 +691,7 @@ static int sqfs_read_inode_table(unsigned char
**inode_table)
goto free_itb;
}
+ dest_offset += dest_len;
} else {
memcpy(*inode_table + (j * SQFS_METADATA_BLOCK_SIZE),
src_table, src_len);
@@ -694,7 +701,7 @@ static int sqfs_read_inode_table(unsigned char
**inode_table)
* Offsets to the decompression destination, to the metadata
* buffer 'itb' and to the decompression source, respectively.
*/
- dest_offset += dest_len;
+
table_offset += src_len + SQFS_HEADER_SIZE;
src_table += src_len + SQFS_HEADER_SIZE;
}
@@ -712,7 +719,7 @@ static int sqfs_read_directory_table(unsigned char
**dir_table, u32 **pos_list)
int j, ret = 0, metablks_count = -1;
unsigned char *src_table, *dtb;
u32 src_len, dest_offset = 0;
- unsigned long dest_len;
+ unsigned long dest_len = 0;
bool compressed;
/* DIRECTORY TABLE */
@@ -781,6 +788,8 @@ static int sqfs_read_directory_table(unsigned char
**dir_table, u32 **pos_list)
dest_offset += dest_len;
break;
}
+
+ dest_offset += dest_len;
} else {
memcpy(*dir_table + (j * SQFS_METADATA_BLOCK_SIZE),
src_table, src_len);
@@ -790,7 +799,6 @@ static int sqfs_read_directory_table(unsigned char
**dir_table, u32 **pos_list)
* Offsets to the decompression destination, to the metadata
[PATCH 0/1] fs/squashfs: Fix Coverity Scan defects
Hello, The following patch fixes most of the defects pointed by Coverity Scan. It depends on the previous patch series adding new decompression algorithms. Some of the defects may be out of SquashFS scope, because they come from a Linux's macro, but sanity checks were added as a possible solution for these tainted values. Please do not hesitate to report any remaining issues. Best regards, Joao Joao Marcos Costa (1): fs/squashfs: Fix Coverity Scan defects fs/squashfs/sqfs.c | 40 +--- fs/squashfs/sqfs_dir.c | 3 +++ fs/squashfs/sqfs_inode.c | 5 - 3 files changed, 36 insertions(+), 12 deletions(-) -- 2.17.1
[PATCH v4 5/5] test/py: Add tests for LZO and ZSTD
Improve SquashFS tests architecture. Add 'Compression' class. LZO
algorithm may crash if the file is fragmented, so the fragments are
disabled when testing LZO.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- No changes since the last version.
Changes in v3:
- No changes since the last version.
Changes in v2:
- This patch was not present in the previous version.
.../test_fs/test_squashfs/sqfs_common.py | 76 ++-
.../test_fs/test_squashfs/test_sqfs_load.py | 40 ++
.../test_fs/test_squashfs/test_sqfs_ls.py | 33 +---
3 files changed, 102 insertions(+), 47 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index 2dc344d1b2..c96f92c1d8 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -5,6 +5,7 @@
import os
import random
import string
+import subprocess
def sqfs_get_random_letters(size):
letters = []
@@ -19,24 +20,57 @@ def sqfs_generate_file(path, size):
file.write(content)
file.close()
-# generate image with three files and a symbolic link
-def sqfs_generate_image(cons):
-src = os.path.join(cons.config.build_dir, "sqfs_src/")
-dest = os.path.join(cons.config.build_dir, "sqfs")
-os.mkdir(src)
-sqfs_generate_file(src + "frag_only", 100)
-sqfs_generate_file(src + "blks_frag", 5100)
-sqfs_generate_file(src + "blks_only", 4096)
-os.symlink("frag_only", src + "sym")
-os.system("mksquashfs " + src + " " + dest + " -b 4096
-always-use-fragments")
-
-# removes all files created by sqfs_generate_image()
-def sqfs_clean(cons):
-src = os.path.join(cons.config.build_dir, "sqfs_src/")
-dest = os.path.join(cons.config.build_dir, "sqfs")
-os.remove(src + "frag_only")
-os.remove(src + "blks_frag")
-os.remove(src + "blks_only")
-os.remove(src + "sym")
-os.rmdir(src)
-os.remove(dest)
+class Compression:
+def __init__(self, name, files, sizes, block_size = 4096):
+self.name = name
+self.files = files
+self.sizes = sizes
+self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
+
+def add_opt(self, opt):
+self.mksquashfs_opts += " " + opt
+
+def gen_image(self, build_dir):
+src = os.path.join(build_dir, "sqfs_src/")
+os.mkdir(src)
+for (f, s) in zip(self.files, self.sizes):
+sqfs_generate_file(src + f, s)
+
+# the symbolic link always targets the first file
+os.symlink(self.files[0], src + "sym")
+
+sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+i_o = src + " " + sqfs_img
+opts = self.mksquashfs_opts
+try:
+subprocess.run(["mksquashfs " + i_o + opts], shell = True, check =
True)
+except:
+print("mksquashfs error. Compression type: " + self.name)
+raise RuntimeError
+
+def clean_source(self, build_dir):
+src = os.path.join(build_dir, "sqfs_src/")
+for f in self.files:
+os.remove(src + f)
+os.remove(src + "sym")
+os.rmdir(src)
+
+def cleanup(self, build_dir):
+self.clean_source(build_dir)
+sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+os.remove(sqfs_img)
+
+files = ["blks_only", "blks_frag", "frag_only"]
+sizes = [4096, 5100, 100]
+gzip = Compression("gzip", files, sizes)
+zstd = Compression("zstd", files, sizes)
+lzo = Compression("lzo", files, sizes)
+
+# use fragment blocks for files larger than block_size
+gzip.add_opt("-always-use-fragments")
+zstd.add_opt("-always-use-fragments")
+
+# avoid fragments if lzo is used
+lzo.add_opt("-no-fragments")
+
+comp_opts = [gzip, zstd, lzo]
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index eb1baae5c5..9e90062384 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -12,23 +12,35 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_load(u_boot_console):
-cons = u_boot_console
-sqfs_generate_image(cons)
+build_dir = u_boot_console.config.build_dir
command = "sqfsload host 0 $kernel_addr_r "
-path = os.path.join(cons.config.build_dir, "sqfs")
-try:
+for opt in comp_opts:
+# generate and load the squashfs image
+try:
+opt
[PATCH v4 3/5] fs/squashfs: add support for ZSTD decompression
Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller
can upper bound the decompressed size, which will be the SquashFS data
block (or metadata block) size, so there is no need to use streaming
API. Add ZSTD's worskpace to squashfs_ctxt structure.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- Add parameter to sqfs_zstd_decompress().
Changes in v3:
- Remove memory leak from sqfs_zstd_decompress().
Changes in v2:
- No changes since last version.
fs/squashfs/sqfs_decompressor.c | 42 +
fs/squashfs/sqfs_filesystem.h | 3 +++
2 files changed, 45 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index cf388ace58..9457ee5f0f 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -13,6 +13,10 @@
#include
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+#include
+#endif
+
#include "sqfs_decompressor.h"
#include "sqfs_utils.h"
@@ -24,6 +28,13 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ctxt->zstd_workspace = malloc(ZSTD_DCtxWorkspaceBound());
+ if (!ctxt->zstd_workspace)
+ return -ENOMEM;
+ break;
#endif
default:
printf("Error: unknown compression type.\n");
@@ -41,6 +52,11 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ free(ctxt->zstd_workspace);
+ break;
#endif
}
}
@@ -62,6 +78,22 @@ static void zlib_decompression_status(int ret)
}
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+static int sqfs_zstd_decompress(struct squashfs_ctxt *ctxt, void *dest,
+ unsigned long dest_len, void *source, u32
src_len)
+{
+ ZSTD_DCtx *ctx;
+ size_t wsize;
+ int ret;
+
+ wsize = ZSTD_DCtxWorkspaceBound();
+ ctx = ZSTD_initDCtx(ctxt->zstd_workspace, wsize);
+ ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len);
+
+ return ZSTD_isError(ret);
+}
+#endif /* CONFIG_ZSTD */
+
int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
unsigned long *dest_len, void *source, u32 src_len)
{
@@ -79,6 +111,16 @@ int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
break;
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ret = sqfs_zstd_decompress(ctxt, dest, *dest_len, source,
src_len);
+ if (ret) {
+ printf("ZSTD Error code: %d\n", ZSTD_getErrorCode(ret));
+ return -EINVAL;
+ }
+
+ break;
+#endif
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h
index ff2b0b1d34..856cd15e34 100644
--- a/fs/squashfs/sqfs_filesystem.h
+++ b/fs/squashfs/sqfs_filesystem.h
@@ -77,6 +77,9 @@ struct squashfs_ctxt {
struct disk_partition cur_part_info;
struct blk_desc *cur_dev;
struct squashfs_super_block *sblk;
+#if IS_ENABLED(CONFIG_ZSTD)
+ void *zstd_workspace;
+#endif
};
struct squashfs_directory_index {
--
2.17.1
[PATCH v4 2/5] fs/squashfs: replace sqfs_decompress() parameter
Replace 'u16 comp_type' by a reference to squashfs_ctxt structure.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- This patch was not present in the previous version.
fs/squashfs/sqfs.c | 25 +
fs/squashfs/sqfs_decompressor.c | 5 +++--
fs/squashfs/sqfs_decompressor.h | 4 ++--
3 files changed, 14 insertions(+), 20 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 598b42cd34..9bd7b98d88 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -104,9 +104,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
struct squashfs_super_block *sblk = ctxt.sblk;
unsigned long dest_len;
int block, offset, ret;
- u16 header, comp_type;
-
- comp_type = get_unaligned_le16(>compression);
+ u16 header;
if (inode_fragment_index >= get_unaligned_le32(>fragments))
return -EINVAL;
@@ -165,7 +163,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
if (SQFS_COMPRESSED_METADATA(header)) {
src_len = SQFS_METADATA_SIZE(header);
dest_len = SQFS_METADATA_BLOCK_SIZE;
- ret = sqfs_decompress(comp_type, entries, _len, metadata,
+ ret = sqfs_decompress(, entries, _len, metadata,
src_len);
if (ret) {
ret = -EINVAL;
@@ -629,13 +627,12 @@ static int sqfs_read_inode_table(unsigned char
**inode_table)
{
struct squashfs_super_block *sblk = ctxt.sblk;
u64 start, n_blks, table_offset, table_size;
- int j, ret = 0, metablks_count, comp_type;
+ int j, ret = 0, metablks_count;
unsigned char *src_table, *itb;
u32 src_len, dest_offset = 0;
unsigned long dest_len;
bool compressed;
- comp_type = get_unaligned_le16(>compression);
table_size = get_unaligned_le64(>directory_table_start) -
get_unaligned_le64(>inode_table_start);
start = get_unaligned_le64(>inode_table_start) /
@@ -680,7 +677,7 @@ static int sqfs_read_inode_table(unsigned char
**inode_table)
sqfs_read_metablock(itb, table_offset, , _len);
if (compressed) {
dest_len = SQFS_METADATA_BLOCK_SIZE;
- ret = sqfs_decompress(comp_type, *inode_table +
+ ret = sqfs_decompress(, *inode_table +
dest_offset, _len,
src_table, src_len);
if (ret) {
@@ -711,15 +708,13 @@ free_itb:
static int sqfs_read_directory_table(unsigned char **dir_table, u32 **pos_list)
{
u64 start, n_blks, table_offset, table_size;
- int j, ret = 0, metablks_count = -1, comp_type;
struct squashfs_super_block *sblk = ctxt.sblk;
+ int j, ret = 0, metablks_count = -1;
unsigned char *src_table, *dtb;
u32 src_len, dest_offset = 0;
unsigned long dest_len;
bool compressed;
- comp_type = get_unaligned_le16(>compression);
-
/* DIRECTORY TABLE */
table_size = get_unaligned_le64(>fragment_table_start) -
get_unaligned_le64(>directory_table_start);
@@ -773,7 +768,7 @@ static int sqfs_read_directory_table(unsigned char
**dir_table, u32 **pos_list)
sqfs_read_metablock(dtb, table_offset, , _len);
if (compressed) {
dest_len = SQFS_METADATA_BLOCK_SIZE;
- ret = sqfs_decompress(comp_type, *dir_table +
+ ret = sqfs_decompress(, *dir_table +
(j * SQFS_METADATA_BLOCK_SIZE),
_len, src_table, src_len);
if (ret) {
@@ -1197,7 +1192,7 @@ int sqfs_read(const char *filename, void *buf, loff_t
offset, loff_t len,
char *dir, *fragment_block, *datablock = NULL, *data_buffer = NULL;
char *fragment, *file, *resolved, *data;
u64 start, n_blks, table_size, data_offset, table_offset;
- int ret, j, i_number, comp_type, datablk_count = 0;
+ int ret, j, i_number, datablk_count = 0;
struct squashfs_super_block *sblk = ctxt.sblk;
struct squashfs_fragment_block_entry frag_entry;
struct squashfs_file_info finfo = {0};
@@ -1213,8 +1208,6 @@ int sqfs_read(const char *filename, void *buf, loff_t
offset, loff_t len,
*actread = 0;
- comp_type = get_unaligned_le16(>compression);
-
/*
* sqfs_opendir will uncompress inode and directory tables, and will
* return a pointer to the directory that contains the requested file.
@@ -1346,7 +1339,7 @@ int sqfs_read(const char *filename, void *buf, loff_t
offset, loff_t len,
/* Load the data */
if (SQFS_COMPRESSED_BLOCK(finfo.blk_sizes[j])) {
[PATCH v4 1/5] fs/squashfs: Add init and clean-up functions to decompression
Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These
functions are called respectively in sqfs_probe() and sqfs_close(). For
now, only ZSTD requires an initialization logic. ZSTD support will be
added in a follow-up commit.
Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is
passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so
it can no longer be local to sqfs.c.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- Undo changes from v3 in squashfs_ctxt instance scope.
Changes in v3:
- Remove memory leak from sqfs_zstd_decompress()
- Rename structure in sqfs.c and remove the 'static' so it can be
exported to sqfs_decompressor.c
Changes in v2:
- This patch was not present in the previous version.
fs/squashfs/sqfs.c | 15 +--
fs/squashfs/sqfs_decompressor.c | 34 ++---
fs/squashfs/sqfs_decompressor.h | 3 +++
fs/squashfs/sqfs_filesystem.h | 9 -
4 files changed, 51 insertions(+), 10 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 340e5ebdb9..598b42cd34 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -23,12 +23,6 @@
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
-struct squashfs_ctxt {
- struct disk_partition cur_part_info;
- struct blk_desc *cur_dev;
- struct squashfs_super_block *sblk;
-};
-
static struct squashfs_ctxt ctxt;
static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void *buf)
@@ -1023,6 +1017,14 @@ int sqfs_probe(struct blk_desc *fs_dev_desc, struct
disk_partition *fs_partition
ctxt.sblk = sblk;
+ ret = sqfs_decompressor_init();
+
+ if (ret) {
+ ctxt.cur_dev = NULL;
+ free(ctxt.sblk);
+ return -EINVAL;
+ }
+
return 0;
}
@@ -1525,6 +1527,7 @@ void sqfs_close(void)
{
free(ctxt.sblk);
ctxt.cur_dev = NULL;
+ sqfs_decompressor_cleanup();
}
void sqfs_closedir(struct fs_dir_stream *dirs)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 09ca6cf6d0..633a8a1818 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -14,9 +14,37 @@
#endif
#include "sqfs_decompressor.h"
-#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
+int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
+{
+ u16 comp_type = get_unaligned_le16(>sblk->compression);
+
+ switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+ case SQFS_COMP_ZLIB:
+ break;
+#endif
+ default:
+ printf("Error: unknown compression type.\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
+{
+ u16 comp_type = get_unaligned_le16(>sblk->compression);
+
+ switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+ case SQFS_COMP_ZLIB:
+ break;
+#endif
+ }
+}
+
#if IS_ENABLED(CONFIG_ZLIB)
static void zlib_decompression_status(int ret)
{
@@ -35,14 +63,14 @@ static void zlib_decompression_status(int ret)
#endif
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
- void *source, u32 lenp)
+ void *source, u32 src_len)
{
int ret = 0;
switch (comp_type) {
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
- ret = uncompress(dest, dest_len, source, lenp);
+ ret = uncompress(dest, dest_len, source, src_len);
if (ret) {
zlib_decompression_status(ret);
return -EINVAL;
diff --git a/fs/squashfs/sqfs_decompressor.h b/fs/squashfs/sqfs_decompressor.h
index 378965dda8..450257e5ce 100644
--- a/fs/squashfs/sqfs_decompressor.h
+++ b/fs/squashfs/sqfs_decompressor.h
@@ -9,6 +9,7 @@
#define SQFS_DECOMPRESSOR_H
#include
+#include "sqfs_filesystem.h"
#define SQFS_COMP_ZLIB 1
#define SQFS_COMP_LZMA 2
@@ -54,5 +55,7 @@ union squashfs_compression_opts {
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 lenp);
+int sqfs_decompressor_init(struct squashfs_ctxt *ctxt);
+void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt);
#endif /* SQFS_DECOMPRESSOR_H */
diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h
index d63e3a41ad..ff2b0b1d34 100644
--- a/fs/squashfs/sqfs_filesystem.h
+++ b/fs/squashfs/sqfs_filesystem.h
@@ -9,8 +9,9 @@
#define SQFS_FILESYSTEM_H
#include
-#include
#include
+#include
+#include
#define SQFS_UNCOMPRESSED_DATA 0x0002
#define SQFS_MAGIC_NUMBER 0x73717368
@@ -72,6 +73,12 @@ struct squashfs_super_block {
__le64 export_table_start;
};
+struct squashfs_ctxt {
+ struct disk_partition cur_part_info;
+
[PATCH v4 4/5] fs/squashfs: add support for LZO decompression
Add call to lzo's lzo1x_decompress_safe() into sqfs_decompress().
U-Boot's LZO sources may still have some unsolved issues that could make the
decompression crash when dealing with fragmented files, so those should be
avoided. The "-no-fragments" option can be passed to mksquashfs.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- No changes since the last version.
Changes in v3:
- Change commit message.
Changes in v2:
- Change commit message.
fs/squashfs/sqfs_decompressor.c | 25 +
1 file changed, 25 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 9457ee5f0f..d69ddb24a4 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,6 +9,11 @@
#include
#include
#include
+
+#if IS_ENABLED(CONFIG_LZO)
+#include
+#endif
+
#if IS_ENABLED(CONFIG_ZLIB)
#include
#endif
@@ -25,6 +30,10 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
u16 comp_type = get_unaligned_le16(>sblk->compression);
switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO:
+ break;
+#endif
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
@@ -49,6 +58,10 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
u16 comp_type = get_unaligned_le16(>sblk->compression);
switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO:
+ break;
+#endif
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
@@ -101,6 +114,18 @@ int sqfs_decompress(struct squashfs_ctxt *ctxt, void *dest,
int ret = 0;
switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO: {
+ size_t lzo_dest_len = *dest_len;
+ ret = lzo1x_decompress_safe(source, src_len, dest,
_dest_len);
+ if (ret) {
+ printf("LZO decompression failed. Error code: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ break;
+ }
+#endif
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
ret = uncompress(dest, dest_len, source, src_len);
--
2.17.1
[PATCH v4 0/5] fs/squashfs: Add new decompression algorithms
Hello, Following the SquashFS support, this series adds support for LZO and ZSTD algorithms. Changes in v4: - Replace sqfs_decompress() 1st parameter by a reference to squashfs_ctxt. - Undo changes in squashfs_ctxt instance at sqfs.c Changes in v3: - Reorganize patches order. - Remove memory leak from sqfs_zstd_decompress(). Changes in v2: - Change tests architecture. - Add tests for LZO and ZSTD. - Skip compression algorithms unsupported by the host's squashfs-tools version. - Add sqfs_decompress_init() and sqfs_decompress_cleanup() to SquashFS support. Joao Marcos Costa (5): fs/squashfs: Add init and clean-up functions to decompression fs/squashfs: replace sqfs_decompress() parameter fs/squashfs: add support for ZSTD decompression fs/squashfs: add support for LZO decompression test/py: Add tests for LZO and ZSTD fs/squashfs/sqfs.c| 40 +++ fs/squashfs/sqfs_decompressor.c | 104 +- fs/squashfs/sqfs_decompressor.h | 7 +- fs/squashfs/sqfs_filesystem.h | 12 +- .../test_fs/test_squashfs/sqfs_common.py | 76 + .../test_fs/test_squashfs/test_sqfs_load.py | 40 --- .../test_fs/test_squashfs/test_sqfs_ls.py | 33 -- 7 files changed, 236 insertions(+), 76 deletions(-) -- 2.17.1
Re: [PATCH v3 2/4] fs/squashfs: add support for ZSTD decompression
On Tue, 18 Aug 2020 16:35:27 +0200 Thomas Petazzoni wrote: > Hello, > > On Tue, 18 Aug 2020 16:31:16 +0200 > Joao Marcos Costa wrote: > > > +#if IS_ENABLED(CONFIG_ZSTD) > > +static int sqfs_zstd_decompress(void *dest, unsigned long dest_len, > > + void *source, u32 src_len) > > Pass the squashfs_context structure as argument to the decompressor > function, so that sqfs_ctxt doesn't need to be a global variable. > > Best regards, > > Thomas Thanks for the remarks, I am preparing a v4 and I will send it in a few minutes. Best regards, Joao
[PATCH v3 0/4] fs/squashfs: Add new decompression algorithms
Hello, Following the SquashFS support, this series adds support for LZO and ZSTD algorithms. Changes in v3: - Reorganize patches order. - Remove memory leak from sqfs_zstd_decompress(). Changes in v2: - Change tests architecture. - Add tests for LZO and ZSTD. - Skip compression algorithms unsupported by the host's squashfs-tools version. - Add sqfs_decompress_init() and sqfs_decompress_cleanup() to SquashFS support. Joao Marcos Costa (4): fs/squashfs: Add init. and clean-up functions to decompression fs/squashfs: add support for ZSTD decompression fs/squashfs: add support for LZO decompression test/py: Add tests for LZO and ZSTD fs/squashfs/sqfs.c| 87 --- fs/squashfs/sqfs_decompressor.c | 103 +- fs/squashfs/sqfs_decompressor.h | 3 + fs/squashfs/sqfs_filesystem.h | 12 +- .../test_fs/test_squashfs/sqfs_common.py | 76 + .../test_fs/test_squashfs/test_sqfs_load.py | 40 --- .../test_fs/test_squashfs/test_sqfs_ls.py | 33 -- 7 files changed, 261 insertions(+), 93 deletions(-) -- 2.17.1
[PATCH v3 1/4] fs/squashfs: Add init. and clean-up functions to decompression
Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These
functions are called respectively in sqfs_probe() and sqfs_close(). For
now, only ZSTD requires an initialization logic. ZSTD support will be
added in a follow-up commit.
Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is
passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so
it can no longer be local to sqfs.c.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- Remove memory leak from sqfs_zstd_decompress()
- Rename structure in sqfs.c and remove the 'static' so it can be
exported to sqfs_decompressor.c
Changes in v2:
- This patch was not present in the previous version.
fs/squashfs/sqfs.c | 87 +
fs/squashfs/sqfs_decompressor.c | 36 --
fs/squashfs/sqfs_decompressor.h | 3 ++
fs/squashfs/sqfs_filesystem.h | 9 +++-
4 files changed, 89 insertions(+), 46 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 340e5ebdb9..2836b3de8b 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -23,22 +23,16 @@
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
-struct squashfs_ctxt {
- struct disk_partition cur_part_info;
- struct blk_desc *cur_dev;
- struct squashfs_super_block *sblk;
-};
-
-static struct squashfs_ctxt ctxt;
+struct squashfs_ctxt sqfs_ctxt;
static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void *buf)
{
ulong ret;
- if (!ctxt.cur_dev)
+ if (!sqfs_ctxt.cur_dev)
return -1;
- ret = blk_dread(ctxt.cur_dev, ctxt.cur_part_info.start + block,
+ ret = blk_dread(sqfs_ctxt.cur_dev, sqfs_ctxt.cur_part_info.start +
block,
nr_blocks, buf);
if (ret != nr_blocks)
@@ -49,7 +43,7 @@ static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void
*buf)
static int sqfs_read_sblk(struct squashfs_super_block **sblk)
{
- *sblk = malloc_cache_aligned(ctxt.cur_dev->blksz);
+ *sblk = malloc_cache_aligned(sqfs_ctxt.cur_dev->blksz);
if (!*sblk)
return -ENOMEM;
@@ -91,10 +85,10 @@ static int sqfs_calc_n_blks(__le64 start, __le64 end, u64
*offset)
u64 start_, table_size;
table_size = le64_to_cpu(end) - le64_to_cpu(start);
- start_ = le64_to_cpu(start) / ctxt.cur_dev->blksz;
- *offset = le64_to_cpu(start) - (start_ * ctxt.cur_dev->blksz);
+ start_ = le64_to_cpu(start) / sqfs_ctxt.cur_dev->blksz;
+ *offset = le64_to_cpu(start) - (start_ * sqfs_ctxt.cur_dev->blksz);
- return DIV_ROUND_UP(table_size + *offset, ctxt.cur_dev->blksz);
+ return DIV_ROUND_UP(table_size + *offset, sqfs_ctxt.cur_dev->blksz);
}
/*
@@ -107,7 +101,7 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
u64 start, n_blks, src_len, table_offset, start_block;
unsigned char *metadata_buffer, *metadata, *table;
struct squashfs_fragment_block_entry *entries;
- struct squashfs_super_block *sblk = ctxt.sblk;
+ struct squashfs_super_block *sblk = sqfs_ctxt.sblk;
unsigned long dest_len;
int block, offset, ret;
u16 header, comp_type;
@@ -118,13 +112,13 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
return -EINVAL;
start = get_unaligned_le64(>fragment_table_start) /
- ctxt.cur_dev->blksz;
+ sqfs_ctxt.cur_dev->blksz;
n_blks = sqfs_calc_n_blks(sblk->fragment_table_start,
sblk->export_table_start,
_offset);
/* Allocate a proper sized buffer to store the fragment index table */
- table = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz);
+ table = malloc_cache_aligned(n_blks * sqfs_ctxt.cur_dev->blksz);
if (!table)
return -ENOMEM;
@@ -143,11 +137,11 @@ static int sqfs_frag_lookup(u32 inode_fragment_index,
start_block = get_unaligned_le64(table + table_offset + block *
sizeof(u64));
- start = start_block / ctxt.cur_dev->blksz;
+ start = start_block / sqfs_ctxt.cur_dev->blksz;
n_blks = sqfs_calc_n_blks(cpu_to_le64(start_block),
sblk->fragment_table_start, _offset);
- metadata_buffer = malloc_cache_aligned(n_blks * ctxt.cur_dev->blksz);
+ metadata_buffer = malloc_cache_aligned(n_blks *
sqfs_ctxt.cur_dev->blksz);
if (!metadata_buffer) {
ret = -ENOMEM;
goto free_table;
@@ -434,7 +428,7 @@ static char *sqfs_resolve_symlink(struct
squashfs_symlink_inode *sym,
static int sqfs_search_dir(struct squashfs_dir_stream *dirs, char **token_list,
int token_count, u32 *m_list, int m_count)
{
- struct squashfs_super_blo
[PATCH v3 2/4] fs/squashfs: add support for ZSTD decompression
Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller
can upper bound the decompressed size, which will be the SquashFS data
block (or metadata block) size, so there is no need to use streaming
API. Add ZSTD's worskpace to squashfs_ctxt structure.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- Remove memory leak from sqfs_zstd_decompress()
Changes in v2:
- No changes since last version.
fs/squashfs/sqfs_decompressor.c | 42 +
fs/squashfs/sqfs_filesystem.h | 3 +++
2 files changed, 45 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 577cffd8bb..8fdaaa853c 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -13,6 +13,10 @@
#include
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+#include
+#endif
+
#include "sqfs_decompressor.h"
#include "sqfs_utils.h"
@@ -26,6 +30,13 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ctxt->zstd_workspace = malloc(ZSTD_DCtxWorkspaceBound());
+ if (!ctxt->zstd_workspace)
+ return -ENOMEM;
+ break;
#endif
default:
printf("Error: unknown compression type.\n");
@@ -43,6 +54,11 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ free(ctxt->zstd_workspace);
+ break;
#endif
}
}
@@ -64,6 +80,22 @@ static void zlib_decompression_status(int ret)
}
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+static int sqfs_zstd_decompress(void *dest, unsigned long dest_len,
+ void *source, u32 src_len)
+{
+ ZSTD_DCtx *ctx;
+ size_t wsize;
+ int ret;
+
+ wsize = ZSTD_DCtxWorkspaceBound();
+ ctx = ZSTD_initDCtx(sqfs_ctxt.zstd_workspace, wsize);
+ ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len);
+
+ return ZSTD_isError(ret);
+}
+#endif /* CONFIG_ZSTD */
+
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 src_len)
{
@@ -80,6 +112,16 @@ int sqfs_decompress(u16 comp_type, void *dest, unsigned
long *dest_len,
break;
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ret = sqfs_zstd_decompress(dest, *dest_len, source, src_len);
+ if (ret) {
+ printf("ZSTD Error code: %d\n", ZSTD_getErrorCode(ret));
+ return -EINVAL;
+ }
+
+ break;
+#endif
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h
index ff2b0b1d34..856cd15e34 100644
--- a/fs/squashfs/sqfs_filesystem.h
+++ b/fs/squashfs/sqfs_filesystem.h
@@ -77,6 +77,9 @@ struct squashfs_ctxt {
struct disk_partition cur_part_info;
struct blk_desc *cur_dev;
struct squashfs_super_block *sblk;
+#if IS_ENABLED(CONFIG_ZSTD)
+ void *zstd_workspace;
+#endif
};
struct squashfs_directory_index {
--
2.17.1
[PATCH v3 4/4] test/py: Add tests for LZO and ZSTD
Improve SquashFS tests architecture. Add 'Compression' class. LZO
algorithm may crash if the file is fragmented, so the fragments are
disabled when testing LZO.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- No changes since the last version.
Changes in v2:
- This patch was not present in the previous version.
.../test_fs/test_squashfs/sqfs_common.py | 76 ++-
.../test_fs/test_squashfs/test_sqfs_load.py | 40 ++
.../test_fs/test_squashfs/test_sqfs_ls.py | 33 +---
3 files changed, 102 insertions(+), 47 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index 2dc344d1b2..c96f92c1d8 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -5,6 +5,7 @@
import os
import random
import string
+import subprocess
def sqfs_get_random_letters(size):
letters = []
@@ -19,24 +20,57 @@ def sqfs_generate_file(path, size):
file.write(content)
file.close()
-# generate image with three files and a symbolic link
-def sqfs_generate_image(cons):
-src = os.path.join(cons.config.build_dir, "sqfs_src/")
-dest = os.path.join(cons.config.build_dir, "sqfs")
-os.mkdir(src)
-sqfs_generate_file(src + "frag_only", 100)
-sqfs_generate_file(src + "blks_frag", 5100)
-sqfs_generate_file(src + "blks_only", 4096)
-os.symlink("frag_only", src + "sym")
-os.system("mksquashfs " + src + " " + dest + " -b 4096
-always-use-fragments")
-
-# removes all files created by sqfs_generate_image()
-def sqfs_clean(cons):
-src = os.path.join(cons.config.build_dir, "sqfs_src/")
-dest = os.path.join(cons.config.build_dir, "sqfs")
-os.remove(src + "frag_only")
-os.remove(src + "blks_frag")
-os.remove(src + "blks_only")
-os.remove(src + "sym")
-os.rmdir(src)
-os.remove(dest)
+class Compression:
+def __init__(self, name, files, sizes, block_size = 4096):
+self.name = name
+self.files = files
+self.sizes = sizes
+self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
+
+def add_opt(self, opt):
+self.mksquashfs_opts += " " + opt
+
+def gen_image(self, build_dir):
+src = os.path.join(build_dir, "sqfs_src/")
+os.mkdir(src)
+for (f, s) in zip(self.files, self.sizes):
+sqfs_generate_file(src + f, s)
+
+# the symbolic link always targets the first file
+os.symlink(self.files[0], src + "sym")
+
+sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+i_o = src + " " + sqfs_img
+opts = self.mksquashfs_opts
+try:
+subprocess.run(["mksquashfs " + i_o + opts], shell = True, check =
True)
+except:
+print("mksquashfs error. Compression type: " + self.name)
+raise RuntimeError
+
+def clean_source(self, build_dir):
+src = os.path.join(build_dir, "sqfs_src/")
+for f in self.files:
+os.remove(src + f)
+os.remove(src + "sym")
+os.rmdir(src)
+
+def cleanup(self, build_dir):
+self.clean_source(build_dir)
+sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+os.remove(sqfs_img)
+
+files = ["blks_only", "blks_frag", "frag_only"]
+sizes = [4096, 5100, 100]
+gzip = Compression("gzip", files, sizes)
+zstd = Compression("zstd", files, sizes)
+lzo = Compression("lzo", files, sizes)
+
+# use fragment blocks for files larger than block_size
+gzip.add_opt("-always-use-fragments")
+zstd.add_opt("-always-use-fragments")
+
+# avoid fragments if lzo is used
+lzo.add_opt("-no-fragments")
+
+comp_opts = [gzip, zstd, lzo]
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index eb1baae5c5..9e90062384 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -12,23 +12,35 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_load(u_boot_console):
-cons = u_boot_console
-sqfs_generate_image(cons)
+build_dir = u_boot_console.config.build_dir
command = "sqfsload host 0 $kernel_addr_r "
-path = os.path.join(cons.config.build_dir, "sqfs")
-try:
+for opt in comp_opts:
+# generate and load the squashfs image
+try:
+opt.gen_image(build_dir)
+except RuntimeError:
+
[PATCH v3 3/4] fs/squashfs: add support for LZO decompression
Add call to lzo's lzo1x_decompress_safe() into sqfs_decompress().
U-Boot's LZO sources may still have some unsolved issues that could make the
decompression crash when dealing with fragmented files, so those should be
avoided. The "-no-fragments" option can be passed to mksquashfs.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- Change commit message.
Changes in v2:
- Change commit message.
fs/squashfs/sqfs_decompressor.c | 25 +
1 file changed, 25 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 8fdaaa853c..f57d7e2626 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,6 +9,11 @@
#include
#include
#include
+
+#if IS_ENABLED(CONFIG_LZO)
+#include
+#endif
+
#if IS_ENABLED(CONFIG_ZLIB)
#include
#endif
@@ -27,6 +32,10 @@ int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
u16 comp_type = get_unaligned_le16(>sblk->compression);
switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO:
+ break;
+#endif
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
@@ -51,6 +60,10 @@ void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
u16 comp_type = get_unaligned_le16(>sblk->compression);
switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO:
+ break;
+#endif
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
break;
@@ -102,6 +115,18 @@ int sqfs_decompress(u16 comp_type, void *dest, unsigned
long *dest_len,
int ret = 0;
switch (comp_type) {
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO: {
+ size_t lzo_dest_len = *dest_len;
+ ret = lzo1x_decompress_safe(source, src_len, dest,
_dest_len);
+ if (ret) {
+ printf("LZO decompression failed. Error code: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ break;
+ }
+#endif
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
ret = uncompress(dest, dest_len, source, src_len);
--
2.17.1
[PATCH v2 4/4] test/py: Add tests for LZO and ZSTD
Improve SquashFS tests architecture. Add 'Compression' class. LZO
algorithm may crash if the file is fragmented, so the fragments are
disabled when testing LZO.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- This patch was not present in the previous version.
.../test_fs/test_squashfs/sqfs_common.py | 76 ++-
.../test_fs/test_squashfs/test_sqfs_load.py | 40 ++
.../test_fs/test_squashfs/test_sqfs_ls.py | 33 +---
3 files changed, 102 insertions(+), 47 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index 2dc344d1b2..c96f92c1d8 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -5,6 +5,7 @@
import os
import random
import string
+import subprocess
def sqfs_get_random_letters(size):
letters = []
@@ -19,24 +20,57 @@ def sqfs_generate_file(path, size):
file.write(content)
file.close()
-# generate image with three files and a symbolic link
-def sqfs_generate_image(cons):
-src = os.path.join(cons.config.build_dir, "sqfs_src/")
-dest = os.path.join(cons.config.build_dir, "sqfs")
-os.mkdir(src)
-sqfs_generate_file(src + "frag_only", 100)
-sqfs_generate_file(src + "blks_frag", 5100)
-sqfs_generate_file(src + "blks_only", 4096)
-os.symlink("frag_only", src + "sym")
-os.system("mksquashfs " + src + " " + dest + " -b 4096
-always-use-fragments")
-
-# removes all files created by sqfs_generate_image()
-def sqfs_clean(cons):
-src = os.path.join(cons.config.build_dir, "sqfs_src/")
-dest = os.path.join(cons.config.build_dir, "sqfs")
-os.remove(src + "frag_only")
-os.remove(src + "blks_frag")
-os.remove(src + "blks_only")
-os.remove(src + "sym")
-os.rmdir(src)
-os.remove(dest)
+class Compression:
+def __init__(self, name, files, sizes, block_size = 4096):
+self.name = name
+self.files = files
+self.sizes = sizes
+self.mksquashfs_opts = " -b " + str(block_size) + " -comp " + self.name
+
+def add_opt(self, opt):
+self.mksquashfs_opts += " " + opt
+
+def gen_image(self, build_dir):
+src = os.path.join(build_dir, "sqfs_src/")
+os.mkdir(src)
+for (f, s) in zip(self.files, self.sizes):
+sqfs_generate_file(src + f, s)
+
+# the symbolic link always targets the first file
+os.symlink(self.files[0], src + "sym")
+
+sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+i_o = src + " " + sqfs_img
+opts = self.mksquashfs_opts
+try:
+subprocess.run(["mksquashfs " + i_o + opts], shell = True, check =
True)
+except:
+print("mksquashfs error. Compression type: " + self.name)
+raise RuntimeError
+
+def clean_source(self, build_dir):
+src = os.path.join(build_dir, "sqfs_src/")
+for f in self.files:
+os.remove(src + f)
+os.remove(src + "sym")
+os.rmdir(src)
+
+def cleanup(self, build_dir):
+self.clean_source(build_dir)
+sqfs_img = os.path.join(build_dir, "sqfs-" + self.name)
+os.remove(sqfs_img)
+
+files = ["blks_only", "blks_frag", "frag_only"]
+sizes = [4096, 5100, 100]
+gzip = Compression("gzip", files, sizes)
+zstd = Compression("zstd", files, sizes)
+lzo = Compression("lzo", files, sizes)
+
+# use fragment blocks for files larger than block_size
+gzip.add_opt("-always-use-fragments")
+zstd.add_opt("-always-use-fragments")
+
+# avoid fragments if lzo is used
+lzo.add_opt("-no-fragments")
+
+comp_opts = [gzip, zstd, lzo]
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index eb1baae5c5..9e90062384 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -12,23 +12,35 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_load(u_boot_console):
-cons = u_boot_console
-sqfs_generate_image(cons)
+build_dir = u_boot_console.config.build_dir
command = "sqfsload host 0 $kernel_addr_r "
-path = os.path.join(cons.config.build_dir, "sqfs")
-try:
+for opt in comp_opts:
+# generate and load the squashfs image
+try:
+opt.gen_image(build_dir)
+except RuntimeError:
+opt.clean_source(build_dir)
+# skip
[PATCH v2 3/4] fs/squashfs: Add init. and clean-up functions to decompression
Add sqfs_decompressor_init() and sqfs_decompressor_cleanup(). These
functions are called respectively in sqfs_probe() and sqfs_close(). For
now, only ZSTD requires an initialization logic.
Move squashfs_ctxt definition to sqfs_filesystem.h. This structure is
passed to sqfs_decompressor_init() and sqfs_decompressor_cleanup(), so
it can no longer be local to sqfs.c.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- This patch was not present in the previous version.
fs/squashfs/sqfs.c | 15 ++
fs/squashfs/sqfs_decompressor.c | 52 +++--
fs/squashfs/sqfs_decompressor.h | 4 +++
fs/squashfs/sqfs_filesystem.h | 12 +++-
4 files changed, 73 insertions(+), 10 deletions(-)
diff --git a/fs/squashfs/sqfs.c b/fs/squashfs/sqfs.c
index 340e5ebdb9..598b42cd34 100644
--- a/fs/squashfs/sqfs.c
+++ b/fs/squashfs/sqfs.c
@@ -23,12 +23,6 @@
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
-struct squashfs_ctxt {
- struct disk_partition cur_part_info;
- struct blk_desc *cur_dev;
- struct squashfs_super_block *sblk;
-};
-
static struct squashfs_ctxt ctxt;
static int sqfs_disk_read(__u32 block, __u32 nr_blocks, void *buf)
@@ -1023,6 +1017,14 @@ int sqfs_probe(struct blk_desc *fs_dev_desc, struct
disk_partition *fs_partition
ctxt.sblk = sblk;
+ ret = sqfs_decompressor_init();
+
+ if (ret) {
+ ctxt.cur_dev = NULL;
+ free(ctxt.sblk);
+ return -EINVAL;
+ }
+
return 0;
}
@@ -1525,6 +1527,7 @@ void sqfs_close(void)
{
free(ctxt.sblk);
ctxt.cur_dev = NULL;
+ sqfs_decompressor_cleanup();
}
void sqfs_closedir(struct fs_dir_stream *dirs)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index b5a9d92808..e536d85055 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -23,9 +23,57 @@
#endif
#include "sqfs_decompressor.h"
-#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
+int sqfs_decompressor_init(struct squashfs_ctxt *ctxt)
+{
+ u16 comp_type = get_unaligned_le16(>sblk->compression);
+
+ switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+ case SQFS_COMP_ZLIB:
+ break;
+#endif
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO:
+ break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ctxt->zstd_workspace = malloc(ZSTD_DCtxWorkspaceBound());
+ if (!ctxt->zstd_workspace)
+ return -ENOMEM;
+ break;
+#endif
+ default:
+ printf("Error: unknown compression type.\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt)
+{
+ u16 comp_type = get_unaligned_le16(>sblk->compression);
+
+ switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+ case SQFS_COMP_ZLIB:
+ break;
+#endif
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO:
+ break;
+#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ free(ctxt->zstd_workspace);
+ break;
+#endif
+ }
+}
+
#if IS_ENABLED(CONFIG_ZLIB)
static void zlib_decompression_status(int ret)
{
@@ -62,8 +110,6 @@ static int sqfs_zstd_decompress(void *dest, unsigned long
dest_len,
ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len);
- free(workspace);
-
return ZSTD_isError(ret);
}
#endif /* CONFIG_ZSTD */
diff --git a/fs/squashfs/sqfs_decompressor.h b/fs/squashfs/sqfs_decompressor.h
index 378965dda8..c67a984425 100644
--- a/fs/squashfs/sqfs_decompressor.h
+++ b/fs/squashfs/sqfs_decompressor.h
@@ -9,6 +9,7 @@
#define SQFS_DECOMPRESSOR_H
#include
+#include "sqfs_filesystem.h"
#define SQFS_COMP_ZLIB 1
#define SQFS_COMP_LZMA 2
@@ -55,4 +56,7 @@ union squashfs_compression_opts {
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 lenp);
+int sqfs_decompressor_init(struct squashfs_ctxt *ctxt);
+void sqfs_decompressor_cleanup(struct squashfs_ctxt *ctxt);
+
#endif /* SQFS_DECOMPRESSOR_H */
diff --git a/fs/squashfs/sqfs_filesystem.h b/fs/squashfs/sqfs_filesystem.h
index d63e3a41ad..856cd15e34 100644
--- a/fs/squashfs/sqfs_filesystem.h
+++ b/fs/squashfs/sqfs_filesystem.h
@@ -9,8 +9,9 @@
#define SQFS_FILESYSTEM_H
#include
-#include
#include
+#include
+#include
#define SQFS_UNCOMPRESSED_DATA 0x0002
#define SQFS_MAGIC_NUMBER 0x73717368
@@ -72,6 +73,15 @@ struct squashfs_super_block {
__le64 export_table_start;
};
+struct squashfs_ctxt {
+ struct disk_partition cur_part_info;
+ struct blk_desc *cur_dev;
+ struct squashfs_super_block *sblk;
+#if IS_ENABLED(CONFIG_ZSTD)
+ void *zstd_works
[PATCH v2 1/4] fs/squashfs: add support for LZO decompression
Add call to lzo's lzo1x_decompress_safe() and rename source length
parameter from 'lenp' to 'src_len'.
U-Boot's LZO sources may still have some unsolved issues that could make the
decompression crash when dealing with fragmented files, so those should be
avoided. The "-no-fragments" option can be passed to mksquashfs.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- Changed commit message.
fs/squashfs/sqfs_decompressor.c | 21 +++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 09ca6cf6d0..9285df5d3b 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,6 +9,11 @@
#include
#include
#include
+
+#if IS_ENABLED(CONFIG_LZO)
+#include
+#endif
+
#if IS_ENABLED(CONFIG_ZLIB)
#include
#endif
@@ -35,20 +40,32 @@ static void zlib_decompression_status(int ret)
#endif
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
- void *source, u32 lenp)
+ void *source, u32 src_len)
{
int ret = 0;
switch (comp_type) {
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
- ret = uncompress(dest, dest_len, source, lenp);
+ ret = uncompress(dest, dest_len, source, src_len);
if (ret) {
zlib_decompression_status(ret);
return -EINVAL;
}
break;
+#endif
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO: {
+ size_t lzo_dest_len = *dest_len;
+ ret = lzo1x_decompress_safe(source, src_len, dest,
_dest_len);
+ if (ret) {
+ printf("LZO decompression failed. Error code: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ break;
+ }
#endif
default:
printf("Error: unknown compression type.\n");
--
2.17.1
[PATCH v2 2/4] fs/squashfs: add support for ZSTD decompression
Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller
can upper bound the decompressed size, which will be the SquashFS data
block (or metadata block) size, so there is no need to use streaming
API.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- No changes since the previous version.
fs/squashfs/sqfs_decompressor.c | 39 +
1 file changed, 39 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 9285df5d3b..b5a9d92808 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -18,6 +18,10 @@
#include
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+#include
+#endif
+
#include "sqfs_decompressor.h"
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
@@ -39,6 +43,31 @@ static void zlib_decompression_status(int ret)
}
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+static int sqfs_zstd_decompress(void *dest, unsigned long dest_len,
+ void *source, u32 src_len)
+{
+ void *workspace;
+ ZSTD_DCtx *ctx;
+ size_t wsize;
+ int ret;
+
+ wsize = ZSTD_DCtxWorkspaceBound();
+
+ workspace = malloc(wsize);
+ if (!workspace)
+ return -ENOMEM;
+
+ ctx = ZSTD_initDCtx(workspace, wsize);
+
+ ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len);
+
+ free(workspace);
+
+ return ZSTD_isError(ret);
+}
+#endif /* CONFIG_ZSTD */
+
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 src_len)
{
@@ -67,6 +96,16 @@ int sqfs_decompress(u16 comp_type, void *dest, unsigned long
*dest_len,
break;
}
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ret = sqfs_zstd_decompress(dest, *dest_len, source, src_len);
+ if (ret) {
+ printf("ZSTD Error code: %d\n", ZSTD_getErrorCode(ret));
+ return -EINVAL;
+ }
+
+ break;
+#endif /* CONFIG_ZSTD */
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
--
2.17.1
[PATCH v2 0/4] fs/squashfs
Hello, Following the SquashFS support, this series adds support for LZO and ZSTD algorithms. Changes in v2: - Change tests architecture. - Add tests for LZO and ZSTD. - Skip compression algorithms unsupported by the host's squashfs-tools version. - Add sqfs_decompress_init() and sqfs_decompress_cleanup() to SquashFS support. Joao Marcos Costa (4): fs/squashfs: add support for LZO decompression fs/squashfs: add support for ZSTD decompression fs/squashfs: Add init. and clean-up functions to decompression test/py: Add tests for LZO and ZSTD fs/squashfs/sqfs.c| 15 ++- fs/squashfs/sqfs_decompressor.c | 108 +- fs/squashfs/sqfs_decompressor.h | 4 + fs/squashfs/sqfs_filesystem.h | 12 +- .../test_fs/test_squashfs/sqfs_common.py | 76 .../test_fs/test_squashfs/test_sqfs_load.py | 40 --- .../test_fs/test_squashfs/test_sqfs_ls.py | 33 -- 7 files changed, 231 insertions(+), 57 deletions(-) -- 2.17.1
Error when decompressing with LZO
Hello, I had an error when decompressing an SquashFS file fragment while running the tests in test/py. The error code is -6 (LZO_E_LOOKBEHIND_OVERRUN). The SquashFS image was generated with mksquashfs (squashfs-tools) latest version. After some research, I found this thread: https://www.mail-archive.com/u-boot@lists.denx.de/msg97035.html Are there any updates on this issue? Best regards, Joao
Re: [PATCH 0/2] fs/squashfs: Add new decompression algorithms
On Tue, 11 Aug 2020 11:13:41 -0400 Tom Rini wrote: > On Tue, Aug 11, 2020 at 03:17:53PM +0200, Joao Marcos Costa wrote: > > > Hello, > > > > Following the SquashFS support, this series adds support for LZO > > and ZSTD algorithms. The only compression type enabled by default > > is ZLIB, so LZO and ZSTD need to be manually selected. > > > > Joao Marcos Costa (2): > > fs/squashfs: add support for LZO decompression > > fs/squashfs: add support for ZSTD decompression > > > > fs/squashfs/sqfs_decompressor.c | 60 > > +++-- 1 file changed, 58 insertions(+), > > 2 deletions(-) > > Can we add tests for this to sandbox? Thanks! > Hello, Is there any way to check the package version with the python API? I already use 'requiredtool' marker, but apparently it is not enough. My host machine (Ubuntu 18.04) has mksquashfs at its version 4.3, without zstd support. However, v4.4 has such support, and I had to download it and build it myself to use zstd compression. The tests will fail if the user has an older version of mksquashfs installed. Would you have any suggestion on how to proceed? Best regards, Joao
Re: [PATCH 0/2] fs/squashfs: Add new decompression algorithms
On Tue, 11 Aug 2020 11:13:41 -0400 Tom Rini wrote: > On Tue, Aug 11, 2020 at 03:17:53PM +0200, Joao Marcos Costa wrote: > > > Hello, > > > > Following the SquashFS support, this series adds support for LZO > > and ZSTD algorithms. The only compression type enabled by default > > is ZLIB, so LZO and ZSTD need to be manually selected. > > > > Joao Marcos Costa (2): > > fs/squashfs: add support for LZO decompression > > fs/squashfs: add support for ZSTD decompression > > > > fs/squashfs/sqfs_decompressor.c | 60 > > +++-- 1 file changed, 58 insertions(+), > > 2 deletions(-) > > Can we add tests for this to sandbox? Thanks! Sure, I will enable the lzo and zstd configs for, and I will make some actual changes to the test scripts I added, because right now they are not really "scalable" (thinking of new compression algorithms). Best regards, Joao
Re: [PATCH 2/2] fs/squashfs: add support for ZSTD decompression
On Tue, 11 Aug 2020 15:29:51 +0200
Thomas Petazzoni wrote:
> On Tue, 11 Aug 2020 15:17:55 +0200
> Joao Marcos Costa wrote:
>
> > +#if IS_ENABLED(CONFIG_ZSTD)
> > +static int sqfs_zstd_decompress(void *dest, unsigned long dest_len,
> > + void *source, u32 src_len)
> > +{
> > + void *workspace;
> > + ZSTD_DCtx *ctx;
> > + size_t wsize;
> > + int ret;
> > +
> > + wsize = ZSTD_DCtxWorkspaceBound();
>
> So apparently this "workspace" has a constant size, which does not
> depend on the size of the input buffer. Correct ?
Yes, correct.
> If that's the case, can we instead allocate it once for all in
> sqfs_probe() if the filesystem is zstd-compressed ? I.e perhaps
> sqfs_probe() could call a sqfs_decompressor_init() function,
> implemented in sqfs_decompressor.c, which will do this sort of
> initialization. And of course this memory area would be freed up in
> sqfs_close(), calling a sqfs_decompressor_{deinit,cleanup,close}
> function.
Sure, it can be done. I will prepare a v2 addressing your suggestion.
Thanks!
> Best regards,
>
> Thomas
Best regards,
Joao
[PATCH 2/2] fs/squashfs: add support for ZSTD decompression
Add call to ZSTD's ZSTD_decompressDCtx(). In this use case, the caller
can upper bound the decompressed size, which will be the SquashFS data
block (or metadata block) size, so there is no need to use streaming
API.
Signed-off-by: Joao Marcos Costa
---
fs/squashfs/sqfs_decompressor.c | 39 +
1 file changed, 39 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 9285df5d3b..b5a9d92808 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -18,6 +18,10 @@
#include
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+#include
+#endif
+
#include "sqfs_decompressor.h"
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
@@ -39,6 +43,31 @@ static void zlib_decompression_status(int ret)
}
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+static int sqfs_zstd_decompress(void *dest, unsigned long dest_len,
+ void *source, u32 src_len)
+{
+ void *workspace;
+ ZSTD_DCtx *ctx;
+ size_t wsize;
+ int ret;
+
+ wsize = ZSTD_DCtxWorkspaceBound();
+
+ workspace = malloc(wsize);
+ if (!workspace)
+ return -ENOMEM;
+
+ ctx = ZSTD_initDCtx(workspace, wsize);
+
+ ret = ZSTD_decompressDCtx(ctx, dest, dest_len, source, src_len);
+
+ free(workspace);
+
+ return ZSTD_isError(ret);
+}
+#endif /* CONFIG_ZSTD */
+
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 src_len)
{
@@ -67,6 +96,16 @@ int sqfs_decompress(u16 comp_type, void *dest, unsigned long
*dest_len,
break;
}
#endif
+#if IS_ENABLED(CONFIG_ZSTD)
+ case SQFS_COMP_ZSTD:
+ ret = sqfs_zstd_decompress(dest, *dest_len, source, src_len);
+ if (ret) {
+ printf("ZSTD Error code: %d\n", ZSTD_getErrorCode(ret));
+ return -EINVAL;
+ }
+
+ break;
+#endif /* CONFIG_ZSTD */
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
--
2.17.1
[PATCH 1/2] fs/squashfs: add support for LZO decompression
Add call to lzo's lzo1x_decompress_safe() and rename source length
parameter from 'lenp' to 'src_len'.
Signed-off-by: Joao Marcos Costa
---
fs/squashfs/sqfs_decompressor.c | 21 +++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index 09ca6cf6d0..9285df5d3b 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,6 +9,11 @@
#include
#include
#include
+
+#if IS_ENABLED(CONFIG_LZO)
+#include
+#endif
+
#if IS_ENABLED(CONFIG_ZLIB)
#include
#endif
@@ -35,20 +40,32 @@ static void zlib_decompression_status(int ret)
#endif
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
- void *source, u32 lenp)
+ void *source, u32 src_len)
{
int ret = 0;
switch (comp_type) {
#if IS_ENABLED(CONFIG_ZLIB)
case SQFS_COMP_ZLIB:
- ret = uncompress(dest, dest_len, source, lenp);
+ ret = uncompress(dest, dest_len, source, src_len);
if (ret) {
zlib_decompression_status(ret);
return -EINVAL;
}
break;
+#endif
+#if IS_ENABLED(CONFIG_LZO)
+ case SQFS_COMP_LZO: {
+ size_t lzo_dest_len = *dest_len;
+ ret = lzo1x_decompress_safe(source, src_len, dest,
_dest_len);
+ if (ret) {
+ printf("LZO decompression failed. Error code: %d\n",
ret);
+ return -EINVAL;
+ }
+
+ break;
+ }
#endif
default:
printf("Error: unknown compression type.\n");
--
2.17.1
[PATCH 0/2] fs/squashfs: Add new decompression algorithms
Hello, Following the SquashFS support, this series adds support for LZO and ZSTD algorithms. The only compression type enabled by default is ZLIB, so LZO and ZSTD need to be manually selected. Joao Marcos Costa (2): fs/squashfs: add support for LZO decompression fs/squashfs: add support for ZSTD decompression fs/squashfs/sqfs_decompressor.c | 60 +++-- 1 file changed, 58 insertions(+), 2 deletions(-) -- 2.17.1
[PATCH] test/py: fix SquashFS tests
Use "cons.config.build_dir" instead of writing to the source directory
(read-only). This will fix the test failures in Azure.
Signed-off-by: Joao Marcos Costa
---
test/py/tests/test_fs/test_squashfs/sqfs_common.py | 12 ++--
.../py/tests/test_fs/test_squashfs/test_sqfs_load.py | 9 +
test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py | 9 +
3 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
index 9ef7b19ad9..2dc344d1b2 100644
--- a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -20,9 +20,9 @@ def sqfs_generate_file(path, size):
file.close()
# generate image with three files and a symbolic link
-def sqfs_generate_image():
-src = "test/py/tests/test_fs/test_squashfs/sqfs_src/"
-dest = "test/py/tests/test_fs/test_squashfs/sqfs"
+def sqfs_generate_image(cons):
+src = os.path.join(cons.config.build_dir, "sqfs_src/")
+dest = os.path.join(cons.config.build_dir, "sqfs")
os.mkdir(src)
sqfs_generate_file(src + "frag_only", 100)
sqfs_generate_file(src + "blks_frag", 5100)
@@ -31,9 +31,9 @@ def sqfs_generate_image():
os.system("mksquashfs " + src + " " + dest + " -b 4096
-always-use-fragments")
# removes all files created by sqfs_generate_image()
-def sqfs_clean():
-src = "test/py/tests/test_fs/test_squashfs/sqfs_src/"
-dest = "test/py/tests/test_fs/test_squashfs/sqfs"
+def sqfs_clean(cons):
+src = os.path.join(cons.config.build_dir, "sqfs_src/")
+dest = os.path.join(cons.config.build_dir, "sqfs")
os.remove(src + "frag_only")
os.remove(src + "blks_frag")
os.remove(src + "blks_only")
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
index 9b828fdf04..eb1baae5c5 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -12,9 +12,10 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_load(u_boot_console):
-sqfs_generate_image()
+cons = u_boot_console
+sqfs_generate_image(cons)
command = "sqfsload host 0 $kernel_addr_r "
-path = "test/py/tests/test_fs/test_squashfs/sqfs"
+path = os.path.join(cons.config.build_dir, "sqfs")
try:
output = u_boot_console.run_command("host bind 0 " + path)
@@ -29,5 +30,5 @@ def test_sqfs_load(u_boot_console):
output = u_boot_console.run_command(command + "sym")
assert "100 bytes read in" in output
except:
-sqfs_clean()
-sqfs_clean()
+sqfs_clean(cons)
+sqfs_clean(cons)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
index dc31f1a50e..3a7b75c778 100644
--- a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
@@ -12,8 +12,9 @@ from sqfs_common import *
@pytest.mark.buildconfigspec('fs_squashfs')
@pytest.mark.requiredtool('mksquashfs')
def test_sqfs_ls(u_boot_console):
-sqfs_generate_image()
-path = "test/py/tests/test_fs/test_squashfs/sqfs"
+cons = u_boot_console
+sqfs_generate_image(cons)
+path = os.path.join(cons.config.build_dir, "sqfs")
try:
output = u_boot_console.run_command("host bind 0 " + path)
output = u_boot_console.run_command("sqfsls host 0")
@@ -22,5 +23,5 @@ def test_sqfs_ls(u_boot_console):
output = u_boot_console.run_command("sqfsls host 0 xxx")
assert "** Cannot find directory. **" in output
except:
-sqfs_clean()
-sqfs_clean()
+sqfs_clean(cons)
+sqfs_clean(cons)
--
2.17.1
Re: [PATCH v4 6/6] test/py: Add tests for the SquashFS commands
Hello! On Sat, 8 Aug 2020 10:41:38 -0400 Tom Rini wrote: > On Sat, Aug 08, 2020 at 08:31:06AM -0400, Tom Rini wrote: > > On Thu, Jul 30, 2020 at 03:33:52PM +0200, Joao Marcos Costa wrote: > > > > > Add Python scripts to test 'ls' and 'load' commands. The scripts > > > generate a SquashFS image and clean the directory after the > > > assertions, or if an exception is raised. > > > > > > Signed-off-by: Joao Marcos Costa > > > > Applied to u-boot/master, thanks! > > But... CI got backed up and I missed a real failure. Over at > https://dev.azure.com/u-boot/u-boot/_build/results?buildId=1053=logs=50449d1b-398e-53ae-48fa-6bf338edeb51=97605dd2-f5a5-5dd7-2118-315ffdc8bcd6=558 > the problem is: > > os.mkdir(src) > E PermissionError: [Errno 13] Permission denied: > 'test/py/tests/test_fs/test_squashfs/sqfs_src/' > > because our source directory is read-only. I believe you need to use > cons.config.build_dir instead to get to our build directory. Both > tests fail in this manner. Can you please confirm my suggestion > fixes this? If you don't want to configure Azure support directly you > can submit a PR against the github mirror and that in turn will > trigger an Azure run you can follow. Thanks! I submitted the PR that applies your suggestion and it really solves the problem. Here is the link to the log: https://travis-ci.org/github/u-boot/u-boot/jobs/716488309#L1307 I will send the patch in a few minutes. Thanks! > > > > > -- > > Tom > > > Joao Marcos
[PATCH v4 1/6] fs/squashfs: new filesystem
Add support for SquashFS filesystem. Right now, it does not support
compression but support for zlib will be added in a follow-up commit.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- Fix issues that happenned while resolving symlinks
- Add missing endianness convertions
- Improve pointers management
Changes in v3:
- Replace CONFIG_IS_ENABLED by IS_ENABLED in fs/fs.c
Changes in v2:
- Add info. to MAINTAINERS
- Replace sqfs_ls() by U-Boot's fs_ls_generic()
- Improve string parsing when resolving symlink targets
- Replace cmd_tbl_t type by the cmd_tbl struct
- Add missing in cmd/sqfs.c
- Fix build failures
- Replace "struct disk_partition_t" by "disk_partition"
- Remove CMD_RET* from the filesystem code
- Simplify superblock handling by adding "struct squashfs_ctxt", which
encloses the complete global context (superblock, cur_dev and
cur_part_info). The struct members are set up during probe() and cleaned
up during close().
MAINTAINERS |6 +
common/spl/Kconfig |9 +
fs/Kconfig |2 +
fs/Makefile |2 +
fs/fs.c | 15 +
fs/squashfs/Kconfig | 10 +
fs/squashfs/Makefile|7 +
fs/squashfs/sqfs.c | 1538 +++
fs/squashfs/sqfs_decompressor.c | 29 +
fs/squashfs/sqfs_decompressor.h | 58 ++
fs/squashfs/sqfs_dir.c | 91 ++
fs/squashfs/sqfs_filesystem.h | 300 ++
fs/squashfs/sqfs_inode.c| 155
fs/squashfs/sqfs_utils.h| 49 +
include/fs.h|1 +
include/squashfs.h | 25 +
16 files changed, 2297 insertions(+)
create mode 100644 fs/squashfs/Kconfig
create mode 100644 fs/squashfs/Makefile
create mode 100644 fs/squashfs/sqfs.c
create mode 100644 fs/squashfs/sqfs_decompressor.c
create mode 100644 fs/squashfs/sqfs_decompressor.h
create mode 100644 fs/squashfs/sqfs_dir.c
create mode 100644 fs/squashfs/sqfs_filesystem.h
create mode 100644 fs/squashfs/sqfs_inode.c
create mode 100644 fs/squashfs/sqfs_utils.h
create mode 100644 include/squashfs.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 889a73f15f..c2501e82f6 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -969,6 +969,12 @@ S: Maintained
F: drivers/spmi/
F: include/spmi/
+SQUASHFS
+M: Joao Marcos Costa
+S: Maintained
+F: fs/squashfs/
+F: include/sqfs.h
+
TARGET_BCMNS3
M: Bharat Gooty
M: Rayagonda Kokatanur
diff --git a/common/spl/Kconfig b/common/spl/Kconfig
index 72c7165dc2..b93944daa1 100644
--- a/common/spl/Kconfig
+++ b/common/spl/Kconfig
@@ -577,6 +577,15 @@ config SPL_FS_EXT4
filesystem from within SPL. Support for the underlying block
device (e.g. MMC or USB) must be enabled separately.
+config SPL_FS_SQUASHFS
+ bool "Support SquashFS filesystems"
+ select FS_SQUASHFS
+ help
+ Enable support for SquashFS filesystems with SPL. This permits
+ U-Boot (or Linux in Falcon mode) to be loaded from a SquashFS
+ filesystem from within SPL. Support for the underlying block
+ device (e.g. MMC or USB) must be enabled separately.
+
config SPL_FS_FAT
bool "Support FAT filesystems"
select FS_FAT
diff --git a/fs/Kconfig b/fs/Kconfig
index 1cb9831be8..620af7f044 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -22,4 +22,6 @@ source "fs/cramfs/Kconfig"
source "fs/yaffs2/Kconfig"
+source "fs/squashfs/Kconfig"
+
endmenu
diff --git a/fs/Makefile b/fs/Makefile
index 42e669c40c..937cbcf6e8 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_FS_LOADER) += fs.o
obj-$(CONFIG_SPL_FS_FAT) += fat/
obj-$(CONFIG_SPL_FS_EXT4) += ext4/
obj-$(CONFIG_SPL_FS_CBFS) += cbfs/
+obj-$(CONFIG_SPL_FS_SQUASHFS) += squashfs/
else
obj-y += fs.o
@@ -23,5 +24,6 @@ obj-$(CONFIG_SANDBOX) += sandbox/
obj-$(CONFIG_CMD_UBIFS) += ubifs/
obj-$(CONFIG_YAFFS2) += yaffs2/
obj-$(CONFIG_CMD_ZFS) += zfs/
+obj-$(CONFIG_FS_SQUASHFS) += squashfs/
endif
obj-y += fs_internal.o
diff --git a/fs/fs.c b/fs/fs.c
index edd8adc21b..5b31a369f7 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -22,6 +22,7 @@
#include
#include
#include
+#include
DECLARE_GLOBAL_DATA_PTR;
@@ -276,6 +277,20 @@ static struct fstype_info fstypes[] = {
.mkdir = fs_mkdir_unsupported,
.ln = fs_ln_unsupported,
},
+#endif
+#if IS_ENABLED(CONFIG_FS_SQUASHFS)
+ {
+ .fstype = FS_TYPE_SQUASHFS,
+ .name = "squashfs",
+ .probe = sqfs_probe,
+ .opendir = sqfs_opendir,
+ .readdir = sqfs_readdir,
+ .ls = fs_ls_generic,
+ .read = sqfs_read,
+
[PATCH v4 3/6] include/u-boot, lib/zlib: add sources for zlib decompression
Add zlib (v1.2.11) uncompr() function to U-Boot. SquashFS depends on
this function to decompress data from a raw disk image. The actual
support for zlib into SquashFS sources will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- Add new config ZLIB_UNCOMPRESS to link zlib's new functionality
only if SquashFS support is enabled.
Changes in v3:
- No changes since v2.
Changes in v2:
- Changed commit title and message, which were wrong and/or misleading
in v1.
fs/squashfs/Kconfig | 1 +
include/u-boot/zlib.h | 32 ++
lib/Kconfig | 7
lib/zlib/uncompr.c| 97 +++
lib/zlib/zlib.c | 3 ++
5 files changed, 140 insertions(+)
create mode 100644 lib/zlib/uncompr.c
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
index b9772e5619..54ab1618f1 100644
--- a/fs/squashfs/Kconfig
+++ b/fs/squashfs/Kconfig
@@ -1,5 +1,6 @@
config FS_SQUASHFS
bool "Enable SquashFS filesystem support"
+ select ZLIB_UNCOMPRESS
help
This provides support for reading images from SquashFS filesystem.
Squashfs is a compressed read-only filesystem for Linux.
diff --git a/include/u-boot/zlib.h b/include/u-boot/zlib.h
index e23ceb50ca..a33cc8780d 100644
--- a/include/u-boot/zlib.h
+++ b/include/u-boot/zlib.h
@@ -110,6 +110,12 @@ extern "C" {
# define voidp z_voidp
#endif
+#if defined(ZLIB_CONST) && !defined(z_const)
+# define z_const const
+#else
+# define z_const
+#endif
+
#if defined(__MSDOS__) && !defined(MSDOS)
# define MSDOS
#endif
@@ -710,6 +716,32 @@ ZEXTERN uInt ZEXPORT crc32 OF((uInt crc, const Bytef
*buf, uInt len));
if (crc != original_crc) error();
*/
+ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong sourceLen));
+/*
+ Decompresses the source buffer into the destination buffer. sourceLen is
+ the byte length of the source buffer. Upon entry, destLen is the total size
+ of the destination buffer, which must be large enough to hold the entire
+ uncompressed data. (The size of the uncompressed data must have been saved
+ previously by the compressor and transmitted to the decompressor by some
+ mechanism outside the scope of this compression library.) Upon exit, destLen
+ is the actual size of the uncompressed data.
+
+ uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
+ enough memory, Z_BUF_ERROR if there was not enough room in the output
+ buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In
+ the case where there is not enough room, uncompress() will fill the output
+ buffer with the uncompressed data up to that point.
+*/
+
+ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong *sourceLen));
+/*
+ Same as uncompress, except that sourceLen is a pointer, where the
+ length of the source is *sourceLen. On return, *sourceLen is the number of
+ source bytes consumed.
+*/
+
ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits,
const char *version, int stream_size));
#define inflateInit(strm) \
diff --git a/lib/Kconfig b/lib/Kconfig
index 2142bd06e6..089348af73 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -460,6 +460,13 @@ config GZIP
help
This enables support for GZIP compression algorithm.
+config ZLIB_UNCOMPRESS
+ bool "Enables zlib's uncompress() functionality"
+ help
+ This enables an extra zlib functionality: the uncompress() function,
+ which decompresses data from a buffer into another, knowing their
+ sizes. Unlike gunzip(), there is no header parsing.
+
config GZIP_COMPRESSED
bool
select ZLIB
diff --git a/lib/zlib/uncompr.c b/lib/zlib/uncompr.c
new file mode 100644
index 00..21e93933b2
--- /dev/null
+++ b/lib/zlib/uncompr.c
@@ -0,0 +1,97 @@
+/* uncompr.c -- decompress a memory buffer
+ * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+/* @(#) $Id$ */
+
+#define ZLIB_INTERNAL
+#include "zlib.h"
+
+/* ===
+ Decompresses the source buffer into the destination buffer. *sourceLen is
+ the byte length of the source buffer. Upon entry, *destLen is the total size
+ of the destination buffer, which must be large enough to hold the entire
+ uncompressed data. (The size of the uncompressed data must have been saved
+ previously by the compressor and transmitted to the decompressor by some
+ mechanism outside the scope of this compression library.) Upon exit,
+ *destLen is the si
[PATCH v4 4/6] fs/squashfs: add support for zlib decompression
Add call to zlib's 'uncompress' function. Add function to display the
right error message depending on the decompression's return value.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- Add preprocessor directives to check if a given compression option
is enabled, starting by ZLIB. As long as other options get supported,
their respective directives will be added as well.
Changes in v3:
- No changes since v2.
Changes in v2:
- No changes since v1.
fs/squashfs/sqfs_decompressor.c | 30 ++
1 file changed, 30 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index a899a5704b..09ca6cf6d0 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,17 +9,47 @@
#include
#include
#include
+#if IS_ENABLED(CONFIG_ZLIB)
+#include
+#endif
#include "sqfs_decompressor.h"
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
+#if IS_ENABLED(CONFIG_ZLIB)
+static void zlib_decompression_status(int ret)
+{
+ switch (ret) {
+ case Z_BUF_ERROR:
+ printf("Error: 'dest' buffer is not large enough.\n");
+ break;
+ case Z_DATA_ERROR:
+ printf("Error: corrupted compressed data.\n");
+ break;
+ case Z_MEM_ERROR:
+ printf("Error: insufficient memory.\n");
+ break;
+ }
+}
+#endif
+
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 lenp)
{
int ret = 0;
switch (comp_type) {
+#if IS_ENABLED(CONFIG_ZLIB)
+ case SQFS_COMP_ZLIB:
+ ret = uncompress(dest, dest_len, source, lenp);
+ if (ret) {
+ zlib_decompression_status(ret);
+ return -EINVAL;
+ }
+
+ break;
+#endif
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
--
2.17.1
[PATCH v4 5/6] fs/fs.c: add symbolic link case to fs_ls_generic()
Adds an 'else if' statement inside the loop to check for symbolic links.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- No changes since v3.
Changes in v3:
- No changes since v2.
Changes in v2:
- This patch was added in v2 and did not exist in the last version.
fs/fs.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/fs/fs.c b/fs/fs.c
index 5b31a369f7..17e4bc33f7 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -59,6 +59,9 @@ static int fs_ls_generic(const char *dirname)
if (dent->type == FS_DT_DIR) {
printf("%s/\n", dent->name);
ndirs++;
+ } else if (dent->type == FS_DT_LNK) {
+ printf(" %s\n", dent->name);
+ nfiles++;
} else {
printf(" %8lld %s\n", dent->size, dent->name);
nfiles++;
--
2.17.1
[PATCH v4 2/6] fs/squashfs: add filesystem commands
Add 'ls' and 'load' commands.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- No changes since v3.
Changes in v3:
- No changes since v2.
Changes in v2:
- Add info. to MAINTAINERS
- Add 'static' keyword to do_sqfs_*(). These functions' prototypes
are no longer needed in include/fs.h.
- Change commit message.
MAINTAINERS | 1 +
cmd/Kconfig | 6 ++
cmd/Makefile | 1 +
cmd/sqfs.c | 42 ++
4 files changed, 50 insertions(+)
create mode 100644 cmd/sqfs.c
diff --git a/MAINTAINERS b/MAINTAINERS
index c2501e82f6..8ec6c5db81 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -974,6 +974,7 @@ M: Joao Marcos Costa
S: Maintained
F: fs/squashfs/
F: include/sqfs.h
+F: cmd/sqfs.c
TARGET_BCMNS3
M: Bharat Gooty
diff --git a/cmd/Kconfig b/cmd/Kconfig
index e2b0a4fbc0..697eba3dd7 100644
--- a/cmd/Kconfig
+++ b/cmd/Kconfig
@@ -2060,6 +2060,12 @@ config CMD_FAT
help
Support for the FAT fs
+config CMD_SQUASHFS
+ bool "SquashFS command support"
+ select FS_SQUASHFS
+ help
+ Enables SquashFS filesystem commands (e.g. load, ls).
+
config CMD_FS_GENERIC
bool "filesystem commands"
help
diff --git a/cmd/Makefile b/cmd/Makefile
index 7952138dc2..ef2da1c928 100644
--- a/cmd/Makefile
+++ b/cmd/Makefile
@@ -62,6 +62,7 @@ obj-$(CONFIG_CMD_EXT4) += ext4.o
obj-$(CONFIG_CMD_EXT2) += ext2.o
obj-$(CONFIG_CMD_FAT) += fat.o
obj-$(CONFIG_CMD_FDT) += fdt.o
+obj-$(CONFIG_CMD_SQUASHFS) += sqfs.o
obj-$(CONFIG_CMD_FLASH) += flash.o
obj-$(CONFIG_CMD_FPGA) += fpga.o
obj-$(CONFIG_CMD_FPGAD) += fpgad.o
diff --git a/cmd/sqfs.c b/cmd/sqfs.c
new file mode 100644
index 00..107038c4cf
--- /dev/null
+++ b/cmd/sqfs.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Bootlin
+ *
+ * Author: Joao Marcos Costa
+ *
+ * squashfs.c: implements SquashFS related commands
+ */
+
+#include
+#include
+#include
+
+static int do_sqfs_ls(struct cmd_tbl *cmdtp, int flag, int argc, char * const
argv[])
+{
+ return do_ls(cmdtp, flag, argc, argv, FS_TYPE_SQUASHFS);
+}
+
+U_BOOT_CMD(sqfsls, 4, 1, do_sqfs_ls,
+ "List files in directory. Default: root (/).",
+ " [] [directory]\n"
+ "- list files from 'dev' on 'interface' in 'directory'\n"
+);
+
+static int do_sqfs_load(struct cmd_tbl *cmdtp, int flag, int argc, char *
const argv[])
+{
+ return do_load(cmdtp, flag, argc, argv, FS_TYPE_SQUASHFS);
+}
+
+U_BOOT_CMD(sqfsload, 7, 0, do_sqfs_load,
+ "load binary file from a SquashFS filesystem",
+ " [ [ [ [bytes [pos]\n"
+ "- Load binary file 'filename' from 'dev' on 'interface'\n"
+ " to address 'addr' from SquashFS filesystem.\n"
+ " 'pos' gives the file position to start loading from.\n"
+ " If 'pos' is omitted, 0 is used. 'pos' requires 'bytes'.\n"
+ " 'bytes' gives the size to load. If 'bytes' is 0 or omitted,\n"
+ " the load stops on end of file.\n"
+ " If either 'pos' or 'bytes' are not aligned to\n"
+ " ARCH_DMA_MINALIGN then a misaligned buffer warning will\n"
+ " be printed and performance will suffer for the load."
+);
--
2.17.1
[PATCH v4 6/6] test/py: Add tests for the SquashFS commands
Add Python scripts to test 'ls' and 'load' commands. The scripts
generate a SquashFS image and clean the directory after the assertions,
or if an exception is raised.
Signed-off-by: Joao Marcos Costa
---
Changes in v4:
- This patch was added in v4 and did not exist in the last version.
MAINTAINERS | 1 +
configs/sandbox_defconfig | 1 +
.../test_fs/test_squashfs/sqfs_common.py | 42 +++
.../test_fs/test_squashfs/test_sqfs_load.py | 33 +++
.../test_fs/test_squashfs/test_sqfs_ls.py | 26
5 files changed, 103 insertions(+)
create mode 100644 test/py/tests/test_fs/test_squashfs/sqfs_common.py
create mode 100644 test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
create mode 100644 test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
diff --git a/MAINTAINERS b/MAINTAINERS
index 8ec6c5db81..b12074ff3f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -975,6 +975,7 @@ S: Maintained
F: fs/squashfs/
F: include/sqfs.h
F: cmd/sqfs.c
+F: test/py/tests/test_fs/test_squashfs/
TARGET_BCMNS3
M: Bharat Gooty
diff --git a/configs/sandbox_defconfig b/configs/sandbox_defconfig
index 829056e9ce..4dd5f87c68 100644
--- a/configs/sandbox_defconfig
+++ b/configs/sandbox_defconfig
@@ -84,6 +84,7 @@ CONFIG_CMD_TPM=y
CONFIG_CMD_TPM_TEST=y
CONFIG_CMD_BTRFS=y
CONFIG_CMD_CBFS=y
+CONFIG_CMD_SQUASHFS=y
CONFIG_CMD_CRAMFS=y
CONFIG_CMD_EXT4_WRITE=y
CONFIG_CMD_MTDPARTS=y
diff --git a/test/py/tests/test_fs/test_squashfs/sqfs_common.py
b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
new file mode 100644
index 00..9ef7b19ad9
--- /dev/null
+++ b/test/py/tests/test_fs/test_squashfs/sqfs_common.py
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2020 Bootlin
+# Author: Joao Marcos Costa
+
+import os
+import random
+import string
+
+def sqfs_get_random_letters(size):
+letters = []
+for i in range(0, size):
+letters.append(random.choice(string.ascii_letters))
+
+return ''.join(letters)
+
+def sqfs_generate_file(path, size):
+content = sqfs_get_random_letters(size)
+file = open(path, "w")
+file.write(content)
+file.close()
+
+# generate image with three files and a symbolic link
+def sqfs_generate_image():
+src = "test/py/tests/test_fs/test_squashfs/sqfs_src/"
+dest = "test/py/tests/test_fs/test_squashfs/sqfs"
+os.mkdir(src)
+sqfs_generate_file(src + "frag_only", 100)
+sqfs_generate_file(src + "blks_frag", 5100)
+sqfs_generate_file(src + "blks_only", 4096)
+os.symlink("frag_only", src + "sym")
+os.system("mksquashfs " + src + " " + dest + " -b 4096
-always-use-fragments")
+
+# removes all files created by sqfs_generate_image()
+def sqfs_clean():
+src = "test/py/tests/test_fs/test_squashfs/sqfs_src/"
+dest = "test/py/tests/test_fs/test_squashfs/sqfs"
+os.remove(src + "frag_only")
+os.remove(src + "blks_frag")
+os.remove(src + "blks_only")
+os.remove(src + "sym")
+os.rmdir(src)
+os.remove(dest)
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
new file mode 100644
index 00..9b828fdf04
--- /dev/null
+++ b/test/py/tests/test_fs/test_squashfs/test_sqfs_load.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2020 Bootlin
+# Author: Joao Marcos Costa
+
+import os
+import pytest
+from sqfs_common import *
+
+@pytest.mark.boardspec('sandbox')
+@pytest.mark.buildconfigspec('cmd_fs_generic')
+@pytest.mark.buildconfigspec('cmd_squashfs')
+@pytest.mark.buildconfigspec('fs_squashfs')
+@pytest.mark.requiredtool('mksquashfs')
+def test_sqfs_load(u_boot_console):
+sqfs_generate_image()
+command = "sqfsload host 0 $kernel_addr_r "
+path = "test/py/tests/test_fs/test_squashfs/sqfs"
+
+try:
+output = u_boot_console.run_command("host bind 0 " + path)
+output = u_boot_console.run_command(command + "xxx")
+assert "File not found." in output
+output = u_boot_console.run_command(command + "frag_only")
+assert "100 bytes read in" in output
+output = u_boot_console.run_command(command + "blks_frag")
+assert "5100 bytes read in" in output
+output = u_boot_console.run_command(command + "blks_only")
+assert "4096 bytes read in" in output
+output = u_boot_console.run_command(command + "sym")
+assert "100 bytes read in" in output
+except:
+sqfs_clean()
+sqfs_clean()
diff --git a/test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py
b/test/py/tests/test_fs/test_squashfs/tes
[PATCH v4 0/6] Add support for the SquashFS filesystem
Hello! This series adds support for the SquashFS filesystem. For now, zlib is the only supported compression type. This is my first contribution to U-Boot as well as to a major Open Source project. Changes in v4: - Add tests for the SquashFS commands - Add missing endianness convertions - Improve pointers management Changes in v3: - Replace CONFIG_IS_ENABLED by IS_ENABLED in fs/fs.c Changes in v2: - Replace sqfs_ls() by U-Boot's fs_ls_generic() - Add info. to MAINTAINERS - Fix build failures - Fix style problems Joao Marcos Costa (6): fs/squashfs: new filesystem fs/squashfs: add filesystem commands include/u-boot, lib/zlib: add sources for zlib decompression fs/squashfs: add support for zlib decompression fs/fs.c: add symbolic link case to fs_ls_generic() test/py: Add tests for the SquashFS commands MAINTAINERS |8 + cmd/Kconfig |6 + cmd/Makefile |1 + cmd/sqfs.c| 42 + common/spl/Kconfig|9 + configs/sandbox_defconfig |1 + fs/Kconfig|2 + fs/Makefile |2 + fs/fs.c | 18 + fs/squashfs/Kconfig | 11 + fs/squashfs/Makefile |7 + fs/squashfs/sqfs.c| 1538 + fs/squashfs/sqfs_decompressor.c | 59 + fs/squashfs/sqfs_decompressor.h | 58 + fs/squashfs/sqfs_dir.c| 91 + fs/squashfs/sqfs_filesystem.h | 300 fs/squashfs/sqfs_inode.c | 155 ++ fs/squashfs/sqfs_utils.h | 49 + include/fs.h |1 + include/squashfs.h| 25 + include/u-boot/zlib.h | 32 + lib/Kconfig |7 + lib/zlib/uncompr.c| 97 ++ lib/zlib/zlib.c |3 + .../test_fs/test_squashfs/sqfs_common.py | 42 + .../test_fs/test_squashfs/test_sqfs_load.py | 33 + .../test_fs/test_squashfs/test_sqfs_ls.py | 26 + 27 files changed, 2623 insertions(+) create mode 100644 cmd/sqfs.c create mode 100644 fs/squashfs/Kconfig create mode 100644 fs/squashfs/Makefile create mode 100644 fs/squashfs/sqfs.c create mode 100644 fs/squashfs/sqfs_decompressor.c create mode 100644 fs/squashfs/sqfs_decompressor.h create mode 100644 fs/squashfs/sqfs_dir.c create mode 100644 fs/squashfs/sqfs_filesystem.h create mode 100644 fs/squashfs/sqfs_inode.c create mode 100644 fs/squashfs/sqfs_utils.h create mode 100644 include/squashfs.h create mode 100644 lib/zlib/uncompr.c create mode 100644 test/py/tests/test_fs/test_squashfs/sqfs_common.py create mode 100644 test/py/tests/test_fs/test_squashfs/test_sqfs_load.py create mode 100644 test/py/tests/test_fs/test_squashfs/test_sqfs_ls.py -- 2.17.1
Re: [PATCH v3 3/5] include/u-boot, lib/zlib: add sources for zlib decompression
On Thu, 30 Jul 2020 08:17:59 -0400 Tom Rini wrote: > On Thu, Jul 30, 2020 at 02:10:03PM +0200, Joao Marcos Costa wrote: > > On Wed, 29 Jul 2020 12:54:23 -0400 > > Tom Rini wrote: > > > > > On Wed, Jul 29, 2020 at 06:00:57PM +0200, Joao Marcos Costa > > > wrote: > > > > On Wed, 29 Jul 2020 11:27:45 -0400 > > > > Tom Rini wrote: > > > > > > > > > On Tue, Jul 21, 2020 at 11:22:57AM +0200, Joao Marcos Costa > > > > > wrote: > > > > > > Add zlib (v1.2.11) uncompr() function to U-Boot. SquashFS > > > > > > depends on this function to decompress data from a raw disk > > > > > > image. The actual support for zlib into SquashFS sources > > > > > > will be added in a follow-up commit. > > > > > > > > > > > > Signed-off-by: Joao Marcos Costa > > > > > > --- > > > > > > Changes in v3: > > > > > > - No changes since v2. > > > > > > Changes in v2: > > > > > > - Changed commit title and message, which were wrong > > > > > > and/or misleading in v1. > > > > > > include/u-boot/zlib.h | 32 ++ > > > > > > lib/zlib/uncompr.c| 97 > > > > > > +++ lib/zlib/zlib.c > > > > > > | 1 + 3 files changed, 130 insertions(+) > > > > > > create mode 100644 lib/zlib/uncompr.c > > > > > > > > > > Can you confirm that on a platform where we aren't enabling > > > > > squashfs the new zlib functionality ends up being discarded at > > > > > link time? Thanks! > > > > > > > > > > > > > I dumped lib/zlib/zlib.o with readelf and the new functionality > > > > is not discarded, even though the SquashFS support is not > > > > enabled, but this issue will be fixed in the next version (v4). > > > > Thank you! > > > > > > Well, don't dump lib/zlib/zlib.o, that won't say. Check > > > u-boot.map :) > > > > Thanks for the tip! Now I can confirm that the functionality is > > discarded when squashfs is not enabled. I just would like to know > > if I should send the tests in the same patch as the SquashFS > > commands, or should it be into a separate patch? > > A separate patch, and are you able to run them on sandbox then? So it > should also enable squashfs on that target. Thanks! > This will be addressed in v4. Thanks!
Re: [PATCH v3 3/5] include/u-boot, lib/zlib: add sources for zlib decompression
On Wed, 29 Jul 2020 12:54:23 -0400 Tom Rini wrote: > On Wed, Jul 29, 2020 at 06:00:57PM +0200, Joao Marcos Costa wrote: > > On Wed, 29 Jul 2020 11:27:45 -0400 > > Tom Rini wrote: > > > > > On Tue, Jul 21, 2020 at 11:22:57AM +0200, Joao Marcos Costa wrote: > > > > > > > Add zlib (v1.2.11) uncompr() function to U-Boot. SquashFS > > > > depends on this function to decompress data from a raw disk > > > > image. The actual support for zlib into SquashFS sources will > > > > be added in a follow-up commit. > > > > > > > > Signed-off-by: Joao Marcos Costa > > > > --- > > > > Changes in v3: > > > > - No changes since v2. > > > > Changes in v2: > > > > - Changed commit title and message, which were wrong > > > > and/or misleading in v1. > > > > include/u-boot/zlib.h | 32 ++ > > > > lib/zlib/uncompr.c| 97 > > > > +++ lib/zlib/zlib.c > > > > | 1 + 3 files changed, 130 insertions(+) > > > > create mode 100644 lib/zlib/uncompr.c > > > > > > Can you confirm that on a platform where we aren't enabling > > > squashfs the new zlib functionality ends up being discarded at > > > link time? Thanks! > > > > > > > I dumped lib/zlib/zlib.o with readelf and the new functionality is > > not discarded, even though the SquashFS support is not enabled, but > > this issue will be fixed in the next version (v4). Thank you! > > Well, don't dump lib/zlib/zlib.o, that won't say. Check u-boot.map :) > Thanks for the tip! Now I can confirm that the functionality is discarded when squashfs is not enabled. I just would like to know if I should send the tests in the same patch as the SquashFS commands, or should it be into a separate patch?
Re: [PATCH v3 0/5] Add support for the SquashFS filesystem
On Wed, 29 Jul 2020 11:21:26 -0400 Tom Rini wrote: > On Wed, Jul 29, 2020 at 02:54:56PM +0200, Joao Marcos Costa wrote: > > On Tue, 21 Jul 2020 11:22:54 +0200 > > Joao Marcos Costa wrote: > > > > > Hello! > > > > > > This series adds support for the SquashFS filesystem. For now, > > > zlib is the only supported compression type. This is my first > > > contribution to U-Boot as well as to a major Open Source project. > > > > > > Changes in v3: > > > - Replace CONFIG_IS_ENABLED by IS_ENABLED in fs/fs.c > > > Changes in v2: > > > - Replace sqfs_ls() by U-Boot's fs_ls_generic() > > > - Add info. to MAINTAINERS > > > - Fix build failures > > > - Fix style problems > > > > > > Best regards, > > > Joao Marcos Costa > > > > > > Joao Marcos Costa (5): > > > fs/squashfs: new filesystem > > > fs/squashfs: add filesystem commands > > > include/u-boot, lib/zlib: add sources for zlib decompression > > > fs/squashfs: add support for zlib decompression > > > fs/fs.c: add symbolic link case to fs_ls_generic() > > > > > > MAINTAINERS |7 + > > > cmd/Kconfig |6 + > > > cmd/Makefile|1 + > > > cmd/sqfs.c | 42 + > > > common/spl/Kconfig |9 + > > > fs/Kconfig |2 + > > > fs/Makefile |2 + > > > fs/fs.c | 18 + > > > fs/squashfs/Kconfig | 10 + > > > fs/squashfs/Makefile|7 + > > > fs/squashfs/sqfs.c | 1521 > > > +++ fs/squashfs/sqfs_decompressor.c | > > > 53 ++ fs/squashfs/sqfs_decompressor.h | 58 ++ > > > fs/squashfs/sqfs_dir.c | 107 +++ > > > fs/squashfs/sqfs_filesystem.h | 300 ++ > > > fs/squashfs/sqfs_inode.c| 142 +++ > > > fs/squashfs/sqfs_utils.h| 49 + > > > include/fs.h|1 + > > > include/squashfs.h | 25 + > > > include/u-boot/zlib.h | 32 + > > > lib/zlib/uncompr.c | 97 ++ > > > lib/zlib/zlib.c |1 + > > > 22 files changed, 2490 insertions(+) > > > create mode 100644 cmd/sqfs.c > > > create mode 100644 fs/squashfs/Kconfig > > > create mode 100644 fs/squashfs/Makefile > > > create mode 100644 fs/squashfs/sqfs.c > > > create mode 100644 fs/squashfs/sqfs_decompressor.c > > > create mode 100644 fs/squashfs/sqfs_decompressor.h > > > create mode 100644 fs/squashfs/sqfs_dir.c > > > create mode 100644 fs/squashfs/sqfs_filesystem.h > > > create mode 100644 fs/squashfs/sqfs_inode.c > > > create mode 100644 fs/squashfs/sqfs_utils.h > > > create mode 100644 include/squashfs.h > > > create mode 100644 lib/zlib/uncompr.c > > > > > > > Hello again, > > > > I am currently writing tests for the SquashFS support. I wrote two > > Python scripts to test 'ls' and 'load' commands, but I need a > > SquashFS image to do so, and I would like to know what is the > > better approach for this: > > - Add a compiled SquashFS image into the same directory as the > > tests, i.e.: test/py/tests/test_fs/test_squashfs. This is a +-7MB > > sized binary file. > > - Use the scripts to generate an image, with mksquashfs. However, > > mksquashfs becomes a dependency. > > We should dynamically make the images, just like ext* and fat. The > tests should skip when the required tools aren't found, in the same > manner we do for other fs tests. Thanks! > Ok, thank you! Best regards, Joao Marcos
Re: [PATCH v3 3/5] include/u-boot, lib/zlib: add sources for zlib decompression
On Wed, 29 Jul 2020 11:27:45 -0400 Tom Rini wrote: > On Tue, Jul 21, 2020 at 11:22:57AM +0200, Joao Marcos Costa wrote: > > > Add zlib (v1.2.11) uncompr() function to U-Boot. SquashFS depends on > > this function to decompress data from a raw disk image. The actual > > support for zlib into SquashFS sources will be added in a follow-up > > commit. > > > > Signed-off-by: Joao Marcos Costa > > --- > > Changes in v3: > > - No changes since v2. > > Changes in v2: > > - Changed commit title and message, which were wrong and/or > > misleading in v1. > > include/u-boot/zlib.h | 32 ++ > > lib/zlib/uncompr.c| 97 > > +++ lib/zlib/zlib.c > > | 1 + 3 files changed, 130 insertions(+) > > create mode 100644 lib/zlib/uncompr.c > > Can you confirm that on a platform where we aren't enabling squashfs > the new zlib functionality ends up being discarded at link time? > Thanks! > I dumped lib/zlib/zlib.o with readelf and the new functionality is not discarded, even though the SquashFS support is not enabled, but this issue will be fixed in the next version (v4). Thank you!
Re: [PATCH v3 0/5] Add support for the SquashFS filesystem
On Tue, 21 Jul 2020 11:22:54 +0200 Joao Marcos Costa wrote: > Hello! > > This series adds support for the SquashFS filesystem. For now, zlib > is the only supported compression type. This is my first contribution > to U-Boot as well as to a major Open Source project. > > Changes in v3: > - Replace CONFIG_IS_ENABLED by IS_ENABLED in fs/fs.c > Changes in v2: > - Replace sqfs_ls() by U-Boot's fs_ls_generic() > - Add info. to MAINTAINERS > - Fix build failures > - Fix style problems > > Best regards, > Joao Marcos Costa > > Joao Marcos Costa (5): > fs/squashfs: new filesystem > fs/squashfs: add filesystem commands > include/u-boot, lib/zlib: add sources for zlib decompression > fs/squashfs: add support for zlib decompression > fs/fs.c: add symbolic link case to fs_ls_generic() > > MAINTAINERS |7 + > cmd/Kconfig |6 + > cmd/Makefile|1 + > cmd/sqfs.c | 42 + > common/spl/Kconfig |9 + > fs/Kconfig |2 + > fs/Makefile |2 + > fs/fs.c | 18 + > fs/squashfs/Kconfig | 10 + > fs/squashfs/Makefile|7 + > fs/squashfs/sqfs.c | 1521 > +++ fs/squashfs/sqfs_decompressor.c | > 53 ++ fs/squashfs/sqfs_decompressor.h | 58 ++ > fs/squashfs/sqfs_dir.c | 107 +++ > fs/squashfs/sqfs_filesystem.h | 300 ++ > fs/squashfs/sqfs_inode.c| 142 +++ > fs/squashfs/sqfs_utils.h| 49 + > include/fs.h|1 + > include/squashfs.h | 25 + > include/u-boot/zlib.h | 32 + > lib/zlib/uncompr.c | 97 ++ > lib/zlib/zlib.c |1 + > 22 files changed, 2490 insertions(+) > create mode 100644 cmd/sqfs.c > create mode 100644 fs/squashfs/Kconfig > create mode 100644 fs/squashfs/Makefile > create mode 100644 fs/squashfs/sqfs.c > create mode 100644 fs/squashfs/sqfs_decompressor.c > create mode 100644 fs/squashfs/sqfs_decompressor.h > create mode 100644 fs/squashfs/sqfs_dir.c > create mode 100644 fs/squashfs/sqfs_filesystem.h > create mode 100644 fs/squashfs/sqfs_inode.c > create mode 100644 fs/squashfs/sqfs_utils.h > create mode 100644 include/squashfs.h > create mode 100644 lib/zlib/uncompr.c > Hello again, I am currently writing tests for the SquashFS support. I wrote two Python scripts to test 'ls' and 'load' commands, but I need a SquashFS image to do so, and I would like to know what is the better approach for this: - Add a compiled SquashFS image into the same directory as the tests, i.e.: test/py/tests/test_fs/test_squashfs. This is a +-7MB sized binary file. - Use the scripts to generate an image, with mksquashfs. However, mksquashfs becomes a dependency. Best regards, Joao Marcos
[PATCH v3 1/5] fs/squashfs: new filesystem
Add support for SquashFS filesystem. Right now, it does not support
compression but support for zlib will be added in a follow-up commit.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- Replace CONFIG_IS_ENABLED by IS_ENABLED in fs/fs.c
Changes in v2:
- Add info. to MAINTAINERS
- Replace sqfs_ls() by U-Boot's fs_ls_generic()
- Improve string parsing when resolving symlink targets
- Replace cmd_tbl_t type by the cmd_tbl struct
- Add missing in cmd/sqfs.c
- Fix build failures
- Replace "struct disk_partition_t" by "disk_partition"
- Remove CMD_RET* from the filesystem code
- Simplify superblock handling by adding "struct squashfs_ctxt", which
encloses the complete global context (superblock, cur_dev and
cur_part_info). The struct members are set up during probe() and cleaned
up during close().
MAINTAINERS |6 +
common/spl/Kconfig |9 +
fs/Kconfig |2 +
fs/Makefile |2 +
fs/fs.c | 15 +
fs/squashfs/Kconfig | 10 +
fs/squashfs/Makefile|7 +
fs/squashfs/sqfs.c | 1521 +++
fs/squashfs/sqfs_decompressor.c | 29 +
fs/squashfs/sqfs_decompressor.h | 58 ++
fs/squashfs/sqfs_dir.c | 107 +++
fs/squashfs/sqfs_filesystem.h | 300 ++
fs/squashfs/sqfs_inode.c| 142 +++
fs/squashfs/sqfs_utils.h| 49 +
include/fs.h|1 +
include/squashfs.h | 25 +
16 files changed, 2283 insertions(+)
create mode 100644 fs/squashfs/Kconfig
create mode 100644 fs/squashfs/Makefile
create mode 100644 fs/squashfs/sqfs.c
create mode 100644 fs/squashfs/sqfs_decompressor.c
create mode 100644 fs/squashfs/sqfs_decompressor.h
create mode 100644 fs/squashfs/sqfs_dir.c
create mode 100644 fs/squashfs/sqfs_filesystem.h
create mode 100644 fs/squashfs/sqfs_inode.c
create mode 100644 fs/squashfs/sqfs_utils.h
create mode 100644 include/squashfs.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 6316c6ca00..198fe2b89e 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -944,6 +944,12 @@ S: Maintained
F: drivers/spmi/
F: include/spmi/
+SQUASHFS
+M: Joao Marcos Costa
+S: Maintained
+F: fs/squashfs/
+F: include/sqfs.h
+
TDA19988 HDMI ENCODER
M: Liviu Dudau
S: Maintained
diff --git a/common/spl/Kconfig b/common/spl/Kconfig
index d09e52e88b..d7a32ca40f 100644
--- a/common/spl/Kconfig
+++ b/common/spl/Kconfig
@@ -577,6 +577,15 @@ config SPL_FS_EXT4
filesystem from within SPL. Support for the underlying block
device (e.g. MMC or USB) must be enabled separately.
+config SPL_FS_SQUASHFS
+ bool "Support SquashFS filesystems"
+ select FS_SQUASHFS
+ help
+ Enable support for SquashFS filesystems with SPL. This permits
+ U-Boot (or Linux in Falcon mode) to be loaded from a SquashFS
+ filesystem from within SPL. Support for the underlying block
+ device (e.g. MMC or USB) must be enabled separately.
+
config SPL_FS_FAT
bool "Support FAT filesystems"
select FS_FAT
diff --git a/fs/Kconfig b/fs/Kconfig
index 1cb9831be8..620af7f044 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -22,4 +22,6 @@ source "fs/cramfs/Kconfig"
source "fs/yaffs2/Kconfig"
+source "fs/squashfs/Kconfig"
+
endmenu
diff --git a/fs/Makefile b/fs/Makefile
index 42e669c40c..937cbcf6e8 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_FS_LOADER) += fs.o
obj-$(CONFIG_SPL_FS_FAT) += fat/
obj-$(CONFIG_SPL_FS_EXT4) += ext4/
obj-$(CONFIG_SPL_FS_CBFS) += cbfs/
+obj-$(CONFIG_SPL_FS_SQUASHFS) += squashfs/
else
obj-y += fs.o
@@ -23,5 +24,6 @@ obj-$(CONFIG_SANDBOX) += sandbox/
obj-$(CONFIG_CMD_UBIFS) += ubifs/
obj-$(CONFIG_YAFFS2) += yaffs2/
obj-$(CONFIG_CMD_ZFS) += zfs/
+obj-$(CONFIG_FS_SQUASHFS) += squashfs/
endif
obj-y += fs_internal.o
diff --git a/fs/fs.c b/fs/fs.c
index edd8adc21b..5b31a369f7 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -22,6 +22,7 @@
#include
#include
#include
+#include
DECLARE_GLOBAL_DATA_PTR;
@@ -276,6 +277,20 @@ static struct fstype_info fstypes[] = {
.mkdir = fs_mkdir_unsupported,
.ln = fs_ln_unsupported,
},
+#endif
+#if IS_ENABLED(CONFIG_FS_SQUASHFS)
+ {
+ .fstype = FS_TYPE_SQUASHFS,
+ .name = "squashfs",
+ .probe = sqfs_probe,
+ .opendir = sqfs_opendir,
+ .readdir = sqfs_readdir,
+ .ls = fs_ls_generic,
+ .read = sqfs_read,
+ .size = sqfs_size,
+ .close = sqfs_close,
+ .closedir = sqfs_closedir,
+ },
#endif
{
.fstype = FS_T
[PATCH v3 5/5] fs/fs.c: add symbolic link case to fs_ls_generic()
Adds an 'else if' statement inside the loop to check for symbolic links.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- No changes since v2.
Changes in v2:
- This patch was added in v2 and did not exist in the last version.
fs/fs.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/fs/fs.c b/fs/fs.c
index 5b31a369f7..17e4bc33f7 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -59,6 +59,9 @@ static int fs_ls_generic(const char *dirname)
if (dent->type == FS_DT_DIR) {
printf("%s/\n", dent->name);
ndirs++;
+ } else if (dent->type == FS_DT_LNK) {
+ printf(" %s\n", dent->name);
+ nfiles++;
} else {
printf(" %8lld %s\n", dent->size, dent->name);
nfiles++;
--
2.17.1
[PATCH v3 3/5] include/u-boot, lib/zlib: add sources for zlib decompression
Add zlib (v1.2.11) uncompr() function to U-Boot. SquashFS depends on
this function to decompress data from a raw disk image. The actual
support for zlib into SquashFS sources will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- No changes since v2.
Changes in v2:
- Changed commit title and message, which were wrong and/or misleading
in v1.
include/u-boot/zlib.h | 32 ++
lib/zlib/uncompr.c| 97 +++
lib/zlib/zlib.c | 1 +
3 files changed, 130 insertions(+)
create mode 100644 lib/zlib/uncompr.c
diff --git a/include/u-boot/zlib.h b/include/u-boot/zlib.h
index e23ceb50ca..a33cc8780d 100644
--- a/include/u-boot/zlib.h
+++ b/include/u-boot/zlib.h
@@ -110,6 +110,12 @@ extern "C" {
# define voidp z_voidp
#endif
+#if defined(ZLIB_CONST) && !defined(z_const)
+# define z_const const
+#else
+# define z_const
+#endif
+
#if defined(__MSDOS__) && !defined(MSDOS)
# define MSDOS
#endif
@@ -710,6 +716,32 @@ ZEXTERN uInt ZEXPORT crc32 OF((uInt crc, const Bytef
*buf, uInt len));
if (crc != original_crc) error();
*/
+ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong sourceLen));
+/*
+ Decompresses the source buffer into the destination buffer. sourceLen is
+ the byte length of the source buffer. Upon entry, destLen is the total size
+ of the destination buffer, which must be large enough to hold the entire
+ uncompressed data. (The size of the uncompressed data must have been saved
+ previously by the compressor and transmitted to the decompressor by some
+ mechanism outside the scope of this compression library.) Upon exit, destLen
+ is the actual size of the uncompressed data.
+
+ uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
+ enough memory, Z_BUF_ERROR if there was not enough room in the output
+ buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In
+ the case where there is not enough room, uncompress() will fill the output
+ buffer with the uncompressed data up to that point.
+*/
+
+ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong *sourceLen));
+/*
+ Same as uncompress, except that sourceLen is a pointer, where the
+ length of the source is *sourceLen. On return, *sourceLen is the number of
+ source bytes consumed.
+*/
+
ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits,
const char *version, int stream_size));
#define inflateInit(strm) \
diff --git a/lib/zlib/uncompr.c b/lib/zlib/uncompr.c
new file mode 100644
index 00..21e93933b2
--- /dev/null
+++ b/lib/zlib/uncompr.c
@@ -0,0 +1,97 @@
+/* uncompr.c -- decompress a memory buffer
+ * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+/* @(#) $Id$ */
+
+#define ZLIB_INTERNAL
+#include "zlib.h"
+
+/* ===
+ Decompresses the source buffer into the destination buffer. *sourceLen is
+ the byte length of the source buffer. Upon entry, *destLen is the total size
+ of the destination buffer, which must be large enough to hold the entire
+ uncompressed data. (The size of the uncompressed data must have been saved
+ previously by the compressor and transmitted to the decompressor by some
+ mechanism outside the scope of this compression library.) Upon exit,
+ *destLen is the size of the decompressed data and *sourceLen is the number
+ of source bytes consumed. Upon return, source + *sourceLen points to the
+ first unused input byte.
+
+ uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough
+ memory, Z_BUF_ERROR if there was not enough room in the output buffer, or
+ Z_DATA_ERROR if the input data was corrupted, including if the input data is
+ an incomplete zlib stream.
+*/
+int ZEXPORT uncompress2(dest, destLen, source, sourceLen)
+ Bytef *dest;
+ uLongf *destLen;
+ const Bytef *source;
+ uLong *sourceLen;
+
+{
+ z_stream stream;
+ int err;
+ const uInt max = (uInt)-1;
+ uLong len, left;
+ /* for detection of incomplete stream when *destLen == 0 */
+ Byte buf[1];
+
+ len = *sourceLen;
+ if (*destLen) {
+ left = *destLen;
+ *destLen = 0;
+ } else {
+ left = 1;
+ dest = buf;
+ }
+
+ stream.next_in = (z_const Bytef *)source;
+ stream.avail_in = 0;
+ stream.zalloc = (alloc_func)0;
+ stream.zfree = (free_func)0;
+ stream.opaque = (voidpf)0;
+
+ err = inflateInit();
+ if (err
[PATCH v3 2/5] fs/squashfs: add filesystem commands
Add 'ls' and 'load' commands.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- No changes since v2.
Changes in v2:
- Add info. to MAINTAINERS
- Add 'static' keyword to do_sqfs_*(). These functions' prototypes
are no longer needed in include/fs.h.
- Change commit message.
MAINTAINERS | 1 +
cmd/Kconfig | 6 ++
cmd/Makefile | 1 +
cmd/sqfs.c | 42 ++
4 files changed, 50 insertions(+)
create mode 100644 cmd/sqfs.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 198fe2b89e..9a5f330148 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -949,6 +949,7 @@ M: Joao Marcos Costa
S: Maintained
F: fs/squashfs/
F: include/sqfs.h
+F: cmd/sqfs.c
TDA19988 HDMI ENCODER
M: Liviu Dudau
diff --git a/cmd/Kconfig b/cmd/Kconfig
index bfe6c163dc..c8f3d34b88 100644
--- a/cmd/Kconfig
+++ b/cmd/Kconfig
@@ -2059,6 +2059,12 @@ config CMD_FAT
help
Support for the FAT fs
+config CMD_SQUASHFS
+ bool "SquashFS command support"
+ select FS_SQUASHFS
+ help
+ Enables SquashFS filesystem commands (e.g. load, ls).
+
config CMD_FS_GENERIC
bool "filesystem commands"
help
diff --git a/cmd/Makefile b/cmd/Makefile
index 7952138dc2..ef2da1c928 100644
--- a/cmd/Makefile
+++ b/cmd/Makefile
@@ -62,6 +62,7 @@ obj-$(CONFIG_CMD_EXT4) += ext4.o
obj-$(CONFIG_CMD_EXT2) += ext2.o
obj-$(CONFIG_CMD_FAT) += fat.o
obj-$(CONFIG_CMD_FDT) += fdt.o
+obj-$(CONFIG_CMD_SQUASHFS) += sqfs.o
obj-$(CONFIG_CMD_FLASH) += flash.o
obj-$(CONFIG_CMD_FPGA) += fpga.o
obj-$(CONFIG_CMD_FPGAD) += fpgad.o
diff --git a/cmd/sqfs.c b/cmd/sqfs.c
new file mode 100644
index 00..107038c4cf
--- /dev/null
+++ b/cmd/sqfs.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Bootlin
+ *
+ * Author: Joao Marcos Costa
+ *
+ * squashfs.c: implements SquashFS related commands
+ */
+
+#include
+#include
+#include
+
+static int do_sqfs_ls(struct cmd_tbl *cmdtp, int flag, int argc, char * const
argv[])
+{
+ return do_ls(cmdtp, flag, argc, argv, FS_TYPE_SQUASHFS);
+}
+
+U_BOOT_CMD(sqfsls, 4, 1, do_sqfs_ls,
+ "List files in directory. Default: root (/).",
+ " [] [directory]\n"
+ "- list files from 'dev' on 'interface' in 'directory'\n"
+);
+
+static int do_sqfs_load(struct cmd_tbl *cmdtp, int flag, int argc, char *
const argv[])
+{
+ return do_load(cmdtp, flag, argc, argv, FS_TYPE_SQUASHFS);
+}
+
+U_BOOT_CMD(sqfsload, 7, 0, do_sqfs_load,
+ "load binary file from a SquashFS filesystem",
+ " [ [ [ [bytes [pos]\n"
+ "- Load binary file 'filename' from 'dev' on 'interface'\n"
+ " to address 'addr' from SquashFS filesystem.\n"
+ " 'pos' gives the file position to start loading from.\n"
+ " If 'pos' is omitted, 0 is used. 'pos' requires 'bytes'.\n"
+ " 'bytes' gives the size to load. If 'bytes' is 0 or omitted,\n"
+ " the load stops on end of file.\n"
+ " If either 'pos' or 'bytes' are not aligned to\n"
+ " ARCH_DMA_MINALIGN then a misaligned buffer warning will\n"
+ " be printed and performance will suffer for the load."
+);
--
2.17.1
[PATCH v3 4/5] fs/squashfs: add support for zlib decompression
Add call to zlib's 'uncompress' function. Add function to display the
right error message depending on the decompression's return value.
Signed-off-by: Joao Marcos Costa
---
Changes in v3:
- No changes since v2.
Changes in v2:
- No changes since v1.
fs/squashfs/sqfs_decompressor.c | 24
1 file changed, 24 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index a899a5704b..a8c37f73b7 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,17 +9,41 @@
#include
#include
#include
+#include
#include "sqfs_decompressor.h"
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
+static void zlib_decompression_status(int ret)
+{
+ switch (ret) {
+ case Z_BUF_ERROR:
+ printf("Error: 'dest' buffer is not large enough.\n");
+ break;
+ case Z_DATA_ERROR:
+ printf("Error: corrupted compressed data.\n");
+ break;
+ case Z_MEM_ERROR:
+ printf("Error: insufficient memory.\n");
+ break;
+ }
+}
+
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 lenp)
{
int ret = 0;
switch (comp_type) {
+ case SQFS_COMP_ZLIB:
+ ret = uncompress(dest, dest_len, source, lenp);
+ if (ret) {
+ zlib_decompression_status(ret);
+ return -EINVAL;
+ }
+
+ break;
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
--
2.17.1
[PATCH v3 0/5] Add support for the SquashFS filesystem
Hello! This series adds support for the SquashFS filesystem. For now, zlib is the only supported compression type. This is my first contribution to U-Boot as well as to a major Open Source project. Changes in v3: - Replace CONFIG_IS_ENABLED by IS_ENABLED in fs/fs.c Changes in v2: - Replace sqfs_ls() by U-Boot's fs_ls_generic() - Add info. to MAINTAINERS - Fix build failures - Fix style problems Best regards, Joao Marcos Costa Joao Marcos Costa (5): fs/squashfs: new filesystem fs/squashfs: add filesystem commands include/u-boot, lib/zlib: add sources for zlib decompression fs/squashfs: add support for zlib decompression fs/fs.c: add symbolic link case to fs_ls_generic() MAINTAINERS |7 + cmd/Kconfig |6 + cmd/Makefile|1 + cmd/sqfs.c | 42 + common/spl/Kconfig |9 + fs/Kconfig |2 + fs/Makefile |2 + fs/fs.c | 18 + fs/squashfs/Kconfig | 10 + fs/squashfs/Makefile|7 + fs/squashfs/sqfs.c | 1521 +++ fs/squashfs/sqfs_decompressor.c | 53 ++ fs/squashfs/sqfs_decompressor.h | 58 ++ fs/squashfs/sqfs_dir.c | 107 +++ fs/squashfs/sqfs_filesystem.h | 300 ++ fs/squashfs/sqfs_inode.c| 142 +++ fs/squashfs/sqfs_utils.h| 49 + include/fs.h|1 + include/squashfs.h | 25 + include/u-boot/zlib.h | 32 + lib/zlib/uncompr.c | 97 ++ lib/zlib/zlib.c |1 + 22 files changed, 2490 insertions(+) create mode 100644 cmd/sqfs.c create mode 100644 fs/squashfs/Kconfig create mode 100644 fs/squashfs/Makefile create mode 100644 fs/squashfs/sqfs.c create mode 100644 fs/squashfs/sqfs_decompressor.c create mode 100644 fs/squashfs/sqfs_decompressor.h create mode 100644 fs/squashfs/sqfs_dir.c create mode 100644 fs/squashfs/sqfs_filesystem.h create mode 100644 fs/squashfs/sqfs_inode.c create mode 100644 fs/squashfs/sqfs_utils.h create mode 100644 include/squashfs.h create mode 100644 lib/zlib/uncompr.c -- 2.17.1
[PATCH v2 3/5] include/u-boot, lib/zlib: add sources for zlib decompression
Add zlib (v1.2.11) uncompr() function to U-Boot. SquashFS depends on
this function to decompress data from a raw disk image. The actual
support for zlib into SquashFS sources will be added in a follow-up
commit.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- Changed commit title and message, which were wrong and/or misleading
in v1.
include/u-boot/zlib.h | 32 ++
lib/zlib/uncompr.c| 97 +++
lib/zlib/zlib.c | 1 +
3 files changed, 130 insertions(+)
create mode 100644 lib/zlib/uncompr.c
diff --git a/include/u-boot/zlib.h b/include/u-boot/zlib.h
index e23ceb50ca..a33cc8780d 100644
--- a/include/u-boot/zlib.h
+++ b/include/u-boot/zlib.h
@@ -110,6 +110,12 @@ extern "C" {
# define voidp z_voidp
#endif
+#if defined(ZLIB_CONST) && !defined(z_const)
+# define z_const const
+#else
+# define z_const
+#endif
+
#if defined(__MSDOS__) && !defined(MSDOS)
# define MSDOS
#endif
@@ -710,6 +716,32 @@ ZEXTERN uInt ZEXPORT crc32 OF((uInt crc, const Bytef
*buf, uInt len));
if (crc != original_crc) error();
*/
+ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong sourceLen));
+/*
+ Decompresses the source buffer into the destination buffer. sourceLen is
+ the byte length of the source buffer. Upon entry, destLen is the total size
+ of the destination buffer, which must be large enough to hold the entire
+ uncompressed data. (The size of the uncompressed data must have been saved
+ previously by the compressor and transmitted to the decompressor by some
+ mechanism outside the scope of this compression library.) Upon exit, destLen
+ is the actual size of the uncompressed data.
+
+ uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
+ enough memory, Z_BUF_ERROR if there was not enough room in the output
+ buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In
+ the case where there is not enough room, uncompress() will fill the output
+ buffer with the uncompressed data up to that point.
+*/
+
+ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong *sourceLen));
+/*
+ Same as uncompress, except that sourceLen is a pointer, where the
+ length of the source is *sourceLen. On return, *sourceLen is the number of
+ source bytes consumed.
+*/
+
ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits,
const char *version, int stream_size));
#define inflateInit(strm) \
diff --git a/lib/zlib/uncompr.c b/lib/zlib/uncompr.c
new file mode 100644
index 00..21e93933b2
--- /dev/null
+++ b/lib/zlib/uncompr.c
@@ -0,0 +1,97 @@
+/* uncompr.c -- decompress a memory buffer
+ * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ */
+
+/* @(#) $Id$ */
+
+#define ZLIB_INTERNAL
+#include "zlib.h"
+
+/* ===
+ Decompresses the source buffer into the destination buffer. *sourceLen is
+ the byte length of the source buffer. Upon entry, *destLen is the total size
+ of the destination buffer, which must be large enough to hold the entire
+ uncompressed data. (The size of the uncompressed data must have been saved
+ previously by the compressor and transmitted to the decompressor by some
+ mechanism outside the scope of this compression library.) Upon exit,
+ *destLen is the size of the decompressed data and *sourceLen is the number
+ of source bytes consumed. Upon return, source + *sourceLen points to the
+ first unused input byte.
+
+ uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough
+ memory, Z_BUF_ERROR if there was not enough room in the output buffer, or
+ Z_DATA_ERROR if the input data was corrupted, including if the input data is
+ an incomplete zlib stream.
+*/
+int ZEXPORT uncompress2(dest, destLen, source, sourceLen)
+ Bytef *dest;
+ uLongf *destLen;
+ const Bytef *source;
+ uLong *sourceLen;
+
+{
+ z_stream stream;
+ int err;
+ const uInt max = (uInt)-1;
+ uLong len, left;
+ /* for detection of incomplete stream when *destLen == 0 */
+ Byte buf[1];
+
+ len = *sourceLen;
+ if (*destLen) {
+ left = *destLen;
+ *destLen = 0;
+ } else {
+ left = 1;
+ dest = buf;
+ }
+
+ stream.next_in = (z_const Bytef *)source;
+ stream.avail_in = 0;
+ stream.zalloc = (alloc_func)0;
+ stream.zfree = (free_func)0;
+ stream.opaque = (voidpf)0;
+
+ err = inflateInit();
+ if (err != Z_OK)
+ return e
[PATCH v2 1/5] fs/squashfs: new filesystem
Add support for SquashFS filesystem. Right now, it does not support
compression but support for zlib will be added in a follow-up commit.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- Add info. to MAINTAINERS
- Replace sqfs_ls() by U-Boot's fs_ls_generic()
- Improve string parsing when resolving symlink targets
- Replace cmd_tbl_t type by the cmd_tbl struct
- Add missing in cmd/sqfs.c
- Fix build failures
- Replace "struct disk_partition_t" by "disk_partition"
- Remove CMD_RET* from the filesystem code
- Simplify superblock handling by adding "struct squashfs_ctxt", which
encloses the complete global context (superblock, cur_dev and
cur_part_info). The struct members are set up during probe() and cleaned
up during close().
MAINTAINERS |6 +
common/spl/Kconfig |9 +
fs/Kconfig |2 +
fs/Makefile |2 +
fs/fs.c | 15 +
fs/squashfs/Kconfig | 10 +
fs/squashfs/Makefile|7 +
fs/squashfs/sqfs.c | 1521 +++
fs/squashfs/sqfs_decompressor.c | 29 +
fs/squashfs/sqfs_decompressor.h | 58 ++
fs/squashfs/sqfs_dir.c | 107 +++
fs/squashfs/sqfs_filesystem.h | 300 ++
fs/squashfs/sqfs_inode.c| 142 +++
fs/squashfs/sqfs_utils.h| 49 +
include/fs.h|1 +
include/squashfs.h | 25 +
16 files changed, 2283 insertions(+)
create mode 100644 fs/squashfs/Kconfig
create mode 100644 fs/squashfs/Makefile
create mode 100644 fs/squashfs/sqfs.c
create mode 100644 fs/squashfs/sqfs_decompressor.c
create mode 100644 fs/squashfs/sqfs_decompressor.h
create mode 100644 fs/squashfs/sqfs_dir.c
create mode 100644 fs/squashfs/sqfs_filesystem.h
create mode 100644 fs/squashfs/sqfs_inode.c
create mode 100644 fs/squashfs/sqfs_utils.h
create mode 100644 include/squashfs.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 6316c6ca00..198fe2b89e 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -944,6 +944,12 @@ S: Maintained
F: drivers/spmi/
F: include/spmi/
+SQUASHFS
+M: Joao Marcos Costa
+S: Maintained
+F: fs/squashfs/
+F: include/sqfs.h
+
TDA19988 HDMI ENCODER
M: Liviu Dudau
S: Maintained
diff --git a/common/spl/Kconfig b/common/spl/Kconfig
index d09e52e88b..d7a32ca40f 100644
--- a/common/spl/Kconfig
+++ b/common/spl/Kconfig
@@ -577,6 +577,15 @@ config SPL_FS_EXT4
filesystem from within SPL. Support for the underlying block
device (e.g. MMC or USB) must be enabled separately.
+config SPL_FS_SQUASHFS
+ bool "Support SquashFS filesystems"
+ select FS_SQUASHFS
+ help
+ Enable support for SquashFS filesystems with SPL. This permits
+ U-Boot (or Linux in Falcon mode) to be loaded from a SquashFS
+ filesystem from within SPL. Support for the underlying block
+ device (e.g. MMC or USB) must be enabled separately.
+
config SPL_FS_FAT
bool "Support FAT filesystems"
select FS_FAT
diff --git a/fs/Kconfig b/fs/Kconfig
index 1cb9831be8..620af7f044 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -22,4 +22,6 @@ source "fs/cramfs/Kconfig"
source "fs/yaffs2/Kconfig"
+source "fs/squashfs/Kconfig"
+
endmenu
diff --git a/fs/Makefile b/fs/Makefile
index 42e669c40c..937cbcf6e8 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_FS_LOADER) += fs.o
obj-$(CONFIG_SPL_FS_FAT) += fat/
obj-$(CONFIG_SPL_FS_EXT4) += ext4/
obj-$(CONFIG_SPL_FS_CBFS) += cbfs/
+obj-$(CONFIG_SPL_FS_SQUASHFS) += squashfs/
else
obj-y += fs.o
@@ -23,5 +24,6 @@ obj-$(CONFIG_SANDBOX) += sandbox/
obj-$(CONFIG_CMD_UBIFS) += ubifs/
obj-$(CONFIG_YAFFS2) += yaffs2/
obj-$(CONFIG_CMD_ZFS) += zfs/
+obj-$(CONFIG_FS_SQUASHFS) += squashfs/
endif
obj-y += fs_internal.o
diff --git a/fs/fs.c b/fs/fs.c
index edd8adc21b..6d776c676a 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -22,6 +22,7 @@
#include
#include
#include
+#include
DECLARE_GLOBAL_DATA_PTR;
@@ -276,6 +277,20 @@ static struct fstype_info fstypes[] = {
.mkdir = fs_mkdir_unsupported,
.ln = fs_ln_unsupported,
},
+#endif
+#if CONFIG_IS_ENABLED(CONFIG_FS_SQUASHFS)
+ {
+ .fstype = FS_TYPE_SQUASHFS,
+ .name = "squashfs",
+ .probe = sqfs_probe,
+ .opendir = sqfs_opendir,
+ .readdir = sqfs_readdir,
+ .ls = fs_ls_generic,
+ .read = sqfs_read,
+ .size = sqfs_size,
+ .close = sqfs_close,
+ .closedir = sqfs_closedir,
+ },
#endif
{
.fstype = FS_TYPE_ANY,
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
new fil
[PATCH v2 4/5] fs/squashfs: add support for zlib decompression
Add call to zlib's 'uncompress' function. Add function to display the
right error message depending on the decompression's return value.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- Changed commit title and message, which were wrong and/or misleading
in v1.
fs/squashfs/sqfs_decompressor.c | 24
1 file changed, 24 insertions(+)
diff --git a/fs/squashfs/sqfs_decompressor.c b/fs/squashfs/sqfs_decompressor.c
index a899a5704b..a8c37f73b7 100644
--- a/fs/squashfs/sqfs_decompressor.c
+++ b/fs/squashfs/sqfs_decompressor.c
@@ -9,17 +9,41 @@
#include
#include
#include
+#include
#include "sqfs_decompressor.h"
#include "sqfs_filesystem.h"
#include "sqfs_utils.h"
+static void zlib_decompression_status(int ret)
+{
+ switch (ret) {
+ case Z_BUF_ERROR:
+ printf("Error: 'dest' buffer is not large enough.\n");
+ break;
+ case Z_DATA_ERROR:
+ printf("Error: corrupted compressed data.\n");
+ break;
+ case Z_MEM_ERROR:
+ printf("Error: insufficient memory.\n");
+ break;
+ }
+}
+
int sqfs_decompress(u16 comp_type, void *dest, unsigned long *dest_len,
void *source, u32 lenp)
{
int ret = 0;
switch (comp_type) {
+ case SQFS_COMP_ZLIB:
+ ret = uncompress(dest, dest_len, source, lenp);
+ if (ret) {
+ zlib_decompression_status(ret);
+ return -EINVAL;
+ }
+
+ break;
default:
printf("Error: unknown compression type.\n");
return -EINVAL;
--
2.17.1
[PATCH v2 5/5] fs/fs.c: add symbolic link case to fs_ls_generic()
Adds an 'else if' statement inside the loop to check for symbolic links.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- Changed commit title and message, which were wrong and/or misleading
in v1.
fs/fs.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/fs/fs.c b/fs/fs.c
index 6d776c676a..e06f87db99 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -59,6 +59,9 @@ static int fs_ls_generic(const char *dirname)
if (dent->type == FS_DT_DIR) {
printf("%s/\n", dent->name);
ndirs++;
+ } else if (dent->type == FS_DT_LNK) {
+ printf(" %s\n", dent->name);
+ nfiles++;
} else {
printf(" %8lld %s\n", dent->size, dent->name);
nfiles++;
--
2.17.1
[PATCH v2 0/5] Add support for the SquashFS filesystem
Hello! This series adds support for the SquashFS filesystem. For now, zlib is the only supported compression type. This is my first contribution to U-Boot as well as to a major Open Source project. Changes in v2: - Replace sqfs_ls() by U-Boot's fs_ls_generic() - Add info. to MAINTAINERS - Fix build failures - Fix style problems Best regards, Joao Marcos Costa Joao Marcos Costa (5): fs/squashfs: new filesystem fs/squashfs: add filesystem commands include/u-boot, lib/zlib: add sources for zlib decompression fs/squashfs: add support for zlib decompression fs/fs.c: add symbolic link case to fs_ls_generic() MAINTAINERS |7 + cmd/Kconfig |6 + cmd/Makefile|1 + cmd/sqfs.c | 42 + common/spl/Kconfig |9 + fs/Kconfig |2 + fs/Makefile |2 + fs/fs.c | 18 + fs/squashfs/Kconfig | 10 + fs/squashfs/Makefile|7 + fs/squashfs/sqfs.c | 1521 +++ fs/squashfs/sqfs_decompressor.c | 53 ++ fs/squashfs/sqfs_decompressor.h | 58 ++ fs/squashfs/sqfs_dir.c | 107 +++ fs/squashfs/sqfs_filesystem.h | 300 ++ fs/squashfs/sqfs_inode.c| 142 +++ fs/squashfs/sqfs_utils.h| 49 + include/fs.h|1 + include/squashfs.h | 25 + include/u-boot/zlib.h | 32 + lib/zlib/uncompr.c | 97 ++ lib/zlib/zlib.c |1 + 22 files changed, 2490 insertions(+) create mode 100644 cmd/sqfs.c create mode 100644 fs/squashfs/Kconfig create mode 100644 fs/squashfs/Makefile create mode 100644 fs/squashfs/sqfs.c create mode 100644 fs/squashfs/sqfs_decompressor.c create mode 100644 fs/squashfs/sqfs_decompressor.h create mode 100644 fs/squashfs/sqfs_dir.c create mode 100644 fs/squashfs/sqfs_filesystem.h create mode 100644 fs/squashfs/sqfs_inode.c create mode 100644 fs/squashfs/sqfs_utils.h create mode 100644 include/squashfs.h create mode 100644 lib/zlib/uncompr.c -- 2.17.1
[PATCH v2 2/5] fs/squashfs: add filesystem commands
Add 'ls' and 'load' commands.
Signed-off-by: Joao Marcos Costa
---
Changes in v2:
- Add info. to MAINTAINERS
- Add 'static' keyword to do_sqfs_*(). These functions' prototypes
are no longer needed in include/fs.h.
- Change commit message.
MAINTAINERS | 1 +
cmd/Kconfig | 6 ++
cmd/Makefile | 1 +
cmd/sqfs.c | 42 ++
4 files changed, 50 insertions(+)
create mode 100644 cmd/sqfs.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 198fe2b89e..9a5f330148 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -949,6 +949,7 @@ M: Joao Marcos Costa
S: Maintained
F: fs/squashfs/
F: include/sqfs.h
+F: cmd/sqfs.c
TDA19988 HDMI ENCODER
M: Liviu Dudau
diff --git a/cmd/Kconfig b/cmd/Kconfig
index bfe6c163dc..c8f3d34b88 100644
--- a/cmd/Kconfig
+++ b/cmd/Kconfig
@@ -2059,6 +2059,12 @@ config CMD_FAT
help
Support for the FAT fs
+config CMD_SQUASHFS
+ bool "SquashFS command support"
+ select FS_SQUASHFS
+ help
+ Enables SquashFS filesystem commands (e.g. load, ls).
+
config CMD_FS_GENERIC
bool "filesystem commands"
help
diff --git a/cmd/Makefile b/cmd/Makefile
index 7952138dc2..ef2da1c928 100644
--- a/cmd/Makefile
+++ b/cmd/Makefile
@@ -62,6 +62,7 @@ obj-$(CONFIG_CMD_EXT4) += ext4.o
obj-$(CONFIG_CMD_EXT2) += ext2.o
obj-$(CONFIG_CMD_FAT) += fat.o
obj-$(CONFIG_CMD_FDT) += fdt.o
+obj-$(CONFIG_CMD_SQUASHFS) += sqfs.o
obj-$(CONFIG_CMD_FLASH) += flash.o
obj-$(CONFIG_CMD_FPGA) += fpga.o
obj-$(CONFIG_CMD_FPGAD) += fpgad.o
diff --git a/cmd/sqfs.c b/cmd/sqfs.c
new file mode 100644
index 00..107038c4cf
--- /dev/null
+++ b/cmd/sqfs.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Bootlin
+ *
+ * Author: Joao Marcos Costa
+ *
+ * squashfs.c: implements SquashFS related commands
+ */
+
+#include
+#include
+#include
+
+static int do_sqfs_ls(struct cmd_tbl *cmdtp, int flag, int argc, char * const
argv[])
+{
+ return do_ls(cmdtp, flag, argc, argv, FS_TYPE_SQUASHFS);
+}
+
+U_BOOT_CMD(sqfsls, 4, 1, do_sqfs_ls,
+ "List files in directory. Default: root (/).",
+ " [] [directory]\n"
+ "- list files from 'dev' on 'interface' in 'directory'\n"
+);
+
+static int do_sqfs_load(struct cmd_tbl *cmdtp, int flag, int argc, char *
const argv[])
+{
+ return do_load(cmdtp, flag, argc, argv, FS_TYPE_SQUASHFS);
+}
+
+U_BOOT_CMD(sqfsload, 7, 0, do_sqfs_load,
+ "load binary file from a SquashFS filesystem",
+ " [ [ [ [bytes [pos]\n"
+ "- Load binary file 'filename' from 'dev' on 'interface'\n"
+ " to address 'addr' from SquashFS filesystem.\n"
+ " 'pos' gives the file position to start loading from.\n"
+ " If 'pos' is omitted, 0 is used. 'pos' requires 'bytes'.\n"
+ " 'bytes' gives the size to load. If 'bytes' is 0 or omitted,\n"
+ " the load stops on end of file.\n"
+ " If either 'pos' or 'bytes' are not aligned to\n"
+ " ARCH_DMA_MINALIGN then a misaligned buffer warning will\n"
+ " be printed and performance will suffer for the load."
+);
--
2.17.1
Re: [PATCH 0/3] squashfs fixes
Hello, Thomas! On Wed, 15 Jul 2020 22:11:40 +0200 Thomas Petazzoni wrote: > Hello Joao, > > As I started reviewing your patch series, I encountered a number of > build issues after applying your patches, and building on the latest > master. The following patches address the build failures I have > encountered. Could you squash them into your commits, so that those > fixes are part of your v2 ? Of course, please do some testing, as I > have only done build testing so far. Thank you for this series, I was actually working on the v2020.04 branch, and I think this explains why I was not seeing such errors. > Also, even with those fixes in place, I do still get a worrying gcc > warning (from gcc 10): I will update my gcc version, because currently I am using gcc 7.5. > > fs/squashfs/sqfs_inode.c: In function ‘sqfs_find_inode’: > fs/squashfs/sqfs_inode.c:72:34: warning: array subscript l is outside > array bounds of ‘struct squashfs_directory_index[0]’ [-Warray-bounds] > 72 | index_list_size += ldir.index[l].size + 1; > |~~^~~ In file included from > fs/squashfs/sqfs_inode.c:16: fs/squashfs/sqfs_filesystem.h:207:34: > note: while referencing ‘index’ 207 | struct > squashfs_directory_index index[0]; | > ^ fs/squashfs/sqfs_inode.c:31:29: note: defined here ‘ldir’ >31 | struct squashfs_ldir_inode ldir; > | ^~~~ > > Could you investigate this ? Surely I will, thanks! > Thanks, > > Thomas > > Thomas Petazzoni (3): > cmd/sqfs.c, include/fs.h: fix build failures, don't expose > do_sqfs_*() functions globally > fs/squashfs: use "struct disk_partition" instead of > "disk_partition_t" fs/squashfs: do not use CMD_RET_* defines in the > filesystem code > > cmd/sqfs.c | 5 ++-- > fs/squashfs/sqfs.c | 66 > -- include/fs.h | > 12 - include/squashfs.h | 4 ++- > 4 files changed, 34 insertions(+), 53 deletions(-) >
Re: [PATCH 1/4] fs/squashfs: new filesystem
Hello! On Thu, 9 Jul 2020 21:50:03 +0200 Rasmus Villemoes wrote: > On 09/07/2020 19.51, Joao Marcos Costa wrote: > > Add support for SquashFS filesystem. Right now, it does not support > > compression but support for zlib will be added in a follow-up > > commit. > > Cool, thanks for proposing this. While I'm not really in a position to > review these, just one comment below. > > > > > +config SPL_FS_SQUASHFS > > + bool "Support SquashFS filesystems" > > + select FS_SQUASHFS > > Is there any reason U-Boot proper must support squashfs if the SPL > does? This isn't quite specific to this patch; I see a number of > 'config SPL_FOO' that either depends on or selects FOO (e.g. > SPL_FS_FAT), and I wonder why. > > Rasmus Well, I don't have enough experience to answer this question properly, but precisely as you mentioned, other filesystems do the same concerning SPL, and I simply decided to follow their example. Best regards, Joao Marcos
[PATCH 1/4] fs/squashfs: new filesystem
Add support for SquashFS filesystem. Right now, it does not support
compression but support for zlib will be added in a follow-up commit.
Signed-off-by: Joao Marcos Costa
---
common/spl/Kconfig |9 +
fs/Kconfig |2 +
fs/Makefile |2 +
fs/fs.c | 15 +
fs/squashfs/Kconfig | 10 +
fs/squashfs/Makefile|7 +
fs/squashfs/sqfs.c | 1625 +++
fs/squashfs/sqfs_decompressor.c | 29 +
fs/squashfs/sqfs_decompressor.h | 58 ++
fs/squashfs/sqfs_dir.c | 107 ++
fs/squashfs/sqfs_filesystem.h | 301 ++
fs/squashfs/sqfs_inode.c| 142 +++
fs/squashfs/sqfs_utils.h| 49 +
include/fs.h| 13 +
include/squashfs.h | 24 +
15 files changed, 2393 insertions(+)
create mode 100644 fs/squashfs/Kconfig
create mode 100644 fs/squashfs/Makefile
create mode 100644 fs/squashfs/sqfs.c
create mode 100644 fs/squashfs/sqfs_decompressor.c
create mode 100644 fs/squashfs/sqfs_decompressor.h
create mode 100644 fs/squashfs/sqfs_dir.c
create mode 100644 fs/squashfs/sqfs_filesystem.h
create mode 100644 fs/squashfs/sqfs_inode.c
create mode 100644 fs/squashfs/sqfs_utils.h
create mode 100644 include/squashfs.h
diff --git a/common/spl/Kconfig b/common/spl/Kconfig
index b03a476b9f..426b783d9d 100644
--- a/common/spl/Kconfig
+++ b/common/spl/Kconfig
@@ -566,6 +566,15 @@ config SPL_FS_EXT4
filesystem from within SPL. Support for the underlying block
device (e.g. MMC or USB) must be enabled separately.
+config SPL_FS_SQUASHFS
+ bool "Support SquashFS filesystems"
+ select FS_SQUASHFS
+ help
+ Enable support for SquashFS filesystems with SPL. This permits
+ U-Boot (or Linux in Falcon mode) to be loaded from a SquashFS
+ filesystem from within SPL. Support for the underlying block
+ device (e.g. MMC or USB) must be enabled separately.
+
config SPL_FS_FAT
bool "Support FAT filesystems"
select FS_FAT
diff --git a/fs/Kconfig b/fs/Kconfig
index 1cb9831be8..620af7f044 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -22,4 +22,6 @@ source "fs/cramfs/Kconfig"
source "fs/yaffs2/Kconfig"
+source "fs/squashfs/Kconfig"
+
endmenu
diff --git a/fs/Makefile b/fs/Makefile
index 42e669c40c..937cbcf6e8 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_FS_LOADER) += fs.o
obj-$(CONFIG_SPL_FS_FAT) += fat/
obj-$(CONFIG_SPL_FS_EXT4) += ext4/
obj-$(CONFIG_SPL_FS_CBFS) += cbfs/
+obj-$(CONFIG_SPL_FS_SQUASHFS) += squashfs/
else
obj-y += fs.o
@@ -23,5 +24,6 @@ obj-$(CONFIG_SANDBOX) += sandbox/
obj-$(CONFIG_CMD_UBIFS) += ubifs/
obj-$(CONFIG_YAFFS2) += yaffs2/
obj-$(CONFIG_CMD_ZFS) += zfs/
+obj-$(CONFIG_FS_SQUASHFS) += squashfs/
endif
obj-y += fs_internal.o
diff --git a/fs/fs.c b/fs/fs.c
index 0c66d60477..2684e6ccce 100644
--- a/fs/fs.c
+++ b/fs/fs.c
@@ -19,6 +19,7 @@
#include
#include
#include
+#include
DECLARE_GLOBAL_DATA_PTR;
@@ -273,6 +274,20 @@ static struct fstype_info fstypes[] = {
.mkdir = fs_mkdir_unsupported,
.ln = fs_ln_unsupported,
},
+#endif
+#ifdef CONFIG_FS_SQUASHFS
+ {
+ .fstype = FS_TYPE_SQUASHFS,
+ .name = "squashfs",
+ .probe = sqfs_probe,
+ .opendir = sqfs_opendir,
+ .readdir = sqfs_readdir,
+ .ls = sqfs_ls,
+ .read = sqfs_read,
+ .size = sqfs_size,
+ .close = sqfs_close,
+ .closedir = sqfs_closedir,
+ },
#endif
{
.fstype = FS_TYPE_ANY,
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
new file mode 100644
index 00..b9772e5619
--- /dev/null
+++ b/fs/squashfs/Kconfig
@@ -0,0 +1,10 @@
+config FS_SQUASHFS
+ bool "Enable SquashFS filesystem support"
+ help
+ This provides support for reading images from SquashFS filesystem.
+ Squashfs is a compressed read-only filesystem for Linux.
+ It uses zlib, lz4, lzo, or xz compression to compress files, inodes
+ and directories. Squashfs is intended for general read-only
+ filesystem use, for archival use (i.e. in cases where a .tar.gz file
+ may be used), and in constrained block device/memory systems (e.g.
+ embedded systems) where low overhead is needed.
diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
new file mode 100644
index 00..ba66ee821c
--- /dev/null
+++ b/fs/squashfs/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0+
+#
+
+obj-$(CONFIG_$(SPL_)FS_SQUASHFS) = sqfs.o \
+ sqfs_inode.o \
+ sqfs_dir.o \
+ sqfs_decompressor.o
diff --git a/fs/squa
