This is an automated email from the ASF dual-hosted git repository.
acassis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nuttx-apps.git
The following commit(s) were added to refs/heads/master by this push:
new c4fd2b9b0 system/nxpkg: fetch repositories over HTTP
c4fd2b9b0 is described below
commit c4fd2b9b0ecc599ed5cdc933ea6555a0c77748e1
Author: aviralgarg05 <[email protected]>
AuthorDate: Sat Aug 15 16:03:16 2026 +0530
system/nxpkg: fetch repositories over HTTP
Download catalogs and artifacts with bounded buffers and atomic staging.
Prepare storage before taking the lock so first-run syncs are safe.
Signed-off-by: aviralgarg05 <[email protected]>
---
system/nxpkg/CMakeLists.txt | 1 +
system/nxpkg/Kconfig | 11 +
system/nxpkg/Makefile | 2 +-
system/nxpkg/pkg_repo.c | 721 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 734 insertions(+), 1 deletion(-)
diff --git a/system/nxpkg/CMakeLists.txt b/system/nxpkg/CMakeLists.txt
index 8ac3e3585..66a57e1ef 100644
--- a/system/nxpkg/CMakeLists.txt
+++ b/system/nxpkg/CMakeLists.txt
@@ -38,6 +38,7 @@ if(CONFIG_SYSTEM_NXPKG)
pkg_log.c
pkg_manifest.c
pkg_metadata.c
+ pkg_repo.c
pkg_store.c
pkg_txn.c)
endif()
diff --git a/system/nxpkg/Kconfig b/system/nxpkg/Kconfig
index afc7fa326..2f08ea623 100644
--- a/system/nxpkg/Kconfig
+++ b/system/nxpkg/Kconfig
@@ -29,4 +29,15 @@ config SYSTEM_NXPKG_STACKSIZE
int "'nxpkg' stack size"
default 16384
+config SYSTEM_NXPKG_ROOT
+ string "'nxpkg' storage root"
+ default "/var/lib/nxpkg"
+ ---help---
+ Base directory used by nxpkg for its local index, installed
+ metadata, temporary downloads, and package payload storage.
+ The default follows the conventional persistent application-data
+ location. A board must mount persistent storage at /var or set
+ this to another persistent location, such as /mnt/sdcard/nxpkg,
+ if packages need to survive a reset.
+
endif
diff --git a/system/nxpkg/Makefile b/system/nxpkg/Makefile
index 757f9fc89..fde7ca89e 100644
--- a/system/nxpkg/Makefile
+++ b/system/nxpkg/Makefile
@@ -28,7 +28,7 @@ STACKSIZE = $(CONFIG_SYSTEM_NXPKG_STACKSIZE)
MODULE = $(CONFIG_SYSTEM_NXPKG)
CSRCS = pkg_compat.c pkg_hash.c pkg_install.c pkg_log.c pkg_manifest.c
-CSRCS += pkg_metadata.c pkg_store.c pkg_txn.c
+CSRCS += pkg_metadata.c pkg_repo.c pkg_store.c pkg_txn.c
MAINSRC = pkg_main.c
include $(APPDIR)/Application.mk
diff --git a/system/nxpkg/pkg_repo.c b/system/nxpkg/pkg_repo.c
new file mode 100644
index 000000000..b1bbe739d
--- /dev/null
+++ b/system/nxpkg/pkg_repo.c
@@ -0,0 +1,721 @@
+/****************************************************************************
+ * apps/system/nxpkg/pkg_repo.c
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+#include <nuttx/config.h>
+
+#include <netutils/cJSON.h>
+
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+# include "netutils/webclient.h"
+#endif
+
+#include "pkg.h"
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Balance network throughput against the 16 KiB caller stacks. */
+
+#define PKG_REPO_FETCH_BUFFER_SIZE 4096
+#define PKG_REPO_HTTP "http://";
+#define PKG_REPO_HTTPS "https://";
+#define PKG_REPO_SOURCE_KEY "_nxpkg_source"
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+struct pkg_fetch_context_s
+{
+ int fd;
+ size_t total;
+};
+#endif
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+static int pkg_repo_copy_string(FAR char *buffer, size_t size,
+ FAR const char *value)
+{
+ int ret;
+
+ ret = snprintf(buffer, size, "%s", value);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ return (size_t)ret >= size ? -ENAMETOOLONG : 0;
+}
+
+static int pkg_repo_source_base(FAR char *buffer, size_t size,
+ FAR const char *source)
+{
+ FAR const char *slash;
+ size_t length;
+
+ slash = strrchr(source, '/');
+ if (slash == NULL)
+ {
+ return pkg_repo_copy_string(buffer, size, ".");
+ }
+
+ length = (size_t)(slash - source);
+ if (length == 0)
+ {
+ length = 1;
+ }
+
+ if (length >= size)
+ {
+ return -ENAMETOOLONG;
+ }
+
+ memcpy(buffer, source, length);
+ buffer[length] = '\0';
+ return 0;
+}
+
+/****************************************************************************
+ * Name: pkg_validate_artifact_relative
+ *
+ * Description:
+ * Reject absolute paths and parent-directory traversal.
+ *
+ ****************************************************************************/
+
+static bool pkg_validate_artifact_relative(FAR const char *value)
+{
+ FAR const char *p;
+
+ if (value == NULL || value[0] == '\0' || value[0] == '/')
+ {
+ return false;
+ }
+
+ p = value;
+ while ((p = strstr(p, "..")) != NULL)
+ {
+ bool at_start = p == value || *(p - 1) == '/';
+ bool at_end = p[2] == '\0' || p[2] == '/';
+
+ if (at_start && at_end)
+ {
+ return false;
+ }
+
+ p++;
+ }
+
+ return true;
+}
+
+static int pkg_repo_read_source(FAR char *buffer, size_t size)
+{
+ FAR cJSON *root;
+ FAR cJSON *source;
+ FAR char *text = NULL;
+ char path[PATH_MAX];
+ size_t length;
+ int ret;
+
+ ret = pkg_store_format_index_path(path, sizeof(path));
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ ret = pkg_store_read_text(path, &text);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ root = cJSON_Parse(text);
+ pkg_free(text);
+ if (root == NULL)
+ {
+ return -EINVAL;
+ }
+
+ source = cJSON_GetObjectItemCaseSensitive(root, PKG_REPO_SOURCE_KEY);
+ if (!cJSON_IsString(source) || source->valuestring == NULL)
+ {
+ cJSON_Delete(root);
+
+ /* Fall back to the legacy source sidecar. */
+
+ ret = pkg_store_format_repo_source_path(path, sizeof(path));
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ ret = pkg_store_read_text(path, &text);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ length = strlen(text);
+ while (length > 0 && isspace((unsigned char)text[length - 1]))
+ {
+ text[--length] = '\0';
+ }
+
+ ret = pkg_repo_copy_string(buffer, size, text);
+ pkg_free(text);
+ return ret;
+ }
+
+ ret = pkg_repo_copy_string(buffer, size, source->valuestring);
+ cJSON_Delete(root);
+ return ret;
+}
+
+static int pkg_repo_attach_source(FAR char **text,
+ FAR const char *source_value)
+{
+ FAR cJSON *root;
+ FAR cJSON *wrapper;
+ FAR char *updated;
+
+ root = cJSON_Parse(*text);
+ if (root == NULL)
+ {
+ return -EINVAL;
+ }
+
+ if (cJSON_IsArray(root))
+ {
+ wrapper = cJSON_CreateObject();
+ if (wrapper == NULL)
+ {
+ cJSON_Delete(root);
+ return -ENOMEM;
+ }
+
+ cJSON_AddItemToObject(wrapper, "packages", root);
+ if (cJSON_GetObjectItemCaseSensitive(wrapper, "packages") != root)
+ {
+ cJSON_Delete(root);
+ cJSON_Delete(wrapper);
+ return -ENOMEM;
+ }
+
+ root = wrapper;
+ }
+ else if (!cJSON_IsObject(root))
+ {
+ cJSON_Delete(root);
+ return -EINVAL;
+ }
+
+ while (cJSON_GetObjectItemCaseSensitive(root,
+ PKG_REPO_SOURCE_KEY) != NULL)
+ {
+ cJSON_DeleteItemFromObjectCaseSensitive(root, PKG_REPO_SOURCE_KEY);
+ }
+
+ if (cJSON_AddStringToObject(root, PKG_REPO_SOURCE_KEY,
+ source_value) == NULL)
+ {
+ cJSON_Delete(root);
+ return -ENOMEM;
+ }
+
+ updated = cJSON_PrintUnformatted(root);
+ cJSON_Delete(root);
+ if (updated == NULL)
+ {
+ return -ENOMEM;
+ }
+
+ pkg_free(*text);
+ *text = updated;
+ return 0;
+}
+
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+static int pkg_repo_sink(FAR char **buffer, int offset, int datend,
+ FAR int *buflen, FAR void *arg)
+{
+ FAR struct pkg_fetch_context_s *ctx;
+ size_t remaining;
+ FAR char *cursor;
+
+ UNUSED(buffer);
+ UNUSED(buflen);
+
+ ctx = arg;
+ cursor = &(*buffer)[offset];
+ remaining = (size_t)(datend - offset);
+
+ /* Enforce the download limit before writing this chunk. */
+
+ if (remaining > 0 &&
+ (ctx->total > PKG_DOWNLOAD_MAX_SIZE ||
+ remaining > PKG_DOWNLOAD_MAX_SIZE - ctx->total))
+ {
+ return -EFBIG;
+ }
+
+ ctx->total += remaining;
+
+ while (remaining > 0)
+ {
+ ssize_t nwritten;
+
+ nwritten = write(ctx->fd, cursor, remaining);
+ if (nwritten < 0)
+ {
+ if (errno == EINTR)
+ {
+ continue;
+ }
+
+ return -errno;
+ }
+
+ if (nwritten == 0)
+ {
+ return -EIO;
+ }
+
+ cursor += nwritten;
+ remaining -= (size_t)nwritten;
+ }
+
+ return 0;
+}
+
+static int pkg_repo_fetch_url(FAR const char *url, FAR const char *dest)
+{
+ struct pkg_fetch_context_s fetch;
+ struct webclient_context client;
+ char reason[64];
+ char buffer[PKG_REPO_FETCH_BUFFER_SIZE];
+ int ret;
+
+ fetch.fd = open(dest, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+ if (fetch.fd < 0)
+ {
+ return -errno;
+ }
+
+ fetch.total = 0;
+ webclient_set_defaults(&client);
+ client.method = "GET";
+ client.url = url;
+ client.buffer = buffer;
+ client.buflen = sizeof(buffer);
+ client.sink_callback = pkg_repo_sink;
+ client.sink_callback_arg = &fetch;
+ client.http_reason = reason;
+ client.http_reason_len = sizeof(reason);
+
+ ret = webclient_perform(&client);
+ if (ret < 0)
+ {
+ close(fetch.fd);
+ unlink(dest);
+ return ret;
+ }
+
+ if (client.http_status / 100 != 2)
+ {
+ close(fetch.fd);
+ unlink(dest);
+ return -EPROTO;
+ }
+
+ ret = close(fetch.fd);
+ if (ret < 0)
+ {
+ ret = -errno;
+ unlink(dest);
+ return ret;
+ }
+
+ return 0;
+}
+#endif
+
+static int pkg_resolve_relative_source(FAR char *buffer, size_t size,
+ FAR const char *relative)
+{
+ char source[PATH_MAX];
+ char base[PATH_MAX];
+ int ret;
+
+ if (buffer == NULL || relative == NULL || relative[0] == '\0')
+ {
+ return -EINVAL;
+ }
+
+ if (pkg_source_is_url(relative))
+ {
+ return pkg_repo_copy_string(buffer, size, relative);
+ }
+
+ if (!pkg_validate_artifact_relative(relative))
+ {
+ return -EINVAL;
+ }
+
+ ret = pkg_repo_read_source(source, sizeof(source));
+ if (ret >= 0)
+ {
+ ret = pkg_repo_source_base(base, sizeof(base), source);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ ret = snprintf(buffer, size, "%s/%s", base, relative);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ return (size_t)ret >= size ? -ENAMETOOLONG : 0;
+ }
+
+ ret = snprintf(buffer, size, "%s/%s", PKG_REPO_DIR, relative);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ return (size_t)ret >= size ? -ENAMETOOLONG : 0;
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+bool pkg_source_is_url(FAR const char *source)
+{
+ if (source == NULL)
+ {
+ return false;
+ }
+
+ return strncasecmp(source, PKG_REPO_HTTP, strlen(PKG_REPO_HTTP)) == 0 ||
+ strncasecmp(source, PKG_REPO_HTTPS, strlen(PKG_REPO_HTTPS)) == 0;
+}
+
+int pkg_resolve_artifact_source(FAR char *buffer, size_t size,
+ FAR const struct pkg_manifest_s *manifest)
+{
+ if (manifest == NULL)
+ {
+ return -EINVAL;
+ }
+
+ return pkg_resolve_relative_source(buffer, size, manifest->artifact);
+}
+
+int pkg_resolve_icon_source(FAR char *buffer, size_t size,
+ FAR const struct pkg_manifest_s *manifest)
+{
+ if (manifest == NULL)
+ {
+ return -EINVAL;
+ }
+
+ return pkg_resolve_relative_source(buffer, size, manifest->icon);
+}
+
+int pkg_acquire_source(FAR const char *source, FAR const char *dest)
+{
+ if (source == NULL || dest == NULL)
+ {
+ return -EINVAL;
+ }
+
+ if (pkg_source_is_url(source))
+ {
+#ifdef CONFIG_NETUTILS_WEBCLIENT
+ return pkg_repo_fetch_url(source, dest);
+#else
+ return -ENOSYS;
+#endif
+ }
+
+ return pkg_store_copy_file(source, dest);
+}
+
+/****************************************************************************
+ * Name: pkg_repo_acquire_sync_lock
+ *
+ * Description:
+ * Serialize catalog synchronization so concurrent downloads cannot
+ * commit out of order.
+ *
+ ****************************************************************************/
+
+static int pkg_repo_acquire_sync_lock(FAR char *path, size_t size)
+{
+ int ret;
+ int tries;
+
+ ret = snprintf(path, size, PKG_ROOT_DIR "/sync.lk");
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ if ((size_t)ret >= size)
+ {
+ return -ENAMETOOLONG;
+ }
+
+ for (tries = 0; tries < 100; tries++)
+ {
+ ret = pkg_lock_create(path);
+ if (ret == 0)
+ {
+ return 0;
+ }
+
+ if (ret != -EEXIST)
+ {
+ return ret;
+ }
+
+ pkg_reclaim_stale_lock(path);
+ usleep(20 * 1000);
+ }
+
+ return -EBUSY;
+}
+
+int pkg_sync(FAR const char *source)
+{
+ FAR struct pkg_index_s *index = NULL;
+ FAR char *text = NULL;
+ FAR char *tmp = NULL;
+ FAR char *index_path = NULL;
+ FAR char *lock;
+ bool remove_tmp = false;
+ int ret;
+
+ if (source == NULL || source[0] == '\0')
+ {
+ pkg_error("sync requires a non-empty index source");
+ return -EINVAL;
+ }
+
+ ret = pkg_store_prepare_layout();
+ if (ret < 0)
+ {
+ pkg_error("unable to prepare package layout: %d", ret);
+ return ret;
+ }
+
+ lock = pkg_malloc(PATH_MAX);
+ if (lock == NULL)
+ {
+ pkg_error("unable to allocate sync lock path buffer");
+ return -ENOMEM;
+ }
+
+ ret = pkg_repo_acquire_sync_lock(lock, PATH_MAX);
+ if (ret < 0)
+ {
+ pkg_error("unable to acquire sync lock: %d", ret);
+ pkg_free(lock);
+ return ret;
+ }
+
+ index = pkg_zalloc(sizeof(*index));
+ tmp = pkg_malloc(PATH_MAX);
+ index_path = pkg_malloc(PATH_MAX);
+ if (index == NULL || tmp == NULL || index_path == NULL)
+ {
+ pkg_error("unable to allocate index metadata buffer");
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ /* Use a per-process FAT-compatible staging name. */
+
+ ret = snprintf(tmp, PATH_MAX, "%s/s%u.jsn", PKG_TMP_DIR,
+ (unsigned int)getpid());
+ if (ret < 0 || (size_t)ret >= PATH_MAX)
+ {
+ pkg_error("temporary sync path is too long");
+ ret = -ENAMETOOLONG;
+ goto out;
+ }
+
+ ret = pkg_acquire_source(source, tmp);
+ if (ret < 0)
+ {
+ pkg_error("unable to fetch index source '%s': %d", source, ret);
+ goto out;
+ }
+
+ remove_tmp = true;
+ ret = pkg_metadata_load_index_path(tmp, index);
+ if (ret < 0)
+ {
+ pkg_error("downloaded index is invalid: %d", ret);
+ goto out;
+ }
+
+ ret = pkg_store_read_text(tmp, &text);
+ if (ret < 0)
+ {
+ pkg_error("unable to read fetched index: %d", ret);
+ goto out;
+ }
+
+ /* Commit the catalog and its source in one atomic file replacement. */
+
+ ret = pkg_repo_attach_source(&text, source);
+ if (ret < 0)
+ {
+ pkg_error("unable to record repository source: %d", ret);
+ goto out;
+ }
+
+ ret = pkg_store_format_index_path(index_path, PATH_MAX);
+ if (ret < 0)
+ {
+ pkg_error("unable to resolve local index path: %d", ret);
+ goto out;
+ }
+
+ ret = pkg_store_write_text_atomic(index_path, text);
+ if (ret < 0)
+ {
+ pkg_error("unable to write local index: %d", ret);
+ goto out;
+ }
+
+ pkg_info("synced package index from %s", source);
+ ret = 0;
+
+out:
+ if (remove_tmp)
+ {
+ pkg_store_remove_file(tmp);
+ }
+
+ pkg_free(text);
+ pkg_free(index);
+ pkg_free(tmp);
+ pkg_free(index_path);
+ pkg_lock_remove(lock);
+ pkg_free(lock);
+ return ret;
+}
+
+int pkg_available(FAR FILE *stream)
+{
+ FAR struct pkg_index_s *index;
+ FAR const char *arch;
+ FAR const char *compat;
+ size_t i;
+ int ret;
+
+ if (stream == NULL)
+ {
+ return EXIT_FAILURE;
+ }
+
+ index = pkg_zalloc(sizeof(*index));
+ if (index == NULL)
+ {
+ pkg_error("unable to allocate index metadata buffer");
+ return EXIT_FAILURE;
+ }
+
+ ret = pkg_store_prepare_layout();
+ if (ret < 0)
+ {
+ pkg_free(index);
+ pkg_error("unable to prepare package layout: %d", ret);
+ return EXIT_FAILURE;
+ }
+
+ ret = pkg_metadata_load_index(index);
+ if (ret < 0)
+ {
+ pkg_free(index);
+ pkg_error("unable to load package index: %d", ret);
+ return EXIT_FAILURE;
+ }
+
+ arch = pkg_runtime_arch();
+ compat = pkg_runtime_compat();
+
+ for (i = 0; i < index->count; i++)
+ {
+ FAR const struct pkg_manifest_s *manifest = &index->manifests[i];
+ FAR const struct pkg_manifest_s *latest;
+
+ if (strcmp(manifest->arch, arch) != 0 ||
+ strcmp(manifest->compat, compat) != 0)
+ {
+ continue;
+ }
+
+ latest = pkg_metadata_find_latest(index, manifest->name);
+ if (latest != manifest)
+ {
+ continue;
+ }
+
+ fprintf(stream,
+ "%s version=%s type=%s arch=%s compat=%s artifact=%s\n",
+ manifest->name,
+ manifest->version,
+ pkg_manifest_type_str(manifest->type),
+ manifest->arch,
+ manifest->compat,
+ manifest->artifact);
+ }
+
+ pkg_free(index);
+ return EXIT_SUCCESS;
+}
