Copilot commented on code in PR #6860:
URL: https://github.com/apache/texera/pull/6860#discussion_r3780016020
##########
common/config/src/main/scala/org/apache/texera/common/config/EnvironmentalVariable.scala:
##########
@@ -35,7 +35,10 @@ object EnvironmentalVariable {
/**
* FileService related endpoint
*/
- val ENV_FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT =
"FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT"
+ val ENV_FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT =
+ "FILE_SERVICE_GET_DATASET_PRESIGNED_URL_ENDPOINT"
+ val ENV_FILE_SERVICE_GET_MODEL_PRESIGNED_URL_ENDPOINT =
+ "FILE_SERVICE_GET_MODEL_PRESIGNED_URL_ENDPOINT"
Review Comment:
Removing the existing `FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT` name is a
silent configuration break. Deployments that inject a custom FileService URL
under the old name will now ignore it and fall back to `localhost` in both JVM
and Python dataset clients. Preserve the old variable as a deprecated fallback
(or provide an explicit migration path) while introducing the resource-specific
names.
##########
common/workflow-core/src/main/scala/org/apache/texera/amber/core/storage/FileResolver.scala:
##########
@@ -76,106 +85,203 @@ object FileResolver {
}
/**
- * Parses a dataset logical path into its components, or None if it is not
a well-formed dataset path.
- * Expected format:
/datasets/ownerEmail/datasetName/versionName/fileRelativePath
- * The legacy unprefixed format is also accepted.
+ * Parses a versioned-resource file path into its components, or None if it
is not well-formed.
+ *
+ * Two accepted forms:
+ * - Prefixed:
/<prefix>/ownerEmail/resourceName/versionName/fileRelativePath (>= 5 segments)
+ * where the leading segment names a known [[ResourceType]] (dataset,
model, …), so a single
+ * caller can dispatch to the right backing table.
+ * - Legacy unprefixed (datasets only, backward compat):
/ownerEmail/datasetName/versionName/
+ * fileRelativePath (>= 4 segments), resolved as a dataset. Models are
new and always require
+ * the /models/ prefix.
*
- * @param fileName The file path to parse
- * @return Some((ownerEmail, datasetName, versionName, fileRelativePath))
if valid, None otherwise
+ * @param fileName the file path to parse
+ * @return Some((resourceType, ownerEmail, resourceName, versionName,
fileRelativePath)) if valid,
+ * None otherwise
*/
- private def parseDatasetFilePath(
+ private def parsePrefixedPath(
fileName: String
- ): Option[(String, String, String, Array[String])] = {
+ ): Option[(ResourceType.Value, String, String, String, Array[String])] = {
val filePath = Paths.get(fileName)
val pathSegments = (0 until
filePath.getNameCount).map(filePath.getName(_).toString).toArray
- // TODO(datasets-prefix): require the prefix once all stored paths are
migrated (36.sql).
- if (pathSegments.headOption.exists(ResourceType.isValidPrefix)) {
- if (pathSegments.length < 5) None
- else Some((pathSegments(1), pathSegments(2), pathSegments(3),
pathSegments.drop(4)))
- } else if (pathSegments.length >= 4) {
- Some((pathSegments(0), pathSegments(1), pathSegments(2),
pathSegments.drop(3)))
- } else None
+ pathSegments.headOption.flatMap(ResourceType.fromPrefix) match {
+ case Some(resourceType) =>
+ // Prefixed: /<prefix>/ownerEmail/resourceName/versionName/<file> (>=
5 segments).
+ if (pathSegments.length < 5) None
+ else
+ Some(
+ (resourceType, pathSegments(1), pathSegments(2), pathSegments(3),
pathSegments.drop(4))
+ )
+ case None =>
+ // Legacy unprefixed dataset path (backward compat):
/ownerEmail/datasetName/versionName/<file>.
+ // TODO(datasets-prefix): require the prefix once all stored paths are
migrated (36.sql).
+ if (pathSegments.length >= 4)
Review Comment:
This fallback reintroduces unprefixed dataset paths, contradicting this PR's
stated “required-prefix enforcement” and the stacked #6502 contract that the
prefix selects the backing table. It also means an unknown five-segment prefix
is interpreted as an owner email and can resolve if matching dataset rows
exist. Remove the legacy fallback and update the legacy-path test to assert
rejection.
##########
sql/updates/37.sql:
##########
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+\c texera_db
+
+SET search_path TO texera_db;
+
+BEGIN;
+
+-- Introduce ML models as a first-class resource (own primary key `mid`, own
LakeFS repo namespace)
+-- and add model-specific attributes (framework, format).
+
+CREATE TABLE IF NOT EXISTS model
+(
+ mid SERIAL PRIMARY KEY,
+ owner_uid INT NOT NULL,
+ name VARCHAR(128) NOT NULL,
+ repository_name VARCHAR(128),
+ is_public BOOLEAN NOT NULL DEFAULT TRUE,
+ is_downloadable BOOLEAN NOT NULL DEFAULT TRUE,
+ description TEXT NOT NULL,
+ creation_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ cover_image varchar(255),
+ framework VARCHAR(32),
+ format VARCHAR(32),
+ FOREIGN KEY (owner_uid) REFERENCES "user"(uid) ON DELETE CASCADE,
+ UNIQUE (owner_uid, name)
+);
+
+CREATE TABLE IF NOT EXISTS model_version
+(
+ mvid SERIAL PRIMARY KEY,
+ mid INT NOT NULL,
+ creator_uid INT NOT NULL,
+ name VARCHAR(128) NOT NULL,
+ version_hash VARCHAR(64) NOT NULL,
+ creation_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ FOREIGN KEY (mid) REFERENCES model(mid) ON DELETE CASCADE
Review Comment:
A model version is addressed by `(mid, name)`, but the table permits
duplicate names for the same model. If duplicates are inserted (including by
concurrent requests), `lookupModel`'s `fetchOneInto` can throw a too-many-rows
exception, so the logical model path no longer resolves to one stable commit.
Add a unique constraint on `(mid, name)`.
##########
sql/texera_ddl.sql:
##########
@@ -418,6 +419,36 @@ CREATE TABLE IF NOT EXISTS dataset_upload_session_part
ON DELETE CASCADE
);
+-- ML models
+CREATE TABLE IF NOT EXISTS model
+(
+ mid SERIAL PRIMARY KEY,
+ owner_uid INT NOT NULL,
+ name VARCHAR(128) NOT NULL,
+ repository_name VARCHAR(128),
+ is_public BOOLEAN NOT NULL DEFAULT TRUE,
+ is_downloadable BOOLEAN NOT NULL DEFAULT TRUE,
+ description TEXT NOT NULL,
+ creation_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ cover_image varchar(255),
+ framework VARCHAR(32),
+ format VARCHAR(32),
+ FOREIGN KEY (owner_uid) REFERENCES "user"(uid) ON DELETE CASCADE,
+ UNIQUE (owner_uid, name)
+ );
+
+-- model_version
+CREATE TABLE IF NOT EXISTS model_version
+(
+ mvid SERIAL PRIMARY KEY,
+ mid INT NOT NULL,
+ creator_uid INT NOT NULL,
+ name VARCHAR(128) NOT NULL,
+ version_hash VARCHAR(64) NOT NULL,
+ creation_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ FOREIGN KEY (mid) REFERENCES model(mid) ON DELETE CASCADE
Review Comment:
Keep the bootstrap DDL consistent with migration 37 by enforcing uniqueness
of a version name within its model. Without this constraint, a logical
`/models/.../<version>/...` path can match multiple rows and fail resolution.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
