This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev-2.1.7
in repository https://gitbox.apache.org/repos/asf/streampark.git
The following commit(s) were added to refs/heads/dev-2.1.7 by this push:
new 6ccae3149 [Improve] maven args check improvement
6ccae3149 is described below
commit 6ccae3149c83c8311b2c884142697cc8cc1054d8
Author: benjobs <[email protected]>
AuthorDate: Sun Oct 26 23:21:01 2025 +0800
[Improve] maven args check improvement
---
.../streampark/console/base/util/MavenUtils.java | 696 ++++++++++++++++
.../streampark/console/core/entity/Project.java | 131 +--
.../console/core/task/ProjectBuildTask.java | 888 +++++++++++++++++----
3 files changed, 1466 insertions(+), 249 deletions(-)
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/MavenUtils.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/MavenUtils.java
new file mode 100644
index 000000000..6001f07aa
--- /dev/null
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/MavenUtils.java
@@ -0,0 +1,696 @@
+package org.apache.streampark.console.base.util;
+
+import org.apache.streampark.common.conf.CommonConfig;
+import org.apache.streampark.common.conf.InternalConfigHolder;
+import org.apache.streampark.common.util.Utils;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.File;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.Normalizer;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class MavenUtils {
+
+ // Security constants for Maven parameter validation
+ private static final int MAX_BUILD_ARGS_LENGTH = 2048;
+ private static final int MAX_MAVEN_ARG_LENGTH = 512;
+ private static final Pattern COMMAND_INJECTION_PATTERN =
+ Pattern.compile(
+ "(`[^`]*`)|"
+ + // Backticks
+ "(\\$\\([^)]*\\))|"
+ + // $() command substitution
+ "(\\$\\{[^}]*})|"
+ + // ${} variable substitution
+ "([;&|><])|"
+ + // Command separators and redirects
+ "(\\\\[nrt])|"
+ + // Escaped newlines/tabs
+ "([\\r\\n\\t])|"
+ + // Actual control characters
+ "(\\\\x[0-9a-fA-F]{2})|"
+ + // Hex encoded characters
+ "(%[0-9a-fA-F]{2})|"
+ + // URL encoded characters
+ "(\\\\u[0-9a-fA-F]{4})", // Unicode encoded characters
+ Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
+
+ // Whitelist of allowed Maven arguments
+ private static final Set<String> ALLOWED_MAVEN_ARGS =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ "-D",
+ "--define",
+ "-P",
+ "--activate-profiles",
+ "-q",
+ "--quiet",
+ "-X",
+ "--debug",
+ "-e",
+ "--errors",
+ "-f",
+ "--file",
+ "-s",
+ "--settings",
+ "-t",
+ "--toolchains",
+ "-T",
+ "--threads",
+ "-B",
+ "--batch-mode",
+ "-V",
+ "--show-version",
+ "-U",
+ "--update-snapshots",
+ "-N",
+ "--non-recursive",
+ "-C",
+ "--strict-checksums",
+ "-c",
+ "--lax-checksums",
+ "-o",
+ "--offline",
+ "--no-snapshot-updates",
+ "--fail-at-end",
+ "--fail-fast",
+ "--fail-never",
+ "--resume-from",
+ "--projects",
+ "--also-make",
+ "--also-make-dependents",
+ "clean",
+ "compile",
+ "test",
+ "package",
+ "install",
+ "deploy",
+ "validate",
+ "initialize",
+ "generate-sources",
+ "process-sources",
+ "generate-resources",
+ "process-resources",
+ "process-classes",
+ "generate-test-sources",
+ "process-test-sources",
+ "generate-test-resources",
+ "process-test-resources",
+ "test-compile",
+ "process-test-classes",
+ "prepare-package",
+ "pre-integration-test",
+ "integration-test",
+ "post-integration-test",
+ "verify",
+ "install",
+ "deploy")));
+
+ // Allowed system properties (commonly used in Maven builds)
+ private static final Set<String> ALLOWED_SYSTEM_PROPERTIES =
+ Collections.unmodifiableSet(
+ new HashSet<>(
+ Arrays.asList(
+ "skipTests",
+ "maven.test.skip",
+ "maven.javadoc.skip",
+ "maven.source.skip",
+ "project.build.sourceEncoding",
+ "project.reporting.outputEncoding",
+ "maven.compiler.source",
+ "maven.compiler.target",
+ "maven.compiler.release",
+ "flink.version",
+ "scala.version",
+ "hadoop.version",
+ "kafka.version",
+ "java.version",
+ "encoding",
+ "file.encoding")));
+
+ @JsonIgnore
+ public static String getMavenArgs(String buildArgs) {
+ try {
+ StringBuilder mvnArgBuffer = new StringBuilder(" clean package
-DskipTests ");
+
+ // Apply security validation to build arguments
+ if (StringUtils.isNotBlank(buildArgs)) {
+ String validatedBuildArgs =
validateAndSanitizeBuildArgs(buildArgs.trim());
+ if (StringUtils.isNotBlank(validatedBuildArgs)) {
+ mvnArgBuffer.append(validatedBuildArgs);
+ }
+ }
+
+ // Enhanced --settings validation with path security
+ String setting =
InternalConfigHolder.get(CommonConfig.MAVEN_SETTINGS_PATH());
+ if (StringUtils.isNotBlank(setting)) {
+ String validatedSettingsPath =
validateAndSanitizeSettingsPath(setting);
+ mvnArgBuffer.append(" --settings ").append(validatedSettingsPath);
+ }
+
+ // Get final Maven arguments string
+ String mvnArgs = mvnArgBuffer.toString();
+
+ // Enhanced security checks
+ validateFinalMavenCommand(mvnArgs);
+
+ // Find and validate Maven executable with enhanced security
+ String mvn = getSecureMavenExecutable();
+
+ log.info(
+ "đ§ Secure Maven command prepared: {} {}",
+ sanitizeForLogging(mvn),
+ sanitizeForLogging(mvnArgs));
+ return mvn.concat(mvnArgs);
+
+ } catch (SecurityException e) {
+ log.error("đ¨ Security validation failed for Maven build: {}",
e.getMessage());
+ throw new IllegalArgumentException(
+ "Maven build security validation failed: " + e.getMessage(), e);
+ } catch (Exception e) {
+ log.error("â Failed to prepare Maven command: {}", e.getMessage());
+ throw new IllegalArgumentException("Failed to prepare Maven command: " +
e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Validates and sanitizes Maven settings file path to prevent path
traversal attacks.
+ *
+ * @param settingsPath the Maven settings file path
+ * @return validated and normalized settings path
+ * @throws SecurityException if path validation fails
+ */
+ private static String validateAndSanitizeSettingsPath(String settingsPath) {
+ if (StringUtils.isBlank(settingsPath)) {
+ logSecurityEvent("ERROR", "PATH_VALIDATION", "Settings path is blank",
"");
+ throw new SecurityException("Settings path cannot be blank");
+ }
+
+ logSecurityEvent(
+ "INFO", "PATH_VALIDATION", "Starting Maven settings path validation",
settingsPath);
+
+ try {
+ // Normalize the path to resolve any relative components
+ Path normalizedPath = Paths.get(settingsPath).normalize();
+ String pathString = normalizedPath.toString();
+
+ // Security checks for path traversal
+ if (pathString.contains("..") || pathString.contains("~")) {
+ logSecurityEvent(
+ "VIOLATION",
+ "PATH_TRAVERSAL_DETECTION",
+ "Path traversal attempt detected in settings path",
+ settingsPath);
+ throw new SecurityException("Path traversal detected in settings
path");
+ }
+
+ // Verify the file exists and is readable
+ File file = normalizedPath.toFile();
+ if (!file.exists()) {
+ throw new SecurityException(
+ String.format(
+ "Maven settings file does not exist: %s",
sanitizeForLogging(pathString)));
+ }
+
+ if (!file.isFile()) {
+ throw new SecurityException(
+ String.format("Maven settings path is not a file: %s",
sanitizeForLogging(pathString)));
+ }
+
+ if (!file.canRead()) {
+ throw new SecurityException(
+ String.format(
+ "Maven settings file is not readable: %s",
sanitizeForLogging(pathString)));
+ }
+
+ // Additional security: check file size to prevent DoS
+ if (file.length() > 10 * 1024 * 1024) { // 10MB limit
+ throw new SecurityException("Maven settings file is too large
(>10MB)");
+ }
+
+ logSecurityEvent(
+ "SUCCESS", "PATH_VALIDATION", "Maven settings path validation
completed", pathString);
+ return pathString;
+
+ } catch (InvalidPathException e) {
+ logSecurityEvent(
+ "VIOLATION", "PATH_SYNTAX_ERROR", "Invalid path syntax in settings
path", settingsPath);
+ throw new SecurityException("Invalid path syntax in settings path", e);
+ }
+ }
+
+ /**
+ * Performs final validation on the complete Maven command string.
+ *
+ * @param mvnArgs the complete Maven arguments string
+ * @throws SecurityException if validation fails
+ */
+ private static void validateFinalMavenCommand(String mvnArgs) {
+ logSecurityEvent(
+ "INFO", "FINAL_COMMAND_VALIDATION", "Starting final Maven command
validation", mvnArgs);
+
+ // Check for newline characters (existing validation)
+ if (mvnArgs.contains("\n") || mvnArgs.contains("\r")) {
+ logSecurityEvent(
+ "VIOLATION",
+ "CONTROL_CHARACTER_DETECTION",
+ "Control characters detected in maven command",
+ mvnArgs);
+ throw new SecurityException("Control characters detected in maven build
parameters");
+ }
+
+ // Additional validation using enhanced pattern matching
+ Matcher dangerousMatcher = COMMAND_INJECTION_PATTERN.matcher(mvnArgs);
+ if (dangerousMatcher.find()) {
+ String dangerousPattern = dangerousMatcher.group();
+ logSecurityEvent(
+ "VIOLATION",
+ "INJECTION_DETECTION",
+ "Command injection pattern in final Maven command",
+ dangerousPattern);
+ throw new SecurityException("Command injection pattern detected in final
Maven command");
+ }
+
+ // Validate total command length
+ if (mvnArgs.length() > MAX_BUILD_ARGS_LENGTH + 100) { // Allow some buffer
for default args
+ logSecurityEvent(
+ "VIOLATION",
+ "COMMAND_LENGTH_EXCEEDED",
+ String.format("Maven command exceeds maximum length: %d",
mvnArgs.length()),
+ mvnArgs);
+ throw new SecurityException("Maven command exceeds maximum allowed
length");
+ }
+
+ logSecurityEvent(
+ "SUCCESS", "FINAL_COMMAND_VALIDATION", "Final Maven command validation
completed", mvnArgs);
+ }
+
+ /**
+ * Securely determines the Maven executable with enhanced validation.
+ *
+ * @return validated Maven executable path
+ * @throws SecurityException if Maven executable validation fails
+ */
+ private static String getSecureMavenExecutable() {
+ boolean windows = Utils.isWindows();
+ String mvn = windows ? "mvn.cmd" : "mvn";
+
+ // Validate environment variables for potential injection
+ String mavenHome = validateMavenHomeEnvironment();
+
+ boolean useWrapper = true;
+ if (mavenHome != null) {
+ mvn = mavenHome + "/bin/" + mvn;
+ try {
+ // Test Maven installation with security considerations
+ ProcessBuilder pb = new ProcessBuilder(mvn, "--version");
+ pb.environment().clear(); // Clear environment to prevent injection
+ Process process = pb.start();
+ boolean finished = process.waitFor(10, TimeUnit.SECONDS);
+
+ if (finished && process.exitValue() == 0) {
+ useWrapper = false;
+ log.info("â
Validated system Maven installation: {}",
sanitizeForLogging(mvn));
+ } else {
+ log.warn("â ī¸ System Maven validation failed, using wrapper");
+ }
+ } catch (Exception e) {
+ log.warn("â ī¸ Maven validation error: {}, using wrapper",
e.getMessage());
+ }
+ }
+
+ if (useWrapper) {
+ String wrapperPath = WebUtils.getAppHome().concat(windows ?
"/bin/mvnw.cmd" : "/bin/mvnw");
+
+ // Validate wrapper executable exists and is secure
+ File wrapperFile = new File(wrapperPath);
+ if (!wrapperFile.exists() || !wrapperFile.canExecute()) {
+ throw new SecurityException("Maven wrapper not found or not
executable: " + wrapperPath);
+ }
+
+ mvn = wrapperPath;
+ log.info("â
Using secure Maven wrapper: {}", sanitizeForLogging(mvn));
+ }
+
+ return mvn;
+ }
+
+ /**
+ * Validates Maven home environment variables for security issues.
+ *
+ * @return validated Maven home path or null if not set/invalid
+ */
+ private static String validateMavenHomeEnvironment() {
+ String mavenHome = System.getenv("M2_HOME");
+ if (mavenHome == null) {
+ mavenHome = System.getenv("MAVEN_HOME");
+ }
+
+ if (mavenHome == null) {
+ return null;
+ }
+
+ try {
+ // Validate and normalize the Maven home path
+ Path normalizedPath = Paths.get(mavenHome).normalize();
+ String pathString = normalizedPath.toString();
+
+ // Security checks
+ if (pathString.contains("..") || pathString.contains("~")) {
+ log.warn("â ī¸ Suspicious Maven home path, ignoring: {}",
sanitizeForLogging(mavenHome));
+ return null;
+ }
+
+ File mavenDir = normalizedPath.toFile();
+ if (!mavenDir.exists() || !mavenDir.isDirectory()) {
+ log.warn("â ī¸ Invalid Maven home directory, ignoring: {}",
sanitizeForLogging(pathString));
+ return null;
+ }
+
+ return pathString;
+
+ } catch (InvalidPathException e) {
+ log.warn("â ī¸ Invalid Maven home path syntax, ignoring: {}",
sanitizeForLogging(mavenHome));
+ return null;
+ }
+ }
+
+ /**
+ * Logs security events for auditing and monitoring purposes. This method
provides comprehensive
+ * security logging for Maven build operations.
+ *
+ * @param eventType the type of security event (SUCCESS, WARNING, VIOLATION,
etc.)
+ * @param operation the operation being performed (BUILD_VALIDATION,
PATH_VALIDATION, etc.)
+ * @param details additional details about the event
+ * @param sensitiveData any sensitive data that should be sanitized for
logging
+ */
+ private static void logSecurityEvent(
+ String eventType, String operation, String details, String
sensitiveData) {
+ String sanitizedData = sanitizeForLogging(sensitiveData);
+
+ switch (eventType.toUpperCase()) {
+ case "SUCCESS":
+ log.info(
+ "â
[SECURITY-AUDIT] {} - {}: {} | Data: {}",
+ eventType,
+ operation,
+ details,
+ sanitizedData);
+ break;
+ case "WARNING":
+ log.warn(
+ "â ī¸ [SECURITY-AUDIT] {} - {}: {} | Data: {}",
+ eventType,
+ operation,
+ details,
+ sanitizedData);
+ break;
+ case "VIOLATION":
+ case "ERROR":
+ log.error(
+ "đ¨ [SECURITY-AUDIT] {} - {}: {} | Data: {}",
+ eventType,
+ operation,
+ details,
+ sanitizedData);
+ break;
+ default:
+ log.info(
+ "âšī¸ [SECURITY-AUDIT] {} - {}: {} | Data: {}",
+ eventType,
+ operation,
+ details,
+ sanitizedData);
+ }
+
+ // Additional structured logging for security monitoring systems
+ log.info(
+ "SECURITY_EVENT_JSON:
{{\"timestamp\":\"{}\",\"event_type\":\"{}\",\"operation\":\"{}\",\"details\":\"{}\",\"source\":\"Project.getMavenArgs\"}}",
+ Instant.now().toString(),
+ eventType,
+ operation,
+ details.replace("\"", "\\\""));
+ }
+
+ /**
+ * Comprehensive security validation for Maven build arguments. Implements
multiple layers of
+ * security checks to prevent command injection attacks.
+ *
+ * @param buildArgs the raw build arguments string
+ * @return sanitized and validated build arguments
+ * @throws SecurityException if dangerous patterns or parameters are detected
+ */
+ private static String validateAndSanitizeBuildArgs(String buildArgs) {
+ if (StringUtils.isBlank(buildArgs)) {
+ logSecurityEvent("SUCCESS", "BUILD_VALIDATION", "Empty build arguments
processed", "");
+ return "";
+ }
+
+ logSecurityEvent(
+ "INFO", "BUILD_VALIDATION", "Starting validation of build arguments",
buildArgs);
+
+ // 1. Length validation to prevent DoS attacks
+ if (buildArgs.length() > MAX_BUILD_ARGS_LENGTH) {
+ logSecurityEvent(
+ "VIOLATION",
+ "LENGTH_VALIDATION",
+ String.format(
+ "Build arguments exceed maximum length. Length: %d, Limit: %d",
+ buildArgs.length(), MAX_BUILD_ARGS_LENGTH),
+ buildArgs);
+ throw new SecurityException(
+ "Build arguments exceed maximum allowed length: " +
MAX_BUILD_ARGS_LENGTH);
+ }
+
+ // 2. Normalize Unicode and decode potential encoding attacks
+ String normalizedArgs = normalizeAndDecode(buildArgs);
+ if (!normalizedArgs.equals(buildArgs)) {
+ logSecurityEvent(
+ "WARNING",
+ "ENCODING_DETECTION",
+ "Encoding or Unicode normalization applied to build arguments",
+ String.format("Original: %s | Normalized: %s", buildArgs,
normalizedArgs));
+ }
+
+ // 3. Advanced command injection pattern detection
+ Matcher dangerousMatcher =
COMMAND_INJECTION_PATTERN.matcher(normalizedArgs);
+ if (dangerousMatcher.find()) {
+ String dangerousPattern = dangerousMatcher.group();
+ logSecurityEvent(
+ "VIOLATION",
+ "INJECTION_DETECTION",
+ "Command injection pattern detected in build arguments",
+ String.format("Pattern: %s | Full args: %s", dangerousPattern,
normalizedArgs));
+ throw new SecurityException(
+ "Dangerous command injection pattern detected: " +
sanitizeForLogging(dangerousPattern));
+ }
+
+ // 4. Whitelist-based argument validation
+ String validatedArgs = validateArgumentsAgainstWhitelist(normalizedArgs);
+ logSecurityEvent(
+ "SUCCESS",
+ "BUILD_VALIDATION",
+ "Build arguments validation completed successfully",
+ validatedArgs);
+ return validatedArgs;
+ }
+
+ /**
+ * Normalizes Unicode characters and decodes potential encoding attacks.
+ *
+ * @param input the input string to normalize
+ * @return normalized string
+ */
+ private static String normalizeAndDecode(String input) {
+ // Normalize Unicode characters to prevent Unicode-based attacks
+ String normalized = Normalizer.normalize(input, Normalizer.Form.NFC);
+
+ // Basic URL decode to catch simple encoding attempts
+ normalized =
+ normalized
+ .replace("%20", " ")
+ .replace("%3B", ";")
+ .replace("%7C", "|")
+ .replace("%26", "&")
+ .replace("%3E", ">")
+ .replace("%3C", "<")
+ .replace("%24", "$")
+ .replace("%60", "`");
+
+ return normalized;
+ }
+
+ /**
+ * Validates Maven arguments against predefined whitelists.
+ *
+ * @param args the normalized arguments string
+ * @return validated arguments string
+ * @throws SecurityException if invalid arguments are found
+ */
+ private static String validateArgumentsAgainstWhitelist(String args) {
+ String[] argArray = args.trim().split("\\s+");
+ StringBuilder validatedArgs = new StringBuilder();
+
+ for (int i = 0; i < argArray.length; i++) {
+ String arg = argArray[i].trim();
+
+ if (StringUtils.isBlank(arg)) {
+ continue;
+ }
+
+ // Validate individual argument length
+ if (arg.length() > MAX_MAVEN_ARG_LENGTH) {
+ log.error(
+ "đ¨ Security Alert: Individual argument exceeds length limit: {}",
+ sanitizeForLogging(arg));
+ throw new SecurityException(
+ "Individual argument exceeds maximum length: " +
MAX_MAVEN_ARG_LENGTH);
+ }
+
+ if (arg.startsWith("-D")) {
+ // Handle system property definitions
+ validateSystemProperty(arg, i < argArray.length - 1 ? argArray[i + 1]
: null);
+ validatedArgs.append(arg).append(" ");
+
+ // Skip next argument if it's the value for this -D parameter
+ if (i < argArray.length - 1 && !argArray[i + 1].startsWith("-")) {
+ i++; // Skip the value part
+ validatedArgs.append(argArray[i]).append(" ");
+ }
+ } else if (arg.startsWith("--define")) {
+ // Handle long form system property definitions
+ validateSystemProperty(arg, null);
+ validatedArgs.append(arg).append(" ");
+ } else if (ALLOWED_MAVEN_ARGS.contains(arg)) {
+ // Standard Maven argument
+ validatedArgs.append(arg).append(" ");
+ } else {
+ // Check if it's a Maven lifecycle phase or goal
+ if (isValidMavenPhaseOrGoal(arg)) {
+ validatedArgs.append(arg).append(" ");
+ } else {
+ logSecurityEvent(
+ "VIOLATION", "WHITELIST_VALIDATION", "Unauthorized Maven
argument detected", arg);
+ throw new SecurityException("Unauthorized Maven argument: " +
sanitizeForLogging(arg));
+ }
+ }
+ }
+
+ return validatedArgs.toString().trim();
+ }
+
+ /**
+ * Validates system property arguments (-D parameters).
+ *
+ * @param arg the system property argument
+ * @param nextArg the next argument (value) if applicable
+ * @throws SecurityException if invalid system property is detected
+ */
+ private static void validateSystemProperty(String arg, String nextArg) {
+ String propertyDefinition = arg;
+
+ if (arg.equals("-D") && nextArg != null) {
+ propertyDefinition = nextArg;
+ } else if (arg.startsWith("-D")) {
+ propertyDefinition = arg.substring(2);
+ } else if (arg.startsWith("--define=")) {
+ propertyDefinition = arg.substring(9);
+ }
+
+ // Extract property name (before = sign)
+ String propertyName = propertyDefinition.split("=")[0];
+
+ if (!ALLOWED_SYSTEM_PROPERTIES.contains(propertyName)) {
+ // Allow some common patterns but be restrictive
+ if (!isValidSystemPropertyPattern(propertyName)) {
+ logSecurityEvent(
+ "VIOLATION",
+ "SYSTEM_PROPERTY_VALIDATION",
+ "Unauthorized system property detected",
+ propertyName);
+ throw new SecurityException(
+ "Unauthorized system property: " +
sanitizeForLogging(propertyName));
+ }
+ }
+ }
+
+ /**
+ * Checks if the property name follows safe patterns.
+ *
+ * @param propertyName the property name to validate
+ * @return true if the property pattern is considered safe
+ */
+ private static boolean isValidSystemPropertyPattern(String propertyName) {
+ // Allow properties that start with safe prefixes
+ List<String> safePatterns =
+ Arrays.asList(
+ "maven.",
+ "project.",
+ "flink.",
+ "scala.",
+ "hadoop.",
+ "kafka.",
+ "java.",
+ "user.",
+ "file.",
+ "encoding");
+
+ return safePatterns.stream().anyMatch(propertyName::startsWith)
+ && propertyName.matches("^[a-zA-Z0-9._-]+$"); // Only allow safe
characters
+ }
+
+ /**
+ * Checks if an argument is a valid Maven lifecycle phase or goal.
+ *
+ * @param arg the argument to check
+ * @return true if it's a valid Maven phase or goal
+ */
+ private static boolean isValidMavenPhaseOrGoal(String arg) {
+ // Basic validation: only allow alphanumeric, hyphens, and colons (for
plugin goals)
+ return arg.matches("^[a-zA-Z0-9:._-]+$") && arg.length() <= 50;
+ }
+
+ /**
+ * Sanitizes sensitive information for safe logging.
+ *
+ * @param input the input to sanitize
+ * @return sanitized string safe for logging
+ */
+ private static String sanitizeForLogging(String input) {
+ if (input == null) {
+ return "null";
+ }
+
+ // Mask potential sensitive patterns and limit length
+ String sanitized =
+ input
+ .replaceAll("(password|pwd|secret|token|key)=\\S*", "$1=***")
+ .replaceAll("(`[^`]*`)", "***BACKTICK_COMMAND***")
+ .replaceAll("(\\$\\([^)]*\\))", "***COMMAND_SUBSTITUTION***")
+ .replaceAll("(\\$\\{[^}]*})", "***VARIABLE_SUBSTITUTION***");
+
+ // Limit length for logging
+ if (sanitized.length() > 100) {
+ sanitized = sanitized.substring(0, 100) + "...";
+ }
+
+ return sanitized;
+ }
+}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
index 1694d9bdb..549ba5a9a 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
@@ -17,13 +17,8 @@
package org.apache.streampark.console.core.entity;
-import org.apache.streampark.common.conf.CommonConfig;
-import org.apache.streampark.common.conf.InternalConfigHolder;
import org.apache.streampark.common.conf.Workspace;
-import org.apache.streampark.common.util.AssertUtils;
-import org.apache.streampark.common.util.Utils;
-import org.apache.streampark.console.base.util.CommonUtils;
-import org.apache.streampark.console.base.util.WebUtils;
+import org.apache.streampark.console.base.util.MavenUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -42,11 +37,7 @@ import org.eclipse.jgit.lib.Constants;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.util.Arrays;
import java.util.Date;
-import java.util.Iterator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
@Slf4j
@Getter
@@ -107,8 +98,9 @@ public class Project implements Serializable {
@JsonIgnore
public File getAppSource() {
File sourcePath = new File(Workspace.PROJECT_LOCAL_PATH());
- if (!sourcePath.exists()) {
- sourcePath.mkdirs();
+ if (!sourcePath.exists() && !sourcePath.mkdirs()) {
+ throw new IllegalStateException(
+ "Failed to create project source path: " +
sourcePath.getAbsolutePath());
} else if (sourcePath.isFile()) {
throw new IllegalArgumentException(
"[StreamPark] project source base path: "
@@ -117,18 +109,12 @@ public class Project implements Serializable {
}
String sourceDir = getSourceDirName();
- File srcFile =
- new File(String.format("%s/%s/%s", sourcePath.getAbsolutePath(), name,
sourceDir));
String newPath = String.format("%s/%s", sourcePath.getAbsolutePath(), id);
- if (srcFile.exists()) {
- File newFile = new File(newPath);
- if (!newFile.exists()) {
- newFile.mkdirs();
- }
- // old project path move to new path
- srcFile.getParentFile().renameTo(newFile);
+ File file = new File(newPath, sourceDir);
+ if (!file.exists() && !file.mkdirs()) {
+ throw new IllegalStateException("Failed to create directory: " +
file.getAbsolutePath());
}
- return new File(newPath, sourceDir);
+ return file;
}
private String getSourceDirName() {
@@ -179,108 +165,20 @@ public class Project implements Serializable {
}
}
- @JsonIgnore
- public String getMavenArgs() {
- StringBuilder mvnArgBuffer = new StringBuilder(" clean package -DskipTests
");
- if (StringUtils.isNotBlank(this.buildArgs)) {
- mvnArgBuffer.append(this.buildArgs.trim());
- }
-
- // --settings
- String setting =
InternalConfigHolder.get(CommonConfig.MAVEN_SETTINGS_PATH());
- if (StringUtils.isNotBlank(setting)) {
- File file = new File(setting);
- if (file.exists() && file.isFile()) {
- mvnArgBuffer.append(" --settings ").append(setting.trim());
- } else {
- throw new IllegalArgumentException(
- String.format(
- "Invalid maven-setting file path \"%s\", the path not exist or
is not file",
- setting));
- }
- }
-
- // check maven args
- String mvnArgs = mvnArgBuffer.toString();
- if (mvnArgs.contains("\n")) {
- throw new IllegalArgumentException(
- String.format(
- "Illegal argument: newline character in maven build parameters:
\"%s\"", mvnArgs));
- }
-
- String args = getIllegalArgs(mvnArgs);
- if (args != null) {
- throw new IllegalArgumentException(
- String.format("Illegal argument: \"%s\" in maven build parameters:
%s", args, mvnArgs));
- }
-
- // find mvn
- boolean windows = Utils.isWindows();
- String mvn = windows ? "mvn.cmd" : "mvn";
-
- String mavenHome = System.getenv("M2_HOME");
- if (mavenHome == null) {
- mavenHome = System.getenv("MAVEN_HOME");
- }
-
- boolean useWrapper = true;
- if (mavenHome != null) {
- mvn = mavenHome + "/bin/" + mvn;
- try {
- Process process = Runtime.getRuntime().exec(mvn + " --version");
- process.waitFor();
- AssertUtils.required(process.exitValue() == 0);
- useWrapper = false;
- } catch (Exception ignored) {
- log.warn("try using user-installed maven failed, now use
maven-wrapper.");
- }
- }
-
- if (useWrapper) {
- if (windows) {
- mvn = WebUtils.getAppHome().concat("/bin/mvnw.cmd");
- } else {
- mvn = WebUtils.getAppHome().concat("/bin/mvnw");
- }
- }
- return mvn.concat(mvnArgs);
- }
-
- private String getIllegalArgs(String param) {
- Pattern pattern = Pattern.compile("(`(.?|\\s)*`)|(\\$\\((.?|\\s)*\\))");
- Matcher matcher = pattern.matcher(param);
- if (matcher.find()) {
- return matcher.group(1) == null ? matcher.group(3) : matcher.group(1);
- }
-
- Iterator<String> iterator = Arrays.asList(";", "|", "&", ">",
"<").iterator();
- String[] argsList = param.split("\\s+");
- while (iterator.hasNext()) {
- String chr = iterator.next();
- for (String arg : argsList) {
- if (arg.contains(chr)) {
- return arg;
- }
- }
- }
- return null;
- }
-
@JsonIgnore
public String getMavenWorkHome() {
String buildHome = this.getAppSource().getAbsolutePath();
- if (CommonUtils.notEmpty(this.getPom())) {
- buildHome =
- new
File(buildHome.concat("/").concat(this.getPom())).getParentFile().getAbsolutePath();
+ if (StringUtils.isBlank(this.getPom())) {
+ return buildHome;
}
- return buildHome;
+ return new
File(buildHome.concat("/").concat(this.getPom())).getParentFile().getAbsolutePath();
}
@JsonIgnore
public String getLog4BuildStart() {
return String.format(
"%sproject : %s\nrefs: %s\ncommand : %s\n\n",
- getLogHeader("maven install"), getName(), getRefs(), getMavenArgs());
+ getLogHeader("maven install"), getName(), getRefs(),
this.getMavenBuildArgs());
}
@JsonIgnore
@@ -294,4 +192,9 @@ public class Project implements Serializable {
private String getLogHeader(String header) {
return "---------------------------------[ " + header + "
]---------------------------------\n";
}
+
+ @JsonIgnore
+ public String getMavenBuildArgs() {
+ return MavenUtils.getMavenArgs(this.buildArgs);
+ }
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
index 329f47247..88415626e 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
@@ -23,204 +23,822 @@ import org.apache.streampark.console.base.util.GitUtils;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildState;
+import org.apache.commons.lang3.StringUtils;
+
import ch.qos.logback.classic.Logger;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.lib.StoredConfig;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;
+import java.util.regex.Pattern;
+/**
+ * This class handles the complete lifecycle of project building including: -
Git repository
+ * cloning/pulling - Maven/Gradle build execution - Artifact deployment and
validation -
+ * Comprehensive logging and monitoring
+ */
@Slf4j
public class ProjectBuildTask extends AbstractLogFileTask {
- final Project project;
+ // ========== Constants ==========
+
+ private static final Duration CLONE_TIMEOUT = Duration.ofMinutes(10);
+ private static final String BUILD_START_MARKER = "=== BUILD STARTED ===";
+ private static final String BUILD_END_MARKER = "=== BUILD COMPLETED ===";
+ private static final Pattern SENSITIVE_INFO_PATTERN =
+ Pattern.compile("password|token|key|secret", Pattern.CASE_INSENSITIVE);
- final Consumer<BuildState> stateUpdateConsumer;
+ // ========== Fields ==========
- final Consumer<Logger> notifyReleaseConsumer;
+ private final Project project;
+ private final Consumer<BuildState> stateUpdateConsumer;
+ private final Consumer<Logger> notifyReleaseConsumer;
+ private final LocalDateTime startTime = LocalDateTime.now();
+ // Build statistics
+ private LocalDateTime cloneStartTime;
+ private LocalDateTime buildStartTime;
+ private LocalDateTime deployStartTime;
+
+ /**
+ * Creates a new ProjectBuildTask with enhanced validation and configuration.
+ *
+ * @param logPath log file path for build output
+ * @param project project to build (must not be null)
+ * @param stateUpdateConsumer callback for build state updates (must not be
null)
+ * @param notifyReleaseConsumer callback for release notifications (must not
be null)
+ * @throws IllegalArgumentException if any parameter is null or invalid
+ */
public ProjectBuildTask(
String logPath,
Project project,
Consumer<BuildState> stateUpdateConsumer,
Consumer<Logger> notifyReleaseConsumer) {
super(logPath, true);
- this.project = project;
- this.stateUpdateConsumer = stateUpdateConsumer;
- this.notifyReleaseConsumer = notifyReleaseConsumer;
+
+ // Enhanced parameter validation
+ this.project = validateProject(project);
+ this.stateUpdateConsumer =
+ Objects.requireNonNull(stateUpdateConsumer, "State update consumer
cannot be null");
+ this.notifyReleaseConsumer =
+ Objects.requireNonNull(notifyReleaseConsumer, "Notify release consumer
cannot be null");
+
+ // Validate log path
+ validateLogPath(logPath);
+
+ log.info(
+ "ProjectBuildTask initialized for project: {} (ID: {})",
+ project.getName(),
+ project.getId());
}
+ // ========== Main Execution Flow ==========
+
@Override
protected void doRun() throws Throwable {
- log.info("Project {} start build", project.getName());
- fileLogger.info(project.getLog4BuildStart());
- boolean cloneSuccess = cloneSourceCode(project);
- if (!cloneSuccess) {
- fileLogger.error("[StreamPark] clone or pull error.");
- stateUpdateConsumer.accept(BuildState.FAILED);
- return;
- }
- boolean build = projectBuild(project);
- if (!build) {
- stateUpdateConsumer.accept(BuildState.FAILED);
- fileLogger.error("build error, project name: {} ", project.getName());
- return;
+ try {
+ logBuildStart();
+ updateBuildState(BuildState.BUILDING);
+
+ // Phase 1: Clone source code with retry mechanism
+ if (!execute("clone", this::performClone)) {
+ handleBuildFailure("Failed to clone source code");
+ return;
+ }
+
+ // Phase 2: Execute build with timeout and monitoring
+ if (!execute("build", this::performBuild)) {
+ handleBuildFailure("Failed to build project");
+ return;
+ }
+
+ // Phase 3: Deploy artifacts
+ if (!execute("deploy", this::performDeploy)) {
+ handleBuildFailure("Failed to deploy artifacts");
+ return;
+ }
+
+ // Success
+ handleBuildSuccess();
+
+ } catch (Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ handleBuildInterruption();
+ } else {
+ handleBuildFailure("Unexpected error during build: " + e.getMessage(),
e);
+ }
}
- stateUpdateConsumer.accept(BuildState.SUCCESSFUL);
- this.deploy(project);
- notifyReleaseConsumer.accept(fileLogger);
}
@Override
protected void processException(Throwable t) {
- stateUpdateConsumer.accept(BuildState.FAILED);
- fileLogger.error("Build error, project name: {}", project.getName(), t);
+ log.error("Build task exception for project: {}", project.getName(), t);
+ updateBuildState(BuildState.FAILED);
+ if (t instanceof Exception) {
+ logBuildError("Build failed with exception", (Exception) t);
+ } else {
+ fileLogger.error("Build failed with throwable: {}", t.getMessage(), t);
+ }
}
@Override
- protected void doFinally() {}
+ protected void doFinally() {
+ try {
+ cleanupResources();
+ logBuildSummary();
+ } catch (Exception e) {
+ log.warn("Error during cleanup for project: {}", project.getName(), e);
+ }
+ }
+
+ // ========== Clone Phase Implementation ==========
+
+ private boolean performClone() throws Exception {
+ cloneStartTime = LocalDateTime.now();
+
+ try {
+ validatePreCloneConditions();
+ prepareCloneDirectory();
+
+ boolean success = executeGitClone();
+ if (success) {
+ logCloneSuccess();
+ }
+
+ return success;
- private boolean cloneSourceCode(Project project) {
+ } catch (Exception e) {
+ logCloneError("Git clone operation failed", e);
+ throw e;
+ }
+ }
+
+ private void validatePreCloneConditions() throws IllegalStateException {
+ if (StringUtils.isBlank(project.getUrl())) {
+ throw new IllegalStateException("Project URL cannot be blank");
+ }
+
+ if (project.getAppSource() == null || !project.getAppSource().exists()) {
+ throw new IllegalStateException("Project source directory does not
exist");
+ }
+
+ // Validate URL format (basic validation)
+ if (!project.getUrl().startsWith("http") &&
!project.getUrl().startsWith("git@")) {
+ throw new IllegalStateException("Invalid Git URL format: " +
sanitizeUrl(project.getUrl()));
+ }
+ }
+
+ private void prepareCloneDirectory() throws IOException {
try {
project.cleanCloned();
- fileLogger.info("clone {}, {} starting...", project.getName(),
project.getUrl());
+
+ // Ensure parent directory exists
+ Path sourcePath = project.getAppSource().toPath();
+ Path parentDir = sourcePath.getParent();
+ if (parentDir != null && !Files.exists(parentDir)) {
+ Files.createDirectories(parentDir);
+ fileLogger.info("Created parent directory: {}", parentDir);
+ }
+
+ } catch (Exception e) {
+ throw new IOException("Failed to prepare clone directory: " +
e.getMessage(), e);
+ }
+ }
+
+ private boolean executeGitClone() {
+ Git git = null;
+ try {
+ fileLogger.info("Starting Git clone for project: {}", project.getName());
+ fileLogger.info("Repository URL: {}", sanitizeUrl(project.getUrl()));
+ fileLogger.info("Target directory: {}", project.getAppSource());
+ fileLogger.info("Branch/Tag: {}",
StringUtils.defaultIfBlank(project.getRefs(), "default"));
fileLogger.info(project.getLog4CloneStart());
- GitUtils.GitCloneRequest request = new GitUtils.GitCloneRequest();
- request.setUrl(project.getUrl());
- request.setRefs(project.getRefs());
- request.setStoreDir(project.getAppSource());
- request.setUsername(project.getUserName());
- request.setPassword(project.getPassword());
- request.setPrivateKey(project.getPrvkeyPath());
-
- Git git = GitUtils.clone(request);
- StoredConfig config = git.getRepository().getConfig();
- config.setBoolean("http", project.getUrl(), "sslVerify", false);
- config.setBoolean("https", project.getUrl(), "sslVerify", false);
- config.save();
- File workTree = git.getRepository().getWorkTree();
- printWorkTree(workTree, "");
- String successMsg =
- String.format("[StreamPark] project [%s] git clone successful!\n",
project.getName());
- fileLogger.info(successMsg);
- git.close();
+ GitUtils.GitCloneRequest request = buildGitCloneRequest();
+ git = GitUtils.clone(request);
+
+ configureGitRepository(git);
+ validateCloneContent(git);
+
return true;
+
} catch (Exception e) {
- fileLogger.error(
- String.format(
- "[StreamPark] project [%s] refs [%s] git clone failed, err: %s",
- project.getName(), project.getRefs(), e));
- fileLogger.error(String.format("project %s clone error ",
project.getName()), e);
+ logCloneError("Git clone failed", e);
return false;
+ } finally {
+ if (git != null) {
+ try {
+ git.close();
+ } catch (Exception e) {
+ log.warn("Failed to close Git repository: {}", e.getMessage());
+ }
+ }
}
}
- private void printWorkTree(File workTree, String space) {
+ private GitUtils.GitCloneRequest buildGitCloneRequest() {
+ GitUtils.GitCloneRequest request = new GitUtils.GitCloneRequest();
+ request.setUrl(project.getUrl());
+ request.setRefs(project.getRefs());
+ request.setStoreDir(project.getAppSource());
+ request.setUsername(project.getUserName());
+ request.setPassword(project.getPassword());
+ request.setPrivateKey(project.getPrvkeyPath());
+ return request;
+ }
+
+ private void configureGitRepository(Git git) throws Exception {
+ StoredConfig config = git.getRepository().getConfig();
+ String url = project.getUrl();
+
+ // Disable SSL verification for HTTP/HTTPS URLs
+ config.setBoolean("http", url, "sslVerify", false);
+ config.setBoolean("https", url, "sslVerify", false);
+
+ // Set timeout configurations
+ config.setInt("http", url, "timeout", (int) CLONE_TIMEOUT.getSeconds());
+ config.setInt("https", url, "timeout", (int) CLONE_TIMEOUT.getSeconds());
+
+ config.save();
+ fileLogger.info("Git repository configuration updated successfully");
+ }
+
+ private void validateCloneContent(Git git) {
+ File workTree = git.getRepository().getWorkTree();
+
+ if (!workTree.exists() || !workTree.isDirectory()) {
+ throw new IllegalStateException("Clone directory does not exist or is
not a directory");
+ }
+
File[] files = workTree.listFiles();
- for (File file : Objects.requireNonNull(files)) {
- if (!file.getName().startsWith(".git")) {
- continue;
- }
- if (file.isFile()) {
- fileLogger.info("{} / {}", space, file.getName());
- } else if (file.isDirectory()) {
- fileLogger.info("{} / {}", space, file.getName());
- printWorkTree(file, space.concat("/").concat(file.getName()));
+ if (files == null || files.length == 0) {
+ throw new IllegalStateException("Cloned repository is empty");
+ }
+
+ // Log directory structure (limited depth for security)
+ logDirectoryStructure(workTree, "", 0, 3);
+
+ fileLogger.info("Clone validation completed. Found {} files/directories",
files.length);
+ }
+
+ // ========== Build Phase Implementation ==========
+
+ private boolean performBuild() throws Exception {
+ buildStartTime = LocalDateTime.now();
+
+ try {
+ validatePreBuildConditions();
+ boolean success = executeMavenBuild();
+
+ if (success) {
+ Duration buildDuration = Duration.between(buildStartTime,
LocalDateTime.now());
+ fileLogger.info("Maven build completed successfully in {}",
formatDuration(buildDuration));
}
+
+ return success;
+
+ } catch (Exception e) {
+ logBuildError("Maven build failed", e);
+ throw e;
+ }
+ }
+
+ private void validatePreBuildConditions() throws IllegalStateException {
+ String mavenWorkHome = project.getMavenWorkHome();
+ if (StringUtils.isBlank(mavenWorkHome)) {
+ throw new IllegalStateException("Maven work home cannot be blank");
+ }
+
+ File workDir = new File(mavenWorkHome);
+ if (!workDir.exists()) {
+ throw new IllegalStateException("Maven work directory does not exist: "
+ mavenWorkHome);
+ }
+
+ // Check for pom.xml or build.gradle
+ File pomFile = new File(workDir, "pom.xml");
+ File gradleFile = new File(workDir, "build.gradle");
+ if (!pomFile.exists() && !gradleFile.exists()) {
+ throw new IllegalStateException(
+ "No build file (pom.xml or build.gradle) found in: " +
mavenWorkHome);
+ }
+
+ String mavenArgs = project.getMavenBuildArgs();
+ if (StringUtils.isBlank(mavenArgs)) {
+ throw new IllegalStateException("Maven arguments cannot be blank");
+ }
+ }
+
+ private boolean executeMavenBuild() {
+ try {
+ fileLogger.info(BUILD_START_MARKER);
+ fileLogger.info("đ¨ Starting Maven build for project: {}",
project.getName());
+ fileLogger.info(" đ Working directory: {}",
project.getMavenWorkHome());
+ fileLogger.info(
+ " âī¸ Build command: {}",
sanitizeBuildCommand(project.getMavenBuildArgs()));
+
+ int exitCode =
+ CommandUtils.execute(
+ project.getMavenWorkHome(),
+ Collections.singletonList(project.getMavenBuildArgs()),
+ this::logBuildOutput);
+
+ fileLogger.info("Maven build completed with exit code: {}", exitCode);
+ fileLogger.info(BUILD_END_MARKER);
+
+ return exitCode == 0;
+
+ } catch (Exception e) {
+ fileLogger.error("Maven build execution failed: {}", e.getMessage());
+ return false;
}
}
- private boolean projectBuild(Project project) {
- int code =
- CommandUtils.execute(
- project.getMavenWorkHome(),
- Collections.singletonList(project.getMavenArgs()),
- (line) -> fileLogger.info(line));
- return code == 0;
+ private void logBuildOutput(String line) {
+ if (StringUtils.isNotBlank(line)) {
+ // Filter sensitive information
+ String sanitizedLine = sanitizeBuildOutput(line);
+ fileLogger.info(sanitizedLine);
+ }
}
- private void deploy(Project project) throws Exception {
- File path = project.getAppSource();
- List<File> apps = new ArrayList<>();
- // find the compiled tar.gz (Stream Park project) file or jar (normal or
official standard flink
- // project) under the project path
- findTarOrJar(apps, path);
- if (apps.isEmpty()) {
+ // ========== Deploy Phase Implementation ==========
+
+ private boolean performDeploy() throws Exception {
+ deployStartTime = LocalDateTime.now();
+
+ try {
+ validatePreDeployConditions();
+ deployBuildArtifacts();
+ validateDeploymentResult();
+ logDeploySuccess();
+
+ return true;
+
+ } catch (Exception e) {
+ logDeployError("Deployment failed", e);
+ throw e;
+ }
+ }
+
+ private void validatePreDeployConditions() throws IllegalStateException {
+ File sourceDir = project.getAppSource();
+ if (sourceDir == null || !sourceDir.exists()) {
+ throw new IllegalStateException("Source directory does not exist");
+ }
+
+ File distHome = project.getDistHome();
+ if (distHome == null) {
+ throw new IllegalStateException("Distribution home directory is not
configured");
+ }
+ }
+
+ private void deployBuildArtifacts() throws Exception {
+ File sourcePath = project.getAppSource();
+ List<File> artifacts = new ArrayList<>();
+
+ // Find build artifacts with improved algorithm
+ findBuildArtifacts(artifacts, sourcePath, 0, 5); // Limit depth to 5
+
+ if (artifacts.isEmpty()) {
throw new RuntimeException(
- "[StreamPark] can't find tar.gz or jar in " +
path.getAbsolutePath());
- }
- for (File app : apps) {
- String appPath = app.getAbsolutePath();
- // 1). tar.gz file
- if (appPath.endsWith("tar.gz")) {
- File deployPath = project.getDistHome();
- if (!deployPath.exists()) {
- deployPath.mkdirs();
- }
- // xzvf jar
- if (app.exists()) {
- String cmd =
- String.format(
- "tar -xzvf %s -C %s", app.getAbsolutePath(),
deployPath.getAbsolutePath());
- CommandUtils.execute(cmd);
- }
- } else {
- // 2) .jar file(normal or official standard flink project)
- Utils.checkJarFile(app.toURI().toURL());
- String moduleName = app.getName().replace(".jar", "");
- File distHome = project.getDistHome();
- File targetDir = new File(distHome, moduleName);
- if (!targetDir.exists()) {
- targetDir.mkdirs();
- }
- File targetJar = new File(targetDir, app.getName());
- app.renameTo(targetJar);
+ "No deployable artifacts (*.tar.gz or *.jar) found in project
directory: "
+ + sourcePath.getAbsolutePath());
+ }
+
+ fileLogger.info("đ Found {} deployable artifact(s)", artifacts.size());
+
+ for (File artifact : artifacts) {
+ deployArtifact(artifact);
+ }
+ }
+
+ private void deployArtifact(File artifact) throws Exception {
+ String artifactPath = artifact.getAbsolutePath();
+ fileLogger.info("đĻ Deploying artifact: {}", artifactPath);
+
+ if (artifactPath.endsWith(".tar.gz")) {
+ deployTarGzArtifact(artifact);
+ } else if (artifactPath.endsWith(".jar")) {
+ deployJarArtifact(artifact);
+ } else {
+ throw new IllegalArgumentException("Unsupported artifact type: " +
artifactPath);
+ }
+
+ fileLogger.info("â
Successfully deployed artifact: {}",
artifact.getName());
+ }
+
+ private void deployTarGzArtifact(File tarGzFile) throws Exception {
+ File deployPath = project.getDistHome();
+ ensureDirectoryExists(deployPath);
+
+ if (!tarGzFile.exists()) {
+ throw new IllegalStateException("Tar.gz file does not exist: " +
tarGzFile.getAbsolutePath());
+ }
+
+ String extractCommand =
+ String.format(
+ "tar -xzf %s -C %s", tarGzFile.getAbsolutePath(),
deployPath.getAbsolutePath());
+
+ fileLogger.info("đĻ Extracting tar.gz: {}", extractCommand);
+
+ CommandUtils.execute(extractCommand);
+ }
+
+ private void deployJarArtifact(File jarFile) throws Exception {
+ // Validate JAR file integrity
+ Utils.checkJarFile(jarFile.toURI().toURL());
+
+ String moduleName = jarFile.getName().replace(".jar", "");
+ File distHome = project.getDistHome();
+ File targetDir = new File(distHome, moduleName);
+
+ ensureDirectoryExists(targetDir);
+
+ File targetJar = new File(targetDir, jarFile.getName());
+
+ // Use Files.move for atomic operation and better error handling
+ try {
+ Files.move(jarFile.toPath(), targetJar.toPath());
+ fileLogger.info("đĻ JAR artifact moved to: {}",
targetJar.getAbsolutePath());
+ } catch (IOException e) {
+ throw new IOException("Failed to move JAR artifact: " + e.getMessage(),
e);
+ }
+ }
+
+ private void validateDeploymentResult() throws Exception {
+ File distHome = project.getDistHome();
+ if (!distHome.exists()) {
+ throw new IllegalStateException("Deployment directory was not created");
+ }
+
+ File[] deployedFiles = distHome.listFiles();
+ if (deployedFiles == null || deployedFiles.length == 0) {
+ throw new IllegalStateException("No files were deployed");
+ }
+
+ fileLogger.info("â
Deployment validation completed. {} items deployed",
deployedFiles.length);
+ }
+
+ // ========== Utility Methods ==========
+
+ private boolean execute(String operationName, ExecuteOperation operation) {
+ try {
+ boolean success = operation.execute();
+ if (success) {
+ fileLogger.info("{} operation succeeded", operationName);
+ return true;
}
+ fileLogger.warn("{} operation failed", operationName);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ fileLogger.error("{} operation interrupted", operationName);
+ return false;
+ } catch (Exception e) {
+ fileLogger.error("{} operation failed: {}", operationName,
e.getMessage());
}
+ return false;
}
- private void findTarOrJar(List<File> list, File path) {
- for (File file : path.listFiles()) {
- // navigate to the target directory:
+ private void findBuildArtifacts(
+ List<File> artifacts, File directory, int currentDepth, int maxDepth) {
+ if (currentDepth >= maxDepth || !directory.isDirectory()) {
+ return;
+ }
+
+ File[] files = directory.listFiles();
+ if (files == null) {
+ return;
+ }
+
+ for (File file : files) {
if (file.isDirectory()) {
if ("target".equals(file.getName())) {
- // find the tar.gz file or the jar file in the target path.
- // note: only one of the two can be selected, which cannot be
satisfied at the same time.
- File tar = null;
- File jar = null;
- for (File targetFile : file.listFiles()) {
- // 1) exit once the tar.gz file is found.
- if (targetFile.getName().endsWith("tar.gz")) {
- tar = targetFile;
- break;
- }
- // 2) try look for jar files, there may be multiple jars found.
- if (!targetFile.getName().startsWith("original-")
- && !targetFile.getName().endsWith("-sources.jar")
- && targetFile.getName().endsWith(".jar")) {
- if (jar == null) {
- jar = targetFile;
- } else {
- if (targetFile.length() > jar.length()) {
- jar = targetFile;
- }
- }
- }
- }
- File target = tar == null ? jar : tar;
- if (target != null) {
- list.add(target);
- }
- } else {
- findTarOrJar(list, file);
+ findArtifactsInTargetDirectory(artifacts, file);
+ } else if (!file.getName().startsWith(".")) {
+ // Recursive search in non-hidden directories
+ findBuildArtifacts(artifacts, file, currentDepth + 1, maxDepth);
+ }
+ }
+ }
+ }
+
+ private void findArtifactsInTargetDirectory(List<File> artifacts, File
targetDir) {
+ File[] files = targetDir.listFiles();
+ if (files == null) {
+ return;
+ }
+
+ File tarGzFile = null;
+ File jarFile = null;
+
+ for (File file : files) {
+ String fileName = file.getName();
+
+ // Priority 1: tar.gz files
+ if (fileName.endsWith(".tar.gz")) {
+ tarGzFile = file;
+ break; // tar.gz has highest priority
+ }
+
+ // Priority 2: JAR files (excluding sources and original)
+ if (fileName.endsWith(".jar")
+ && !fileName.startsWith("original-")
+ && !fileName.endsWith("-sources.jar")
+ && !fileName.endsWith("-javadoc.jar")) {
+
+ if (jarFile == null || file.length() > jarFile.length()) {
+ jarFile = file; // Keep the largest JAR
}
}
}
+
+ File selectedArtifact = tarGzFile != null ? tarGzFile : jarFile;
+ if (selectedArtifact != null) {
+ artifacts.add(selectedArtifact);
+ fileLogger.info(
+ "đ Found build artifact: {} ({})",
+ selectedArtifact.getName(),
+ formatFileSize(selectedArtifact.length()));
+ }
+ }
+
+ private void logDirectoryStructure(
+ File directory, String indent, int currentDepth, int maxDepth) {
+ if (currentDepth >= maxDepth || !directory.isDirectory()) {
+ return;
+ }
+
+ File[] files = directory.listFiles();
+ if (files == null) {
+ return;
+ }
+
+ for (File file : files) {
+ // Only show git-related files and important build files
+ if (file.getName().startsWith(".git")
+ || file.getName().equals("pom.xml")
+ || file.getName().equals("build.gradle")
+ || file.getName().equals("target")
+ || file.getName().equals("src")) {
+
+ String type = file.isDirectory() ? "/" : "";
+ fileLogger.info("{}âââ {}{}", indent, file.getName(), type);
+
+ if (file.isDirectory() && currentDepth < maxDepth - 1) {
+ logDirectoryStructure(file, indent + "â ", currentDepth + 1,
maxDepth);
+ }
+ }
+ }
+ }
+
+ // ========== State Management ==========
+
+ private void updateBuildState(BuildState state) {
+ try {
+ stateUpdateConsumer.accept(state);
+ } catch (Exception e) {
+ log.warn("Failed to update build state to {}: {}", state,
e.getMessage());
+ }
+ }
+
+ private void handleBuildSuccess() {
+ updateBuildState(BuildState.SUCCESSFUL);
+ fileLogger.info("=== BUILD SUCCESSFUL ===");
+ fileLogger.info(
+ "Project {} built successfully in {}",
+ project.getName(),
+ formatDuration(Duration.between(startTime, LocalDateTime.now())));
+
+ try {
+ notifyReleaseConsumer.accept(fileLogger);
+ } catch (Exception e) {
+ log.warn("Failed to notify release consumer: {}", e.getMessage());
+ }
+ }
+
+ private void handleBuildFailure(String message) {
+ handleBuildFailure(message, null);
+ }
+
+ private void handleBuildFailure(String message, Throwable cause) {
+ updateBuildState(BuildState.FAILED);
+ fileLogger.error("=== BUILD FAILED ===");
+ fileLogger.error("Project {} build failed: {}", project.getName(),
message);
+
+ if (cause != null) {
+ fileLogger.error("Cause: {}", cause.getMessage());
+ }
+ }
+
+ private void handleBuildInterruption() {
+ updateBuildState(BuildState.FAILED);
+ fileLogger.warn("=== BUILD INTERRUPTED ===");
+ fileLogger.warn("Project {} build was interrupted", project.getName());
+ }
+
+ // ========== Logging Methods ==========
+
+ private void logBuildStart() {
+ fileLogger.info("===============================================");
+ fileLogger.info("StreamPark Project Build Started");
+ fileLogger.info("===============================================");
+ fileLogger.info("Project: {}", project.getName());
+ fileLogger.info("ID: {}", project.getId());
+ fileLogger.info("Start Time: {}",
startTime.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
+ fileLogger.info("Repository: {}", sanitizeUrl(project.getUrl()));
+ fileLogger.info("Branch/Tag: {}",
StringUtils.defaultIfBlank(project.getRefs(), "default"));
+ fileLogger.info("Build Args: {}",
sanitizeBuildCommand(project.getMavenBuildArgs()));
+ fileLogger.info("===============================================");
+ fileLogger.info(project.getLog4BuildStart());
+ }
+
+ private void logBuildSummary() {
+ LocalDateTime endTime = LocalDateTime.now();
+ Duration totalDuration = Duration.between(startTime, endTime);
+
+ fileLogger.info("===============================================");
+ fileLogger.info("Build Summary for Project: {}", project.getName());
+ fileLogger.info("===============================================");
+ fileLogger.info("Total Duration: {}", formatDuration(totalDuration));
+ fileLogger.info("End Time: {}",
endTime.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME));
+ fileLogger.info("===============================================");
+ }
+
+ private void logCloneSuccess() {
+ if (cloneStartTime != null) {
+ Duration cloneDuration = Duration.between(cloneStartTime,
LocalDateTime.now());
+ fileLogger.info("Git clone completed successfully in {}",
formatDuration(cloneDuration));
+ fileLogger.info(
+ String.format("[StreamPark] project [%s] git clone successful!",
project.getName()));
+ }
+ }
+
+ private void logCloneError(String message, Exception e) {
+ fileLogger.error("[StreamPark] {}: {}", message, e.getMessage());
+ fileLogger.error(
+ "Project: {}, Refs: {}, URL: {}",
+ project.getName(),
+ project.getRefs(),
+ sanitizeUrl(project.getUrl()));
+ }
+
+ private void logBuildSuccess() {
+ if (buildStartTime != null) {}
+ }
+
+ private void logBuildError(String message, Exception e) {
+ fileLogger.error("[StreamPark] {}: {}", message, e.getMessage());
+ fileLogger.error(
+ "Project: {}, Working Directory: {}", project.getName(),
project.getMavenWorkHome());
+ }
+
+ private void logDeploySuccess() {
+ if (deployStartTime != null) {
+ Duration deployDuration = Duration.between(deployStartTime,
LocalDateTime.now());
+ fileLogger.info("Deployment completed successfully in {}",
formatDuration(deployDuration));
+ }
+ }
+
+ private void logDeployError(String message, Exception e) {
+ fileLogger.error("[StreamPark] {}: {}", message, e.getMessage());
+ fileLogger.error(
+ "Project: {}, Dist Home: {}", project.getName(),
project.getDistHome().getAbsolutePath());
+ }
+
+ // ========== Validation Methods ==========
+
+ private Project validateProject(Project project) {
+ Objects.requireNonNull(project, "Project cannot be null");
+
+ if (project.getId() == null) {
+ throw new IllegalArgumentException("Project ID cannot be null");
+ }
+
+ if (StringUtils.isBlank(project.getName())) {
+ throw new IllegalArgumentException("Project name cannot be blank");
+ }
+
+ return project;
+ }
+
+ private void validateLogPath(String logPath) {
+ if (StringUtils.isBlank(logPath)) {
+ throw new IllegalArgumentException("Log path cannot be blank");
+ }
+
+ try {
+ Path path = Paths.get(logPath);
+ Path parent = path.getParent();
+ if (parent != null && !Files.exists(parent)) {
+ Files.createDirectories(parent);
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid log path: " + logPath, e);
+ }
+ }
+
+ // ========== Utility Helper Methods ==========
+
+ private void ensureDirectoryExists(File directory) throws IOException {
+ if (!directory.exists() && !directory.mkdirs()) {
+ throw new IOException("Failed to create directory: " +
directory.getAbsolutePath());
+ }
+ }
+
+ private void cleanupResources() {
+ // Cleanup any temporary files or resources
+ try {
+ // Force garbage collection to clean up any file handles
+ System.gc();
+ } catch (Exception e) {
+ log.debug("Error during resource cleanup: {}", e.getMessage());
+ }
+ }
+
+ private String sanitizeUrl(String url) {
+ if (StringUtils.isBlank(url)) {
+ return "[BLANK_URL]";
+ }
+
+ // Remove credentials from URL for logging
+ return url.replaceAll("://[^@/]+@", "://***@");
+ }
+
+ private String sanitizeBuildCommand(String command) {
+ if (StringUtils.isBlank(command)) {
+ return "[NO_COMMAND]";
+ }
+
+ // Remove sensitive information from build commands
+ return SENSITIVE_INFO_PATTERN.matcher(command).replaceAll("***");
+ }
+
+ private String sanitizeBuildOutput(String output) {
+ if (StringUtils.isBlank(output)) {
+ return "";
+ }
+
+ // Remove sensitive information from build output
+ return SENSITIVE_INFO_PATTERN.matcher(output).replaceAll("***");
+ }
+
+ private String formatDuration(Duration duration) {
+ long seconds = duration.getSeconds();
+ long minutes = seconds / 60;
+ seconds = seconds % 60;
+
+ if (minutes > 0) {
+ return String.format("%dm %ds", minutes, seconds);
+ } else {
+ return String.format("%ds", seconds);
+ }
+ }
+
+ private String formatFileSize(long bytes) {
+ if (bytes < 1024) {
+ return bytes + " B";
+ } else if (bytes < 1024 * 1024) {
+ return String.format("%.1f KB", bytes / 1024.0);
+ } else {
+ return String.format("%.1f MB", bytes / (1024.0 * 1024.0));
+ }
+ }
+
+ /**
+ * Truncates a string to the specified length, adding ellipsis if necessary.
+ *
+ * @param str the string to truncate
+ * @param maxLength maximum length allowed
+ * @return truncated string
+ */
+ private String truncateString(String str, int maxLength) {
+ if (str == null) {
+ return "";
+ }
+ if (str.length() <= maxLength) {
+ return str;
+ }
+ return str.substring(0, Math.max(0, maxLength - 3)) + "...";
+ }
+
+ /** Functional interface for retryable operations. */
+ @FunctionalInterface
+ private interface ExecuteOperation {
+
+ boolean execute() throws Exception;
}
}
