This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new b1fc2798726 Generate cloud-init multipart user data for template
append policy (#7643)
b1fc2798726 is described below
commit b1fc2798726bd5ca0782bce59749e8ee30399916
Author: Nicolas Vazquez <[email protected]>
AuthorDate: Mon Jul 10 04:47:03 2023 -0300
Generate cloud-init multipart user data for template append policy (#7643)
Signed-off-by: Abhishek Kumar <[email protected]>
Co-authored-by: Abhishek Kumar <[email protected]>
---
.../cloudstack/userdata/UserDataManager.java | 24 +++
client/pom.xml | 10 +
...ing-core-lifecycle-core-context-inheritable.xml | 7 +-
.../core/spring-core-registry-core-context.xml | 4 +
engine/pom.xml | 2 +
engine/userdata/cloud-init/pom.xml | 36 ++++
.../userdata/CloudInitUserDataProvider.java | 208 +++++++++++++++++++++
.../core/spring-userdata-cloud-init-context.xml | 27 +++
.../userdata/CloudInitUserDataProviderTest.java | 139 ++++++++++++++
engine/userdata/pom.xml | 47 +++++
.../cloudstack/userdata/UserDataManagerImpl.java | 82 ++++++++
.../cloudstack/userdata/UserDataProvider.java | 28 +++
.../core/spring-engine-userdata-core-context.xml | 34 ++++
.../main/java/com/cloud/vm/UserVmManagerImpl.java | 18 +-
.../java/com/cloud/vm/UserVmManagerImplTest.java | 27 +--
test/integration/smoke/test_register_userdata.py | 19 +-
16 files changed, 669 insertions(+), 43 deletions(-)
diff --git
a/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java
b/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java
new file mode 100644
index 00000000000..2fc3acd45d1
--- /dev/null
+++ b/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.userdata;
+
+import com.cloud.utils.component.Manager;
+import org.apache.cloudstack.framework.config.Configurable;
+
+public interface UserDataManager extends Manager, Configurable {
+ String concatenateUserData(String userdata1, String userdata2, String
userdataProvider);
+}
diff --git a/client/pom.xml b/client/pom.xml
index a548d676a67..cbe45d88b6c 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -352,6 +352,16 @@
<artifactId>cloud-plugin-outofbandmanagement-driver-redfish</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-engine-userdata-cloud-init</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-engine-userdata</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-mom-rabbitmq</artifactId>
diff --git
a/core/src/main/resources/META-INF/cloudstack/core/spring-core-lifecycle-core-context-inheritable.xml
b/core/src/main/resources/META-INF/cloudstack/core/spring-core-lifecycle-core-context-inheritable.xml
index b754d6bfe62..3e57a01e211 100644
---
a/core/src/main/resources/META-INF/cloudstack/core/spring-core-lifecycle-core-context-inheritable.xml
+++
b/core/src/main/resources/META-INF/cloudstack/core/spring-core-lifecycle-core-context-inheritable.xml
@@ -39,5 +39,10 @@
<property name="typeClass"
value="com.cloud.utils.component.PluggableService" />
</bean>
-
+
+ <bean
class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
+ <property name="registry" ref="userDataProvidersRegistry" />
+ <property name="typeClass"
value="org.apache.cloudstack.userdata.UserDataProvider" />
+ </bean>
+
</beans>
\ No newline at end of file
diff --git
a/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
b/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
index cb559131a3d..a7f384c76a9 100644
---
a/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
+++
b/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
@@ -342,4 +342,8 @@
<bean id="kubernetesClusterHelperRegistry"
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
</bean>
+
+ <bean id="userDataProvidersRegistry"
+
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
+ </bean>
</beans>
diff --git a/engine/pom.xml b/engine/pom.xml
index 608affec392..91f430ba300 100644
--- a/engine/pom.xml
+++ b/engine/pom.xml
@@ -58,6 +58,8 @@
<module>storage/image</module>
<module>storage/snapshot</module>
<module>storage/volume</module>
+ <module>userdata/cloud-init</module>
+ <module>userdata</module>
</modules>
<profiles>
<profile>
diff --git a/engine/userdata/cloud-init/pom.xml
b/engine/userdata/cloud-init/pom.xml
new file mode 100644
index 00000000000..7e641bd6b5e
--- /dev/null
+++ b/engine/userdata/cloud-init/pom.xml
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+<modelVersion>4.0.0</modelVersion>
+<artifactId>cloud-engine-userdata-cloud-init</artifactId>
+<name>Apache CloudStack Engine Cloud-Init Userdata Component</name>
+<parent>
+ <artifactId>cloud-engine</artifactId>
+ <groupId>org.apache.cloudstack</groupId>
+ <version>4.19.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+</parent>
+<dependencies>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-engine-userdata</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+</dependencies>
+</project>
diff --git
a/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java
b/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java
new file mode 100644
index 00000000000..c61f37a1896
--- /dev/null
+++
b/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java
@@ -0,0 +1,208 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.userdata;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
+
+import javax.mail.BodyPart;
+import javax.mail.MessagingException;
+import javax.mail.Multipart;
+import javax.mail.Session;
+import javax.mail.internet.MimeBodyPart;
+import javax.mail.internet.MimeMessage;
+import javax.mail.internet.MimeMultipart;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class CloudInitUserDataProvider extends AdapterBase implements
UserDataProvider {
+
+ protected enum FormatType {
+ CLOUD_CONFIG, BASH_SCRIPT, MIME, CLOUD_BOOTHOOK, INCLUDE_FILE
+ }
+
+ private static final String CLOUD_CONFIG_CONTENT_TYPE =
"text/cloud-config";
+ private static final String BASH_SCRIPT_CONTENT_TYPE =
"text/x-shellscript";
+ private static final String INCLUDE_FILE_CONTENT_TYPE =
"text/x-include-url";
+ private static final String CLOUD_BOOTHOOK_CONTENT_TYPE =
"text/cloud-boothook";
+
+ private static final Map<FormatType, String> formatContentTypeMap =
Map.ofEntries(
+ Map.entry(FormatType.CLOUD_CONFIG, CLOUD_CONFIG_CONTENT_TYPE),
+ Map.entry(FormatType.BASH_SCRIPT, BASH_SCRIPT_CONTENT_TYPE),
+ Map.entry(FormatType.CLOUD_BOOTHOOK, CLOUD_BOOTHOOK_CONTENT_TYPE),
+ Map.entry(FormatType.INCLUDE_FILE, INCLUDE_FILE_CONTENT_TYPE)
+ );
+
+ private static final Logger LOGGER =
Logger.getLogger(CloudInitUserDataProvider.class);
+
+ private static final Session session = Session.getDefaultInstance(new
Properties());
+
+ @Override
+ public String getName() {
+ return "cloud-init";
+ }
+
+ protected boolean isGZipped(String userdata) {
+ if (StringUtils.isEmpty(userdata)) {
+ return false;
+ }
+ byte[] data = userdata.getBytes(StandardCharsets.ISO_8859_1);
+ if (data.length < 2) {
+ return false;
+ }
+ int magic = data[0] & 0xff | ((data[1] << 8) & 0xff00);
+ return magic == GZIPInputStream.GZIP_MAGIC;
+ }
+
+ protected String extractUserDataHeader(String userdata) {
+ if (isGZipped(userdata)) {
+ throw new CloudRuntimeException("Gzipped user data can not be used
together with other user data formats");
+ }
+ List<String> lines = Arrays.stream(userdata.split("\n"))
+ .filter(x -> (x.startsWith("#") && !x.startsWith("##")) ||
(x.startsWith("Content-Type:")))
+ .collect(Collectors.toList());
+ if (CollectionUtils.isEmpty(lines)) {
+ throw new CloudRuntimeException("Failed to detect the user data
format type as it " +
+ "does not contain a header");
+ }
+ return lines.get(0);
+ }
+
+ protected FormatType mapUserDataHeaderToFormatType(String header) {
+ if (header.equalsIgnoreCase("#cloud-config")) {
+ return FormatType.CLOUD_CONFIG;
+ } else if (header.startsWith("#!")) {
+ return FormatType.BASH_SCRIPT;
+ } else if (header.equalsIgnoreCase("#cloud-boothook")) {
+ return FormatType.CLOUD_BOOTHOOK;
+ } else if (header.startsWith("#include")) {
+ return FormatType.INCLUDE_FILE;
+ } else if (header.startsWith("Content-Type:")) {
+ return FormatType.MIME;
+ } else {
+ String msg = String.format("Cannot recognise the user data format
type from the header line: %s." +
+ "Supported types are: cloud-config, bash script,
cloud-boothook, include file or MIME", header);
+ LOGGER.error(msg);
+ throw new CloudRuntimeException(msg);
+ }
+ }
+
+ /**
+ * Detect the user data type
+ * Reference: <a
href="https://canonical-cloud-init.readthedocs-hosted.com/en/latest/explanation/format.html#user-data-formats";
/>
+ */
+ protected FormatType getUserDataFormatType(String userdata) {
+ if (StringUtils.isBlank(userdata)) {
+ String msg = "User data expected but provided empty user data";
+ LOGGER.error(msg);
+ throw new CloudRuntimeException(msg);
+ }
+
+ String header = extractUserDataHeader(userdata);
+ return mapUserDataHeaderToFormatType(header);
+ }
+
+ private String getContentType(String userData, FormatType formatType)
throws MessagingException {
+ if (formatType == FormatType.MIME) {
+ MimeMessage msg = new MimeMessage(session, new
ByteArrayInputStream(userData.getBytes()));
+ return msg.getContentType();
+ }
+ if (!formatContentTypeMap.containsKey(formatType)) {
+ throw new CloudRuntimeException(String.format("Cannot get the user
data content type as " +
+ "its format type %s is invalid", formatType.name()));
+ }
+ return formatContentTypeMap.get(formatType);
+ }
+
+ protected MimeBodyPart generateBodyPartMIMEMessage(String userData,
FormatType formatType) throws MessagingException {
+ MimeBodyPart bodyPart = new MimeBodyPart();
+ String contentType = getContentType(userData, formatType);
+ bodyPart.setContent(userData, contentType);
+ bodyPart.addHeader("Content-Transfer-Encoding", "base64");
+ return bodyPart;
+ }
+
+ private Multipart getMessageContent(MimeMessage message) {
+ Multipart messageContent;
+ try {
+ messageContent = (MimeMultipart) message.getContent();
+ } catch (IOException | MessagingException e) {
+ messageContent = new MimeMultipart();
+ }
+ return messageContent;
+ }
+
+ private void addBodyPartsToMessageContentFromUserDataContent(Multipart
messageContent,
+ MimeMessage
msgFromUserdata) throws MessagingException, IOException {
+ Multipart msgFromUserdataParts = (MimeMultipart)
msgFromUserdata.getContent();
+ int count = msgFromUserdataParts.getCount();
+ int i = 0;
+ while (i < count) {
+ BodyPart bodyPart = msgFromUserdataParts.getBodyPart(0);
+ messageContent.addBodyPart(bodyPart);
+ i++;
+ }
+ }
+
+ private MimeMessage createMultipartMessageAddingUserdata(String userData,
FormatType formatType,
+ MimeMessage
message) throws MessagingException, IOException {
+ MimeMessage newMessage = new MimeMessage(session);
+ Multipart messageContent = getMessageContent(message);
+
+ if (formatType == FormatType.MIME) {
+ MimeMessage msgFromUserdata = new MimeMessage(session, new
ByteArrayInputStream(userData.getBytes()));
+ addBodyPartsToMessageContentFromUserDataContent(messageContent,
msgFromUserdata);
+ } else {
+ MimeBodyPart part = generateBodyPartMIMEMessage(userData,
formatType);
+ messageContent.addBodyPart(part);
+ }
+ newMessage.setContent(messageContent);
+ return newMessage;
+ }
+
+ @Override
+ public String appendUserData(String userData1, String userData2) {
+ try {
+ FormatType formatType1 = getUserDataFormatType(userData1);
+ FormatType formatType2 = getUserDataFormatType(userData2);
+ MimeMessage message = new MimeMessage(session);
+ message = createMultipartMessageAddingUserdata(userData1,
formatType1, message);
+ message = createMultipartMessageAddingUserdata(userData2,
formatType2, message);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ message.writeTo(output);
+ return output.toString();
+ } catch (MessagingException | IOException | CloudRuntimeException e) {
+ String msg = String.format("Error attempting to merge user data as
a multipart user data. " +
+ "Reason: %s", e.getMessage());
+ LOGGER.error(msg, e);
+ throw new CloudRuntimeException(msg, e);
+ }
+ }
+}
diff --git
a/engine/userdata/cloud-init/src/main/resources/META-INF/cloudstack/core/spring-userdata-cloud-init-context.xml
b/engine/userdata/cloud-init/src/main/resources/META-INF/cloudstack/core/spring-userdata-cloud-init-context.xml
new file mode 100644
index 00000000000..742398e0b86
--- /dev/null
+++
b/engine/userdata/cloud-init/src/main/resources/META-INF/cloudstack/core/spring-userdata-cloud-init-context.xml
@@ -0,0 +1,27 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd";
+>
+ <bean id="cloudInitUserDataProvider"
class="org.apache.cloudstack.userdata.CloudInitUserDataProvider">
+ <property name="name" value="cloud-init" />
+ </bean>
+</beans>
diff --git
a/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java
b/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java
new file mode 100644
index 00000000000..b91438c5a36
--- /dev/null
+++
b/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.userdata;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.GZIPOutputStream;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class CloudInitUserDataProviderTest {
+
+ private final CloudInitUserDataProvider provider = new
CloudInitUserDataProvider();
+ private final static String CLOUD_CONFIG_USERDATA = "## template: jinja\n"
+
+ "#cloud-config\n" +
+ "runcmd:\n" +
+ " - echo 'TestVariable {{ ds.meta_data.variable1 }}' >>
/tmp/variable\n" +
+ " - echo 'Hostname {{ ds.meta_data.public_hostname }}' >
/tmp/hostname";
+
+ @Test
+ public void testGetUserDataFormatType() {
+ CloudInitUserDataProvider.FormatType type =
provider.getUserDataFormatType(CLOUD_CONFIG_USERDATA);
+ Assert.assertEquals(CloudInitUserDataProvider.FormatType.CLOUD_CONFIG,
type);
+ }
+
+ @Test(expected = CloudRuntimeException.class)
+ public void testGetUserDataFormatTypeNoHeader() {
+ String userdata = "password: password\nchpasswd: { expire: False
}\nssh_pwauth: True";
+ provider.getUserDataFormatType(userdata);
+ }
+
+ @Test(expected = CloudRuntimeException.class)
+ public void testGetUserDataFormatTypeInvalidType() {
+ String userdata = "#invalid-type\n" +
+ "password: password\nchpasswd: { expire: False }\nssh_pwauth:
True";
+ provider.getUserDataFormatType(userdata);
+ }
+
+ @Test
+ public void testAppendUserData() {
+ String templateData = "#cloud-config\n" +
+ "password: atomic\n" +
+ "chpasswd: { expire: False }\n" +
+ "ssh_pwauth: True";
+ String vmData = "#!/bin/bash\n" +
+ "date > /provisioned";
+ String multipartUserData = provider.appendUserData(templateData,
vmData);
+ Assert.assertTrue(multipartUserData.contains("Content-Type:
multipart"));
+ }
+
+ @Test
+ public void testAppendUserDataMIMETemplateData() {
+ String templateData = "Content-Type: multipart/mixed;
boundary=\"//\"\n" +
+ "MIME-Version: 1.0\n" +
+ "\n" +
+ "--//\n" +
+ "Content-Type: text/cloud-config; charset=\"us-ascii\"\n" +
+ "MIME-Version: 1.0\n" +
+ "Content-Transfer-Encoding: 7bit\n" +
+ "Content-Disposition: attachment;
filename=\"cloud-config.txt\"\n" +
+ "\n" +
+ "#cloud-config\n" +
+ "\n" +
+ "# Upgrade the instance on first boot\n" +
+ "# (ie run apt-get upgrade)\n" +
+ "#\n" +
+ "# Default: false\n" +
+ "# Aliases: apt_upgrade\n" +
+ "package_upgrade: true";
+ String vmData = "#!/bin/bash\n" +
+ "date > /provisioned";
+ String multipartUserData = provider.appendUserData(templateData,
vmData);
+ Assert.assertTrue(multipartUserData.contains("Content-Type:
multipart"));
+ }
+
+ @Test(expected = CloudRuntimeException.class)
+ public void testAppendUserDataInvalidUserData() {
+ String templateData = "password: atomic\n" +
+ "chpasswd: { expire: False }\n" +
+ "ssh_pwauth: True";
+ String vmData = "#!/bin/bash\n" +
+ "date > /provisioned";
+ provider.appendUserData(templateData, vmData);
+ }
+
+ @Test
+ public void testIsGzippedUserDataWithCloudConfigData() {
+ Assert.assertFalse(provider.isGZipped(CLOUD_CONFIG_USERDATA));
+ }
+
+ private String createGzipDataAsString() throws IOException {
+ byte[] input =
CLOUD_CONFIG_USERDATA.getBytes(StandardCharsets.ISO_8859_1);
+
+ ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream();
+ GZIPOutputStream outputStream = new
GZIPOutputStream(arrayOutputStream);
+ outputStream.write(input,0, input.length);
+ outputStream.close();
+
+ return arrayOutputStream.toString(StandardCharsets.ISO_8859_1);
+ }
+
+ @Test
+ public void testIsGzippedUserDataWithValidGzipData() {
+ try {
+ String gzipped = createGzipDataAsString();
+ Assert.assertTrue(provider.isGZipped(gzipped));
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test(expected = CloudRuntimeException.class)
+ public void testAppendUserDataWithGzippedData() {
+ try {
+ provider.appendUserData(CLOUD_CONFIG_USERDATA,
createGzipDataAsString());
+ Assert.fail("Gzipped data shouldn't be appended with other data");
+ } catch (IOException e) {
+ Assert.fail("Exception encountered: " + e.getMessage());
+ }
+ }
+}
diff --git a/engine/userdata/pom.xml b/engine/userdata/pom.xml
new file mode 100644
index 00000000000..2e00ebd9786
--- /dev/null
+++ b/engine/userdata/pom.xml
@@ -0,0 +1,47 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>cloud-engine-userdata</artifactId>
+ <name>Apache CloudStack Engine Userdata Component</name>
+ <parent>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-engine</artifactId>
+ <version>4.19.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git
a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java
b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java
new file mode 100644
index 00000000000..b2ee9dfd607
--- /dev/null
+++
b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.userdata;
+
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class UserDataManagerImpl extends ManagerBase implements
UserDataManager {
+ private List<UserDataProvider> userDataProviders;
+ private static Map<String, UserDataProvider> userDataProvidersMap = new
HashMap<>();
+
+ public void setUserDataProviders(final List<UserDataProvider>
userDataProviders) {
+ this.userDataProviders = userDataProviders;
+ }
+
+ private void initializeUserdataProvidersMap() {
+ if (userDataProviders != null) {
+ for (final UserDataProvider provider : userDataProviders) {
+ userDataProvidersMap.put(provider.getName().toLowerCase(),
provider);
+ }
+ }
+ }
+
+ @Override
+ public boolean start() {
+ initializeUserdataProvidersMap();
+ return true;
+ }
+
+ @Override
+ public String getConfigComponentName() {
+ return UserDataManagerImpl.class.getSimpleName();
+ }
+
+ @Override
+ public ConfigKey<?>[] getConfigKeys() {
+ return new ConfigKey[] {};
+ }
+
+ protected UserDataProvider getUserdataProvider(String name) {
+ if (StringUtils.isEmpty(name)) {
+ // Use cloud-init as the default userdata provider
+ name = "cloud-init";
+ }
+ if (!userDataProvidersMap.containsKey(name)) {
+ throw new CloudRuntimeException("Failed to find userdata provider
by the name: " + name);
+ }
+ return userDataProvidersMap.get(name);
+ }
+
+ @Override
+ public String concatenateUserData(String userdata1, String userdata2,
String userdataProvider) {
+ byte[] userdata1Bytes = Base64.decodeBase64(userdata1.getBytes());
+ byte[] userdata2Bytes = Base64.decodeBase64(userdata2.getBytes());
+ String userData1Str = new String(userdata1Bytes);
+ String userData2Str = new String(userdata2Bytes);
+ UserDataProvider provider = getUserdataProvider(userdataProvider);
+ String appendUserData = provider.appendUserData(userData1Str,
userData2Str);
+ return Base64.encodeBase64String(appendUserData.getBytes());
+ }
+}
diff --git
a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataProvider.java
b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataProvider.java
new file mode 100644
index 00000000000..9ac577b54ef
--- /dev/null
+++
b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataProvider.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.userdata;
+
+public interface UserDataProvider {
+ String getName();
+
+ /**
+ * Append user data into a single user data.
+ * NOTE: userData1 and userData2 are decoded user data strings
+ * @return a non-encrypted string containing both user data inputs
+ */
+ String appendUserData(String userData1, String userData2);
+}
diff --git
a/engine/userdata/src/main/resources/META-INF/cloudstack/core/spring-engine-userdata-core-context.xml
b/engine/userdata/src/main/resources/META-INF/cloudstack/core/spring-engine-userdata-core-context.xml
new file mode 100644
index 00000000000..3e067044e53
--- /dev/null
+++
b/engine/userdata/src/main/resources/META-INF/cloudstack/core/spring-engine-userdata-core-context.xml
@@ -0,0 +1,34 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xmlns:context="http://www.springframework.org/schema/context";
+ xmlns:aop="http://www.springframework.org/schema/aop";
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+
http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/context
+
http://www.springframework.org/schema/context/spring-context.xsd";
+ >
+
+ <bean id="userDataManager"
class="org.apache.cloudstack.userdata.UserDataManagerImpl">
+ <property name="userDataProviders"
value="#{userDataProvidersRegistry.registered}" />
+ </bean>
+
+</beans>
\ No newline at end of file
diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
index a3d36d6bfb6..6d3b7acaa9c 100644
--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -123,6 +123,7 @@ import
org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
+import org.apache.cloudstack.userdata.UserDataManager;
import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
import org.apache.cloudstack.utils.security.ParserUtils;
import org.apache.cloudstack.vm.schedule.VMScheduleManager;
@@ -614,6 +615,9 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
@Inject
private ManagementService _mgr;
+ @Inject
+ private UserDataManager userDataManager;
+
private static final ConfigKey<Integer> VmIpFetchWaitInterval = new
ConfigKey<Integer>("Advanced", Integer.class,
"externaldhcp.vmip.retrieval.interval", "180",
"Wait Interval (in seconds) for shared network vm dhcp ip addr
fetch for next iteration ", true);
@@ -5731,9 +5735,9 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
}
if (userDataId != null) {
UserData apiUserDataVO =
userDataDao.findById(userDataId);
- return
doConcateUserDatas(templateUserDataVO.getUserData(),
apiUserDataVO.getUserData());
+ return
userDataManager.concatenateUserData(templateUserDataVO.getUserData(),
apiUserDataVO.getUserData(), null);
} else if (StringUtils.isNotEmpty(userData)) {
- return
doConcateUserDatas(templateUserDataVO.getUserData(), userData);
+ return
userDataManager.concatenateUserData(templateUserDataVO.getUserData(), userData,
null);
} else {
return templateUserDataVO.getUserData();
}
@@ -5751,16 +5755,6 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
return null;
}
- private String doConcateUserDatas(String userdata1, String userdata2) {
- byte[] userdata1Bytes = Base64.decodeBase64(userdata1.getBytes());
- byte[] userdata2Bytes = Base64.decodeBase64(userdata2.getBytes());
- byte[] finalUserDataBytes = new byte[userdata1Bytes.length +
userdata2Bytes.length];
- System.arraycopy(userdata1Bytes, 0, finalUserDataBytes, 0,
userdata1Bytes.length);
- System.arraycopy(userdata2Bytes, 0, finalUserDataBytes,
userdata1Bytes.length, userdata2Bytes.length);
-
- return Base64.encodeBase64String(finalUserDataBytes);
- }
-
@Override
public UserVm createVirtualMachine(DeployVMCmd cmd) throws
InsufficientCapacityException, ResourceUnavailableException,
ConcurrentOperationException,
StorageUnavailableException, ResourceAllocationException {
diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java
b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java
index 93995bc860a..f91b52b867b 100644
--- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java
+++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java
@@ -43,6 +43,7 @@ import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd;
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
import org.apache.cloudstack.context.CallContext;
import
org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.userdata.UserDataManager;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.junit.After;
@@ -217,6 +218,9 @@ public class UserVmManagerImplTest {
@Mock
private ServiceOfferingVO serviceOffering;
+ @Mock
+ UserDataManager userDataManager;
+
private static final long vmId = 1l;
private static final long zoneId = 2L;
private static final long accountId = 3L;
@@ -713,29 +717,6 @@ public class UserVmManagerImplTest {
Assert.assertEquals(finalUserdata, templateUserData);
}
- @Test
- public void testUserDataAppend() {
- String userData = "testUserdata";
- String templateUserData = "testTemplateUserdata";
- Long userDataId = 1L;
-
- VirtualMachineTemplate template =
Mockito.mock(VirtualMachineTemplate.class);
- when(template.getUserDataId()).thenReturn(2L);
-
when(template.getUserDataOverridePolicy()).thenReturn(UserData.UserDataOverridePolicy.APPEND);
-
- UserDataVO templateUserDataVO = Mockito.mock(UserDataVO.class);
- doReturn(templateUserDataVO).when(userDataDao).findById(2L);
- when(templateUserDataVO.getUserData()).thenReturn(templateUserData);
-
- UserDataVO apiUserDataVO = Mockito.mock(UserDataVO.class);
- doReturn(apiUserDataVO).when(userDataDao).findById(userDataId);
- when(apiUserDataVO.getUserData()).thenReturn(userData);
-
- String finalUserdata = userVmManagerImpl.finalizeUserData(null,
userDataId, template);
-
- Assert.assertEquals(finalUserdata, templateUserData+userData);
- }
-
@Test
public void testUserDataWithoutTemplate() {
String userData = "testUserdata";
diff --git a/test/integration/smoke/test_register_userdata.py
b/test/integration/smoke/test_register_userdata.py
index bc38cd989c0..5c954a876ec 100644
--- a/test/integration/smoke/test_register_userdata.py
+++ b/test/integration/smoke/test_register_userdata.py
@@ -589,21 +589,27 @@ class TestRegisteredUserdata(cloudstackTestCase):
2. Link a userdata to template with override policy is append
3. Deploy a VM with that template and also by passing another
userdata id
4. Since the override policy is append, userdata passed during VM
deployment will be appended to template's
- userdata and configured to VM. Verify the same by SSH into VM.
+ userdata and configured to VM as a multipart MIME userdata. Verify
the same by SSH into VM.
"""
+ # #!/bin/bash
+ # date > /provisioned
self.apiUserdata = UserData.register(
self.apiclient,
name="ApiUserdata",
- userdata="QVBJdXNlcmRhdGE=", #APIuserdata
+ userdata="IyEvYmluL2Jhc2gKZGF0ZSA+IC9wcm92aXNpb25lZA==",
account=self.account.name,
domainid=self.account.domainid
)
+ # #cloud-config
+ # password: atomic
+ # chpasswd: { expire: False }
+ # ssh_pwauth: True
self.templateUserdata = UserData.register(
self.apiclient,
name="TemplateUserdata",
- userdata="VGVtcGxhdGVVc2VyRGF0YQ==", #TemplateUserData
+
userdata="I2Nsb3VkLWNvbmZpZwpwYXNzd29yZDogYXRvbWljCmNocGFzc3dkOiB7IGV4cGlyZTogRmFsc2UgfQpzc2hfcHdhdXRoOiBUcnVl",
account=self.account.name,
domainid=self.account.domainid
)
@@ -700,10 +706,9 @@ class TestRegisteredUserdata(cloudstackTestCase):
cmd = "curl http://%s/latest/user-data"; % vr_ip
res = ssh.execute(cmd)
self.debug("Verifying userdata in the VR")
- self.assertEqual(
- str(res[0]),
- "TemplateUserDataAPIuserdata",
- "Failed to match userdata"
+ self.assertTrue(
+ "Content-Type: multipart" in str(res[2]),
+ "Failed to match multipart userdata"
)
@attr(tags=['advanced', 'simulator', 'basic', 'sg', 'testnow'],
required_hardware=True)
