This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/master by this push:
new 750abf3 FEATURE-3823: kvm agent hooks (#3839)
750abf3 is described below
commit 750abf355171303f3f9edc8dd464b166f270b812
Author: Bitworks LLC <[email protected]>
AuthorDate: Sat Mar 14 15:22:08 2020 +0700
FEATURE-3823: kvm agent hooks (#3839)
---
agent/conf/agent.properties | 25 ++++++
plugins/hypervisors/kvm/pom.xml | 5 ++
.../kvm/resource/LibvirtComputingResource.java | 70 ++++++++++++++++
.../kvm/resource/LibvirtKvmAgentHook.java | 76 +++++++++++++++++
.../wrapper/LibvirtStartCommandWrapper.java | 32 +++++++-
.../wrapper/LibvirtStopCommandWrapper.java | 13 +++
.../kvm/resource/LibvirtKvmAgentHookTest.java | 94 ++++++++++++++++++++++
7 files changed, 314 insertions(+), 1 deletion(-)
diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties
index bb9bf40..85c85a5 100644
--- a/agent/conf/agent.properties
+++ b/agent/conf/agent.properties
@@ -97,6 +97,31 @@ domr.scripts.dir=scripts/network/domr/kvm
# migration will finish quickly. Less than 1 means disabled.
#vm.migrate.pauseafter=0
+# Agent hooks is the way to override default agent behavior to extend the
functionality without excessive coding
+# for a custom deployment. The first hook promoted is
libvirt-vm-xml-transformer which allows provider to modify
+# VM XML specification before send to libvirt. Hooks are implemented in Groovy
and must be implemented in the way
+# to keep default CS behaviour is something goes wrong.
+# All hooks are located in a special directory defined in 'agent.hooks.basedir'
+#
+# agent.hooks.basedir=/etc/cloudstack/agent/hooks
+
+# every hook has two major attributes - script name, specified in
'agent.hooks.*.script' and method name
+# specified in 'agent.hooks.*.method'.
+
+# Libvirt XML transformer hook does XML-to-XML transformation which provider
can use to add/remove/modify some
+# sort of attributes in Libvirt XML domain specification.
+#
agent.hooks.libvirt_vm_xml_transformer.script=libvirt-vm-xml-transformer.groovy
+# agent.hooks.libvirt_vm_xml_transformer.method=transform
+#
+# The hook is called right after libvirt successfuly launched VM
+# agent.hooks.libvirt_vm_on_start.script=libvirt-vm-state-change.groovy
+# agent.hooks.libvirt_vm_on_start.method=onStart
+#
+# The hook is called right after libvirt successfuly stopped VM
+# agent.hooks.libvirt_vm_on_stop.script=libvirt-vm-state-change.groovy
+# agent.hooks.libvirt_vm_on_stop.method=onStop
+#
+
# set the type of bridge used on the hypervisor, this defines what commands
the resource
# will use to setup networking. Currently supported NATIVE, OPENVSWITCH
#network.bridge.type=native
diff --git a/plugins/hypervisors/kvm/pom.xml b/plugins/hypervisors/kvm/pom.xml
index 3c3a635..9d0c786 100644
--- a/plugins/hypervisors/kvm/pom.xml
+++ b/plugins/hypervisors/kvm/pom.xml
@@ -29,6 +29,11 @@
</parent>
<dependencies>
<dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <version>${cs.groovy.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index fd9075e..79958ef 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -289,6 +289,18 @@ public class LibvirtComputingResource extends
ServerResourceBase implements Serv
protected String _rngPath = "/dev/random";
protected int _rngRatePeriod = 1000;
protected int _rngRateBytes = 2048;
+ protected String _agentHooksBasedir = "/etc/cloudstack/agent/hooks";
+
+ protected String _agentHooksLibvirtXmlScript =
"libvirt-vm-xml-transformer.groovy";
+ protected String _agentHooksLibvirtXmlMethod = "transform";
+
+ protected String _agentHooksVmOnStartScript =
"libvirt-vm-state-change.groovy";
+ protected String _agentHooksVmOnStartMethod = "onStart";
+
+ protected String _agentHooksVmOnStopScript =
"libvirt-vm-state-change.groovy";
+ protected String _agentHooksVmOnStopMethod = "onStop";
+
+
protected File _qemuSocketsPath;
private final String _qemuGuestAgentSocketName = "org.qemu.guest_agent.0";
protected WatchDogAction _watchDogAction = WatchDogAction.NONE;
@@ -391,6 +403,18 @@ public class LibvirtComputingResource extends
ServerResourceBase implements Serv
return new ExecutionResult(true, null);
}
+ public LibvirtKvmAgentHook getTransformer() throws IOException {
+ return new LibvirtKvmAgentHook(_agentHooksBasedir,
_agentHooksLibvirtXmlScript, _agentHooksLibvirtXmlMethod);
+ }
+
+ public LibvirtKvmAgentHook getStartHook() throws IOException {
+ return new LibvirtKvmAgentHook(_agentHooksBasedir,
_agentHooksVmOnStartScript, _agentHooksVmOnStartMethod);
+ }
+
+ public LibvirtKvmAgentHook getStopHook() throws IOException {
+ return new LibvirtKvmAgentHook(_agentHooksBasedir,
_agentHooksVmOnStopScript, _agentHooksVmOnStopMethod);
+ }
+
public LibvirtUtilitiesHelper getLibvirtUtilitiesHelper() {
return libvirtUtilitiesHelper;
}
@@ -1097,6 +1121,8 @@ public class LibvirtComputingResource extends
ServerResourceBase implements Serv
value = (String) params.get("vm.migrate.pauseafter");
_migratePauseAfter = NumbersUtil.parseInt(value, -1);
+ configureAgentHooks(params);
+
value = (String)params.get("vm.migrate.speed");
_migrateSpeed = NumbersUtil.parseInt(value, -1);
if (_migrateSpeed == -1) {
@@ -1155,6 +1181,50 @@ public class LibvirtComputingResource extends
ServerResourceBase implements Serv
return true;
}
+ private void configureAgentHooks(final Map<String, Object> params) {
+ String value = (String) params.get("agent.hooks.basedir");
+ if (null != value) {
+ _agentHooksBasedir = value;
+ }
+ s_logger.debug("agent.hooks.basedir is " + _agentHooksBasedir);
+
+ value = (String)
params.get("agent.hooks.libvirt_vm_xml_transformer.script");
+ if (null != value) {
+ _agentHooksLibvirtXmlScript = value;
+ }
+ s_logger.debug("agent.hooks.libvirt_vm_xml_transformer.script is " +
_agentHooksLibvirtXmlScript);
+
+ value = (String)
params.get("agent.hooks.libvirt_vm_xml_transformer.method");
+ if (null != value) {
+ _agentHooksLibvirtXmlMethod = value;
+ }
+ s_logger.debug("agent.hooks.libvirt_vm_xml_transformer.method is " +
_agentHooksLibvirtXmlMethod);
+
+ value = (String) params.get("agent.hooks.libvirt_vm_on_start.script");
+ if (null != value) {
+ _agentHooksVmOnStartScript = value;
+ }
+ s_logger.debug("agent.hooks.libvirt_vm_on_start.script is " +
_agentHooksVmOnStartScript);
+
+ value = (String) params.get("agent.hooks.libvirt_vm_on_start.method");
+ if (null != value) {
+ _agentHooksVmOnStartMethod = value;
+ }
+ s_logger.debug("agent.hooks.libvirt_vm_on_start.method is " +
_agentHooksVmOnStartMethod);
+
+ value = (String) params.get("agent.hooks.libvirt_vm_on_stop.script");
+ if (null != value) {
+ _agentHooksVmOnStopScript = value;
+ }
+ s_logger.debug("agent.hooks.libvirt_vm_on_stop.script is " +
_agentHooksVmOnStopScript);
+
+ value = (String) params.get("agent.hooks.libvirt_vm_on_stop.method");
+ if (null != value) {
+ _agentHooksVmOnStopMethod = value;
+ }
+ s_logger.debug("agent.hooks.libvirt_vm_on_stop.method is " +
_agentHooksVmOnStopMethod);
+ }
+
private void loadUefiProperties() throws FileNotFoundException {
if (_uefiProperties != null &&
_uefiProperties.getProperty("guest.loader.legacy") != null) {
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtKvmAgentHook.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtKvmAgentHook.java
new file mode 100644
index 0000000..3627d6e
--- /dev/null
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtKvmAgentHook.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.resource;
+
+import groovy.lang.Binding;
+import groovy.lang.GroovyObject;
+import groovy.util.GroovyScriptEngine;
+import groovy.util.ResourceException;
+import groovy.util.ScriptException;
+import org.apache.log4j.Logger;
+import org.codehaus.groovy.runtime.metaclass.MissingMethodExceptionNoStack;
+
+import java.io.File;
+import java.io.IOException;
+
+public class LibvirtKvmAgentHook {
+ private final String script;
+ private final String method;
+ private final GroovyScriptEngine gse;
+ private final Binding binding = new Binding();
+
+ private static final Logger s_logger =
Logger.getLogger(LibvirtKvmAgentHook.class);
+
+ public LibvirtKvmAgentHook(String path, String script, String method)
throws IOException {
+ this.script = script;
+ this.method = method;
+ File full_path = new File(path, script);
+ if (!full_path.canRead()) {
+ s_logger.warn("Groovy script '" + full_path.toString() + "' is not
available. Transformations will not be applied.");
+ this.gse = null;
+ } else {
+ this.gse = new GroovyScriptEngine(path);
+ }
+ }
+
+ public boolean isInitialized() {
+ return this.gse != null;
+ }
+
+ public Object handle(Object arg) throws ResourceException, ScriptException
{
+ if (!isInitialized()) {
+ s_logger.warn("Groovy scripting engine is not initialized. Data
transformation skipped.");
+ return arg;
+ }
+
+ GroovyObject cls = (GroovyObject) this.gse.run(this.script, binding);
+ if (null == cls) {
+ s_logger.warn("Groovy object is not received from script '" +
this.script + "'.");
+ return arg;
+ } else {
+ Object[] params = {s_logger, arg};
+ try {
+ Object res = cls.invokeMethod(this.method, params);
+ return res;
+ } catch (MissingMethodExceptionNoStack e) {
+ s_logger.error("Error occured when calling method from groovy
script, {}", e);
+ return arg;
+ }
+ }
+ }
+}
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java
index 068b24e..dbb9571 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java
@@ -35,6 +35,7 @@ import
com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef;
+import com.cloud.hypervisor.kvm.resource.LibvirtKvmAgentHook;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.network.Networks.TrafficType;
import com.cloud.resource.CommandWrapper;
@@ -79,7 +80,10 @@ public final class LibvirtStartCommandWrapper extends
CommandWrapper<StartComman
libvirtComputingResource.createVifs(vmSpec, vm);
s_logger.debug("starting " + vmName + ": " + vm.toString());
- libvirtComputingResource.startVM(conn, vmName, vm.toString());
+ String vmInitialSpecification = vm.toString();
+ String vmFinalSpecification =
performXmlTransformHook(vmInitialSpecification, libvirtComputingResource);
+ libvirtComputingResource.startVM(conn, vmName,
vmFinalSpecification);
+ performAgentStartHook(vmName, libvirtComputingResource);
libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec,
false);
@@ -136,4 +140,30 @@ public final class LibvirtStartCommandWrapper extends
CommandWrapper<StartComman
}
}
}
+
+ private void performAgentStartHook(String vmName, LibvirtComputingResource
libvirtComputingResource) {
+ try {
+ LibvirtKvmAgentHook onStartHook =
libvirtComputingResource.getStartHook();
+ onStartHook.handle(vmName);
+ } catch (Exception e) {
+ s_logger.warn("Exception occurred when handling LibVirt VM onStart
hook: {}", e);
+ }
+ }
+
+ private String performXmlTransformHook(String vmInitialSpecification,
final LibvirtComputingResource libvirtComputingResource) {
+ String vmFinalSpecification;
+ try {
+ // if transformer fails, everything must go as it's just skipped.
+ LibvirtKvmAgentHook t = libvirtComputingResource.getTransformer();
+ vmFinalSpecification = (String) t.handle(vmInitialSpecification);
+ if (null == vmFinalSpecification) {
+ s_logger.warn("Libvirt XML transformer returned NULL, will use
XML specification unchanged.");
+ vmFinalSpecification = vmInitialSpecification;
+ }
+ } catch(Exception e) {
+ s_logger.warn("Exception occurred when handling LibVirt XML
transformer hook: {}", e);
+ vmFinalSpecification = vmInitialSpecification;
+ }
+ return vmFinalSpecification;
+ }
}
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java
index ad12971..cb57dbc 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import com.cloud.agent.api.to.DpdkTO;
+import com.cloud.hypervisor.kvm.resource.LibvirtKvmAgentHook;
import com.cloud.utils.Pair;
import com.cloud.utils.script.Script;
import com.cloud.utils.ssh.SshHelper;
@@ -92,6 +93,8 @@ public final class LibvirtStopCommandWrapper extends
CommandWrapper<StopCommand,
libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName);
final String result = libvirtComputingResource.stopVM(conn,
vmName, command.isForceStop());
+ performAgentStopHook(vmName, libvirtComputingResource);
+
if (result == null) {
if (disks != null && disks.size() > 0) {
for (final DiskDef disk : disks) {
@@ -147,4 +150,14 @@ public final class LibvirtStopCommandWrapper extends
CommandWrapper<StopCommand,
return new StopAnswer(command, e.getMessage(), false);
}
}
+
+ private void performAgentStopHook(String vmName, final
LibvirtComputingResource libvirtComputingResource) {
+ try {
+ LibvirtKvmAgentHook onStopHook =
libvirtComputingResource.getStopHook();
+ onStopHook.handle(vmName);
+ } catch (Exception e) {
+ s_logger.warn("Exception occurred when handling LibVirt VM onStop
hook: {}", e);
+ }
+ }
+
}
diff --git
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtKvmAgentHookTest.java
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtKvmAgentHookTest.java
new file mode 100644
index 0000000..1f63914
--- /dev/null
+++
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtKvmAgentHookTest.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.hypervisor.kvm.resource;
+
+import groovy.util.ResourceException;
+import groovy.util.ScriptException;
+import junit.framework.TestCase;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.UUID;
+
+public class LibvirtKvmAgentHookTest extends TestCase {
+
+ private final String source = "<xml />";
+ private final String dir = "/tmp";
+ private final String script = "xml-transform-test.groovy";
+ private final String method = "transform";
+ private final String methodNull = "transform2";
+ private final String testImpl = "package groovy\n" +
+ "\n" +
+ "class BaseTransform {\n" +
+ " String transform(Object logger, String xml) {\n" +
+ " return xml + xml\n" +
+ " }\n" +
+ " String transform2(Object logger, String xml) {\n" +
+ " return null\n" +
+ " }\n" +
+ "}\n" +
+ "\n" +
+ "new BaseTransform()\n" +
+ "\n";
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ PrintWriter pw = new PrintWriter(new File(dir, script));
+ pw.println(testImpl);
+ pw.close();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ new File(dir, script).delete();
+ super.tearDown();
+ }
+
+ public void testTransform() throws IOException, ResourceException,
ScriptException {
+ LibvirtKvmAgentHook t = new LibvirtKvmAgentHook(dir, script, method);
+ assertEquals(t.isInitialized(), true);
+ String result = (String)t.handle(source);
+ assertEquals(result, source + source);
+ }
+
+ public void testWrongMethod() throws IOException, ResourceException,
ScriptException {
+ LibvirtKvmAgentHook t = new LibvirtKvmAgentHook(dir, script,
"methodX");
+ assertEquals(t.isInitialized(), true);
+ assertEquals(t.handle(source), source);
+ }
+
+ public void testNullMethod() throws IOException, ResourceException,
ScriptException {
+ LibvirtKvmAgentHook t = new LibvirtKvmAgentHook(dir, script,
methodNull);
+ assertEquals(t.isInitialized(), true);
+ assertEquals(t.handle(source), null);
+ }
+
+ public void testWrongScript() throws IOException, ResourceException,
ScriptException {
+ LibvirtKvmAgentHook t = new LibvirtKvmAgentHook(dir,
"wrong-script.groovy", method);
+ assertEquals(t.isInitialized(), false);
+ assertEquals(t.handle(source), source);
+ }
+
+ public void testWrongDir() throws IOException, ResourceException,
ScriptException {
+ LibvirtKvmAgentHook t = new LibvirtKvmAgentHook("/" +
UUID.randomUUID().toString() + "-dir", script, method);
+ assertEquals(t.isInitialized(), false);
+ assertEquals(t.handle(source), source);
+ }
+}
