- deps/whql_submission_15.cs:
* Handle multiple machines instead of one.
* Split the code into smaller functions.
* Get job parameters from the user: each parameter is a regex that should
match a device name in a machine, and is set using GetParameterByName().
Note that job parameters differ from DeviceData objects and descriptors.
* Get machine dimensions from the user and set them for each machine.
* Set the WDKSubmissionId dimension for all machines (it seems to be
required by some or all submissions).
* Instead of AddDeviceJob() use submission.ProcessJobs() and set the .Device
attribute of the ScheduleItem objects returned. This is required for
NDISTest to work.
* Don't set status to 'Unsafe' before resetting machines (unnecessary).
* If the machine pool already exists, delete it and create a new one.
- tests/whql_submission.py:
* Use all vms instead of just main_vm.
* Pass job parameters for each machine to the automation program.
* Pass machine dimensions for each machine.
* Write result summary to a file named 'summary' in addition to the regular
logs.
* Sort results in the result summary by failures and total test count in
descending order.
* Before starting the test, delete the machines from the data store, reboot
them and wait for them to reappear in the data store.
* Don't restart any services on the client machines (not necessary).
- Set 'restart_vm' to 'yes' in tests_base.cfg.sample to force a restart of the
VMs before each test.
Signed-off-by: Michael Goldish <[email protected]>
---
client/tests/kvm/deps/whql_submission_15.cs | 662 +++++++++++++++-----------
client/tests/kvm/deps/whql_submission_15.exe | Bin 10240 -> 12288 bytes
client/tests/kvm/tests/whql_submission.py | 182 +++++---
client/tests/kvm/tests_base.cfg.sample | 5 +-
4 files changed, 497 insertions(+), 352 deletions(-)
diff --git a/client/tests/kvm/deps/whql_submission_15.cs
b/client/tests/kvm/deps/whql_submission_15.cs
index 8fa6856..0928548 100644
--- a/client/tests/kvm/deps/whql_submission_15.cs
+++ b/client/tests/kvm/deps/whql_submission_15.cs
@@ -1,289 +1,373 @@
-// DTM submission automation program
-// Author: Michael Goldish <[email protected]>
-// Based on sample code by Microsoft.
-
-// Note: this program has only been tested with DTM version 1.5.
-// It might fail to work with other versions, specifically because it uses
-// a few undocumented methods/attributes.
-
-using System;
-using System.Collections.Generic;
-using System.Text.RegularExpressions;
-using Microsoft.DistributedAutomation.DeviceSelection;
-using Microsoft.DistributedAutomation.SqlDataStore;
-
-namespace automate0
-{
- class AutoJob
- {
- static int Main(string[] args)
- {
- if (args.Length != 5)
- {
- Console.WriteLine("Error: incorrect number of command line
arguments");
- Console.WriteLine("Usage: {0} serverName clientName
machinePoolName submissionName timeout",
- System.Environment.GetCommandLineArgs()[0]);
- return 1;
- }
- string serverName = args[0];
- string clientName = args[1];
- string machinePoolName = args[2];
- string submissionName = args[3];
- double timeout = Convert.ToDouble(args[4]);
-
- try
- {
- // Initialize DeviceScript and connect to data store
- Console.WriteLine("Initializing DeviceScript object");
- DeviceScript script = new DeviceScript();
- Console.WriteLine("Connecting to data store");
-
- script.ConnectToNamedDataStore(serverName);
-
- // Find client machine
- IResourcePool rootPool = script.GetResourcePoolByName("$");
- Console.WriteLine("Looking for client machine '{0}'",
clientName);
- IResource machine = null;
- while (true)
- {
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- // Make sure the machine is valid
- if (machine != null &&
- machine.OperatingSystem != null &&
- machine.OperatingSystem.Length > 0 &&
- machine.ProcessorArchitecture != null &&
- machine.ProcessorArchitecture.Length > 0 &&
- machine.GetDevices().Length > 0)
- break;
- System.Threading.Thread.Sleep(1000);
- }
- Console.WriteLine("Client machine '{0}' found ({1}, {2})",
- clientName, machine.OperatingSystem,
machine.ProcessorArchitecture);
-
- // Create machine pool and add client machine to it
- // (this must be done because jobs cannot be scheduled for
machines in the
- // default pool)
- try
- {
- script.CreateResourcePool(machinePoolName,
rootPool.ResourcePoolId);
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- IResourcePool newPool =
script.GetResourcePoolByName(machinePoolName);
- Console.WriteLine("Moving the client machine to pool '{0}'",
machinePoolName);
- machine.ChangeResourcePool(newPool);
-
- // Reset client machine
- if (machine.Status != "Ready")
- {
- Console.WriteLine("Changing the client machine's status to
'Reset'");
- while (true)
- {
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- machine.ChangeResourceStatus("Unsafe");
- System.Threading.Thread.Sleep(5000);
- machine.ChangeResourceStatus("Reset");
- break;
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- System.Threading.Thread.Sleep(5000);
- }
- Console.WriteLine("Waiting for client machine to be
ready");
- while (machine.Status != "Ready")
- {
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- System.Threading.Thread.Sleep(1000);
- }
- }
- Console.WriteLine("Client machine is ready");
-
- // Get requested device regex and look for a matching device
- Console.WriteLine("Device to test: ");
- Regex deviceRegex = new Regex(Console.ReadLine(),
RegexOptions.IgnoreCase);
- Console.WriteLine("Looking for device '{0}'", deviceRegex);
- IDevice device;
- DateTime endTime = DateTime.Now.AddSeconds(120);
- while (DateTime.Now < endTime)
- {
- machine = rootPool.GetResourceByName(clientName);
- Console.WriteLine("(Client machine has {0} devices)",
machine.GetDevices().Length);
- foreach (IDevice d in machine.GetDevices())
- {
- if (deviceRegex.IsMatch(d.FriendlyName))
- {
- device = d;
- goto deviceFound;
- }
- }
- System.Threading.Thread.Sleep(5000);
- }
- Console.WriteLine("Error: device '{0}' not found",
deviceRegex);
- return 1;
-
- deviceFound:
- Console.WriteLine("Found device '{0}'", device.FriendlyName);
-
- // Get requested jobs regex
- Console.WriteLine("Jobs to run: ");
- Regex jobRegex = new Regex(Console.ReadLine(),
RegexOptions.IgnoreCase);
-
- // Create submission
- Object[] existingSubmissions =
script.GetSubmissionByName(submissionName);
- if (existingSubmissions.Length > 0)
- {
- Console.WriteLine("Submission '{0}' already exists --
removing it",
- submissionName);
-
script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
- }
- Console.WriteLine("Creating submission '{0}'", submissionName);
- ISubmission submission =
script.CreateHardwareSubmission(submissionName,
- newPool.ResourcePoolId, device.InstanceId);
-
- // Get DeviceData objects from the user
- List<Object> deviceDataList = new List<Object>();
- while (true)
- {
- ISubmissionDeviceData dd =
script.CreateNewSubmissionDeviceData();
- Console.WriteLine("DeviceData name: ");
- dd.Name = Console.ReadLine();
- if (dd.Name.Length == 0)
- break;
- Console.WriteLine("DeviceData data: ");
- dd.Data = Console.ReadLine();
- deviceDataList.Add(dd);
- }
-
- // Set the submission's DeviceData
- submission.SetDeviceData(deviceDataList.ToArray());
-
- // Get descriptors from the user
- List<Object> descriptorList = new List<Object>();
- while (true)
- {
- Console.WriteLine("Descriptor path: ");
- string descriptorPath = Console.ReadLine();
- if (descriptorPath.Length == 0)
- break;
-
descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
- }
-
- // Set the submission's descriptors
- submission.SetLogoDescriptors(descriptorList.ToArray());
-
- // Create a schedule
- ISchedule schedule = script.CreateNewSchedule();
- Console.WriteLine("Scheduling jobs:");
- int jobCount = 0;
- foreach (IJob j in submission.GetJobs())
- {
- if (jobRegex.IsMatch(j.Name))
- {
- Console.WriteLine(" " + j.Name);
- schedule.AddDeviceJob(device, j);
- jobCount++;
- }
- }
- if (jobCount == 0)
- {
- Console.WriteLine("Error: no submission jobs match pattern
'{0}'", jobRegex);
- return 1;
- }
- schedule.AddSubmission(submission);
- schedule.SetResourcePool(newPool);
- script.RunSchedule(schedule);
-
- // Wait for jobs to complete
- Console.WriteLine("Waiting for all jobs to complete
(timeout={0})", timeout);
- endTime = DateTime.Now.AddSeconds(timeout);
- int numCompleted = 0, numFailed = 0;
- while (numCompleted < submission.GetResults().Length &&
DateTime.Now < endTime)
- {
- // Sleep for 30 seconds
- System.Threading.Thread.Sleep(30000);
- // Count completed submission jobs
- numCompleted = 0;
- foreach (IResult r in submission.GetResults())
- if (r.ResultStatus != "InProgress")
- numCompleted++;
- // Report results in a Python readable format and count
failed schedule jobs
- // (submission jobs are a subset of schedule jobs)
- Console.WriteLine();
- Console.WriteLine("---- [");
- numFailed = 0;
- foreach (IResult r in schedule.GetResults())
- {
- Console.WriteLine(" {");
- Console.WriteLine(" 'id': {0}, 'job': r'''{1}''',",
r.Job.Id, r.Job.Name);
- Console.WriteLine(" 'logs': r'''{0}''',",
r.LogLocation);
- if (r.ResultStatus != "InProgress")
- Console.WriteLine(" 'report': r'''{0}''',",
- submission.GetSubmissionResultReport(r));
- Console.WriteLine(" 'status': '{0}',",
r.ResultStatus);
- Console.WriteLine(" 'pass': {0}, 'fail': {1},
'notrun': {2}, 'notapplicable': {3}",
- r.Pass, r.Fail, r.NotRun, r.NotApplicable);
- Console.WriteLine(" },");
- numFailed += r.Fail;
- }
- Console.WriteLine("] ----");
- }
- Console.WriteLine();
-
- // Cancel incomplete jobs
- foreach (IResult r in schedule.GetResults())
- if (r.ResultStatus == "InProgress")
- r.Cancel();
-
- // Set the machine's status to Unsafe and then Reset
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- machine.ChangeResourceStatus("Unsafe");
- System.Threading.Thread.Sleep(5000);
- machine.ChangeResourceStatus("Reset");
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
-
- // Report failures
- if (numCompleted < submission.GetResults().Length)
- Console.WriteLine("Some jobs did not complete on time.");
- if (numFailed > 0)
- Console.WriteLine("Some jobs failed.");
-
- if (numFailed > 0 || numCompleted <
submission.GetResults().Length)
- return 1;
-
- Console.WriteLine("All jobs completed.");
- return 0;
- }
- catch (Exception e)
- {
- Console.WriteLine("Error: " + e.Message);
- return 1;
- }
- }
- }
-}
+// DTM submission automation program
+// Author: Michael Goldish <[email protected]>
+// Based on sample code by Microsoft.
+
+// Note: this program has only been tested with DTM version 1.5.
+// It might fail to work with other versions, specifically because it uses
+// a few undocumented methods/attributes.
+
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Microsoft.DistributedAutomation.DeviceSelection;
+using Microsoft.DistributedAutomation.SqlDataStore;
+
+namespace automate0
+{
+ class AutoJob
+ {
+ // Wait for a machine to show up in the data store
+ static void FindMachine(IResourcePool rootPool, string machineName)
+ {
+ Console.WriteLine("Looking for machine '{0}'", machineName);
+ IResource machine = null;
+ while (true)
+ {
+ try
+ {
+ machine = rootPool.GetResourceByName(machineName);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ }
+ // Make sure the machine is valid
+ if (machine != null &&
+ machine.OperatingSystem != null &&
+ machine.OperatingSystem.Length > 0 &&
+ machine.ProcessorArchitecture != null &&
+ machine.ProcessorArchitecture.Length > 0 &&
+ machine.GetDevices().Length > 0)
+ break;
+ System.Threading.Thread.Sleep(1000);
+ }
+ Console.WriteLine("Client machine '{0}' found ({1}, {2})",
+ machineName, machine.OperatingSystem,
machine.ProcessorArchitecture);
+ }
+
+ // Delete a machine pool if it exists
+ static void DeleteResourcePool(IDeviceScript script, string poolName)
+ {
+ while (true)
+ {
+ try
+ {
+ IResourcePool pool =
script.GetResourcePoolByName(poolName);
+ if (pool != null)
+ script.DeleteResourcePool(pool);
+ break;
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ System.Threading.Thread.Sleep(1000);
+ }
+ }
+ }
+
+ // Set the machine's status to 'Reset' and optionally wait for it to
become ready
+ static void ResetMachine(IResourcePool rootPool, string machineName,
bool wait)
+ {
+ Console.WriteLine("Resetting machine '{0}'", machineName);
+ IResource machine;
+ while (true)
+ {
+ try
+ {
+ machine = rootPool.GetResourceByName(machineName);
+ machine.ChangeResourceStatus("Reset");
+ break;
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ System.Threading.Thread.Sleep(5000);
+ }
+ }
+ if (wait)
+ {
+ Console.WriteLine("Waiting for machine '{0}' to be ready",
machineName);
+ while (machine.Status != "Ready")
+ {
+ try
+ {
+ machine = rootPool.GetResourceByName(machineName);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ }
+ System.Threading.Thread.Sleep(1000);
+ }
+ Console.WriteLine("Machine '{0}' is ready", machineName);
+ }
+ }
+
+ // Look for a device in a machine, and if not found, keep trying for 3
minutes
+ static IDevice GetDevice(IResourcePool rootPool, string machineName,
string regexStr)
+ {
+ Regex deviceRegex = new Regex(regexStr, RegexOptions.IgnoreCase);
+ int numAttempts = 1;
+ DateTime endTime = DateTime.Now.AddSeconds(180);
+ while (DateTime.Now < endTime)
+ {
+ IResource machine = rootPool.GetResourceByName(machineName);
+ Console.WriteLine("Looking for device '{0}' in machine '{1}'
(machine has {2} devices)",
+ regexStr, machineName, machine.GetDevices().Length);
+ foreach (IDevice d in machine.GetDevices())
+ {
+ if (deviceRegex.IsMatch(d.FriendlyName))
+ {
+ Console.WriteLine("Found device '{0}'",
d.FriendlyName);
+ return d;
+ }
+ }
+ Console.WriteLine("Device not found");
+ if (numAttempts % 5 == 0)
+ ResetMachine(rootPool, machineName, true);
+ else
+ System.Threading.Thread.Sleep(5000);
+ numAttempts++;
+ }
+ Console.WriteLine("Error: device '{0}' not found", deviceRegex);
+ return null;
+ }
+
+ static int Main(string[] args)
+ {
+ if (args.Length < 5)
+ {
+ Console.WriteLine("Error: incorrect number of command line
arguments");
+ Console.WriteLine("Usage: {0} serverName machinePoolName
submissionName timeout machineName0 machineName1 ...",
+ System.Environment.GetCommandLineArgs()[0]);
+ return 1;
+ }
+ string serverName = args[0];
+ string machinePoolName = args[1];
+ string submissionName = args[2];
+ double timeout = Convert.ToDouble(args[3]);
+
+ List<string> machines = new List<string>();
+ for (int i = 4; i < args.Length; i++)
+ machines.Add(args[i]);
+
+ try
+ {
+ // Initialize DeviceScript and connect to data store
+ Console.WriteLine("Initializing DeviceScript object");
+ DeviceScript script = new DeviceScript();
+ Console.WriteLine("Connecting to data store");
+ script.ConnectToNamedDataStore(serverName);
+
+ // Wait for client machines to become available
+ IResourcePool rootPool = script.GetResourcePoolByName("$");
+ foreach (string machineName in machines)
+ FindMachine(rootPool, machineName);
+
+ // Delete the machine pool if it already exists
+ DeleteResourcePool(script, machinePoolName);
+
+ // Create the machine pool and add the client machines to it
+ // (this must be done because jobs cannot be scheduled for
machines in the
+ // default pool)
+ try
+ {
+ script.CreateResourcePool(machinePoolName,
rootPool.ResourcePoolId);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ }
+ IResourcePool newPool =
script.GetResourcePoolByName(machinePoolName);
+ foreach (string machineName in machines)
+ {
+ Console.WriteLine("Moving machine '{0}' to pool '{1}'",
machineName, machinePoolName);
+
rootPool.GetResourceByName(machineName).ChangeResourcePool(newPool);
+ }
+
+ // Reset client machine
+ foreach (string machineName in machines)
+ ResetMachine(rootPool, machineName, true);
+
+ // Get requested device regex and look for a matching device
in the first machine
+ Console.WriteLine("Device to test:");
+ IDevice device = GetDevice(rootPool, machines[0],
Console.ReadLine());
+ if (device == null)
+ return 1;
+
+ // Get requested jobs regex
+ Console.WriteLine("Jobs to run:");
+ Regex jobRegex = new Regex(Console.ReadLine(),
RegexOptions.IgnoreCase);
+
+ // Create a submission
+ Object[] existingSubmissions =
script.GetSubmissionByName(submissionName);
+ if (existingSubmissions.Length > 0)
+ {
+ Console.WriteLine("Submission '{0}' already exists --
removing it",
+ submissionName);
+
script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
+ }
+ string hardwareId =
device.InstanceId.Remove(device.InstanceId.LastIndexOf("\\"));
+ Console.WriteLine("Creating submission '{0}' (hardware ID:
{1})", submissionName, hardwareId);
+ ISubmission submission =
script.CreateHardwareSubmission(submissionName, newPool.ResourcePoolId,
hardwareId);
+
+ // Set submission DeviceData
+ List<Object> deviceDataList = new List<Object>();
+ while (true)
+ {
+ ISubmissionDeviceData dd =
script.CreateNewSubmissionDeviceData();
+ Console.WriteLine("DeviceData name:");
+ dd.Name = Console.ReadLine();
+ if (dd.Name.Length == 0)
+ break;
+ Console.WriteLine("DeviceData data:");
+ dd.Data = Console.ReadLine();
+ deviceDataList.Add(dd);
+ }
+ submission.SetDeviceData(deviceDataList.ToArray());
+
+ // Set submission descriptors
+ List<Object> descriptorList = new List<Object>();
+ while (true)
+ {
+ Console.WriteLine("Descriptor path:");
+ string descriptorPath = Console.ReadLine();
+ if (descriptorPath.Length == 0)
+ break;
+
descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
+ }
+ submission.SetLogoDescriptors(descriptorList.ToArray());
+
+ // Set machine dimensions
+ foreach (string machineName in machines)
+ {
+ IResource machine =
rootPool.GetResourceByName(machineName);
+ while (true)
+ {
+ Console.WriteLine("Dimension name ({0}):",
machineName);
+ string dimName = Console.ReadLine();
+ if (dimName.Length == 0)
+ break;
+ Console.WriteLine("Dimension value ({0}):",
machineName);
+ machine.SetDimension(dimName, Console.ReadLine());
+ }
+ // Set the WDKSubmissionId dimension for all machines
+ machine.SetDimension("WDKSubmissionId",
submission.Id.ToString() + "_" + submission.Name);
+ }
+
+ // Get job parameters
+ List<string> paramNames = new List<string>();
+ List<string> paramValues = new List<string>();
+ foreach (string machineName in machines)
+ {
+ while (true)
+ {
+ Console.WriteLine("Parameter name ({0}):",
machineName);
+ string paramName = Console.ReadLine();
+ if (paramName.Length == 0)
+ break;
+ Console.WriteLine("Device regex ({0}):", machineName);
+ IDevice d = GetDevice(rootPool, machineName,
Console.ReadLine());
+ if (d == null)
+ return 1;
+ string deviceName =
d.GetAttribute("name")[0].ToString();
+ Console.WriteLine("Setting parameter value to '{0}'",
deviceName);
+ paramNames.Add(paramName);
+ paramValues.Add(deviceName);
+ }
+ }
+
+ // Find jobs that match the requested pattern
+ Console.WriteLine("Scheduling jobs:");
+ List<IJob> jobs = new List<IJob>();
+ foreach (IJob j in submission.GetJobs())
+ {
+ if (jobRegex.IsMatch(j.Name))
+ {
+ Console.WriteLine(" " + j.Name);
+ // Set job parameters
+ for (int i = 0; i < paramNames.Count; i++)
+ {
+ IParameter p = j.GetParameterByName(paramNames[i]);
+ if (p != null)
+ p.ScheduleValue = paramValues[i];
+ }
+ jobs.Add(j);
+ }
+ }
+ if (jobs.Count == 0)
+ {
+ Console.WriteLine("Error: no submission jobs match pattern
'{0}'", jobRegex);
+ return 1;
+ }
+
+ // Create a schedule, add jobs to it and run it
+ ISchedule schedule = script.CreateNewSchedule();
+ foreach (IScheduleItem item in
submission.ProcessJobs(jobs.ToArray()))
+ {
+ item.Device = device;
+ schedule.AddScheduleItem(item);
+ }
+ schedule.AddSubmission(submission);
+ schedule.SetResourcePool(newPool);
+ script.RunSchedule(schedule);
+
+ // Wait for jobs to complete
+ Console.WriteLine("Waiting for all jobs to complete
(timeout={0}s)", timeout);
+ DateTime endTime = DateTime.Now.AddSeconds(timeout);
+ int numCompleted, numFailed;
+ do
+ {
+ System.Threading.Thread.Sleep(30000);
+ // Report results in a Python readable format and count
completed and failed schedule jobs
+ numCompleted = numFailed = 0;
+ Console.WriteLine();
+ Console.WriteLine("---- [");
+ foreach (IResult r in schedule.GetResults())
+ {
+ if (r.ResultStatus != "InProgress") numCompleted++;
+ if (r.ResultStatus == "Investigate") numFailed++;
+ Console.WriteLine(" {");
+ Console.WriteLine(" 'id': {0}, 'job': r'''{1}''',",
r.Job.Id, r.Job.Name);
+ Console.WriteLine(" 'logs': r'''{0}''',",
r.LogLocation);
+ if (r.ResultStatus != "InProgress")
+ Console.WriteLine(" 'report': r'''{0}''',",
+ submission.GetSubmissionResultReport(r));
+ Console.WriteLine(" 'status': '{0}',",
r.ResultStatus);
+ Console.WriteLine(" 'pass': {0}, 'fail': {1},
'notrun': {2}, 'notapplicable': {3}",
+ r.Pass, r.Fail, r.NotRun, r.NotApplicable);
+ Console.WriteLine(" },");
+ }
+ Console.WriteLine("] ----");
+ } while (numCompleted < schedule.GetResults().Length &&
DateTime.Now < endTime);
+
+ Console.WriteLine();
+
+ // Cancel incomplete jobs
+ foreach (IResult r in schedule.GetResults())
+ if (r.ResultStatus == "InProgress")
+ r.Cancel();
+
+ // Reset the machines
+ foreach (string machineName in machines)
+ ResetMachine(rootPool, machineName, false);
+
+ // Report failures
+ if (numCompleted < schedule.GetResults().Length)
+ Console.WriteLine("Some jobs did not complete on time.");
+ if (numFailed > 0)
+ Console.WriteLine("Some jobs failed.");
+ if (numFailed > 0 || numCompleted <
schedule.GetResults().Length)
+ return 1;
+
+ Console.WriteLine("All jobs completed.");
+ return 0;
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Error: " + e.Message);
+ return 1;
+ }
+ }
+ }
+}
diff --git a/client/tests/kvm/deps/whql_submission_15.exe
b/client/tests/kvm/deps/whql_submission_15.exe
index
4f30aa801c8f200bd96b0de6c10d2b59c2bad268..605e2e3aa1f0761155f4aac5191e3ad0f0ddecc6
100644
GIT binary patch
literal 12288
zcmeHNdvIJ=dH?P{u2w5c$=S8_uq69RerRRAWhb#6JFzU;i5$sREZN3H#$N4SOPlQO
zQSRN9r6d#)+JakS$V^B<lTI?7{LyqKEoG7cTA+l&l(ec...@r41{!ds45b~)s|C{E
z...@qud*!#qyihmy65qi-}zqwjks5bce{ipehly%2=8mj5q%6bkl=f#`o%kpb({a9
zj-IXlLfglTu`jeu&1GC`-jTCTGH0cd`MfMxGq&Xv^HwHrjU1b_ax!iA*49S1daNf7
z5seu^YX8Yk#(ZhtqJU9jM2NNnf{LF{ji9jbp2N%9S*QY9Z?u&d...@?y0p}-32y!d>
zM)hB+y{;yqH29w4_Bo<|w+$wdm...@_utfg%uay)^{rLa6;%ZNJIm;M3-)3G{m-%^
z...@5dzvwv3tii$u9hgf>2?kc6K?`<*>eC6>2>U^1R?86%XpbrFW$1}0MWs+u4+CB
z`}twr0u-i...@f$fyh8vtfevf4l{boa$l*uf<v+*=7R}%HM%|;2ukpaMz1v1I5z=a
zC1FW4QN;!3IC_Q5?psV*jb?YBu4NRR`hhl6)-be5*B-RSl(02=IaA-i&U!Tnr_`d`
z^+9Kpg=z+RK%x$a9jp^VfvA?E4b%vZMst6FZUF?Y5>W33kVVbzs(>j...@vdzadam
zuKeu|k0;Uk_0;b3d>lNXkZ?Y*YQJy2>3j|_DPE8Nd*YrkS6&epplayywzyb...@i^
zpb}...@m+anka|`POCxl32=<|e8%ZSThGT1Lj`jUZLqZ7M8_#Qt^E?m9DW@(VB$om
z!CjBR(T1AD2GnsgW^MrrF?Rkyq8^ZSyJ01Hiq$%&AR-j3_U7gi3^kPhH6&hDACGkH
z7f8m$quh4+9x_epbhp61...@{^dqv&Kq_)XC?39YTtph)5|4<8sJfgP{Ip{RU=&A9
zRr8r1gjPcW{x-X!)~fHS8s4HcjwG5uxA&gcUVF_+V3?`nafb-+XzmZwcM*VyOA)x7
zY3^%v(mrg7^w55qq4g_jm8l3i{...@uu>CfTv-=rT1hkmS?j`&(RgTMJlrKjC?LWs
zvK0^!hQ47D2HU|`op%I1b=^%&Z932U0AZSPGt-;Sj{%bw<EvnQ8#j8oVzHh?J2zpu
zh$OI}Oh7OL(Vp*Ksf+dex*oyXEVlyibzxS)Kr&H_D!C0Hi3A$mer034T1AVfzA9Wm
zt7s%g...@xw4&5tjq@|;Bx<S>9pD26YSa=IQHcZ;1(nj{cfs...@*k^uspwvr#y>|
z;B|VE08+!ggJFH;!<_vg&r2W<)yq!enmpau+5k...@c-h8sgpg*oc~|obhQDg^tm9
zoiHO3`70W~LEM1EFkm?sKo`+*WQrT=R|<$SLw18k_Mi~85;@YZoG*b|_A#;_Mf^tR
zWgaeW)b^VZSk7)0h_8fv3wm...@wr|Zp59PVc&$fQcUh*^0-98i>kcU-R7KFOLH^R
z#6X2y6z%$m5F#db^ME}lw21Z21%H&{8{Z%{XnrxVUYO|Y<pGMp7r|h?W^gNj{=D{g
zEyeovY}CWS^=c~P4bF$aSkx;QY1WORQL}DwK3Bqt2GOWZ+6SgxpBD|zj~OeJGvfoX
zfy90woVclgx1pW59ff3a2<AE8`ydt}roF#>+Vuiz($z*^+rixrBdf#)wZhF#1G8-|
z)3QahXf4k;!zCOHYgSt7m}wQQI%ZTVYjy4er91#l>R03ts`w_S$b&cO2x$UOwRks*
zO{&rLZfKx1ceLh%<(fMm#aNYv+=I+)gn_hzjh*%ju0rv...@v*0+nigw?v(=<Opy&
z%iZI=%wjdDv<243+q$}iC7MK=ihg;BxwVV-?i)qB*yuwK11j2(U)n^QXp(oJJNSaw
z40hpwq...@ytmtb^k$0)MW`({OKgeJp7qWvEVNG9$_wcSNEQg)BS6NtAZRwryU>(l
z+`p+leskAWag(@trGT7e#J7sASS;hfi7jHQJj#umA?apuv$zRK?u!5mc}(m...@a)KMC
z*dp&n...@0er=bqcu&l!3K)lPCt?3QY^gT%L4{Y7uftKSyyh;L6E=TT2!lyWcFOac(!
zT_4}GyBRr9>=t`g3dog4JRuTtim9JrYToC%O|cz{ZTAs>&4?2k(E&t<^EIEb^9ov^
zI|-~jg+lfdk7i*-t$e2=qr8i...@tr&$=wt?wn#80nbni%k3dgjmy=a...@8_+a)b
z#P^E5^&oyLa7ySOiM<Wz-&59aM1Q)h--P~|vVP)i=s0_;eJ|~SaeKrbu^SWr)n8+~
z8(Bqd_gr{6(<jtef1...@no&@Z~H#JwLZQNL3pd!=Q+4jbe1_-`LPP1vyd(JLE=8K
zPuz+Cc+p?FJxKfBrnLj>aYwz{oBdsj(|+ZyRi<0iIliiLNBuz5St;xG*NQUIFe3O$
zxAQ5VpdQ`Uc$f390En(N`yA7f=;kxv&}a40_%XB*kn|khI=ue#cWB_;MBl)R-JkeO
zhjl_x%o>tB?MLm?Gkgkt3w2=){9k~)2>5-$O8WPs{{g_aA&$!V+=X)p^sk...@z{}
z0F(jjb9j-Pls=mRiF43s$HB=X2Ms=H<kc7v!ed...@j%;3}PSPQ%x4-BIIoOEu!zB
z{xEjVEt3T&lb>~&;!zm)^J>6OOwy$d`g75iJ5G#_0QWfh)39O7!K|ENVm#;?cWekm
zFofv8jh%$^A4|9J0;fKjfF+&73p4p##fewZa34B*+M2YR2GDzm#!x<{%Fm)SsoQ8W
z8t4Gnnk0>5hDq;J<#Ptp|Bb;kub^$xFH|`lxGPXcF9x1L`A;ZK`euN!|D($7L2e&X
z<^92...@ahdu<Xl)G_HZfJf=35bN-$Du1EMV_}9&tMV~behQ_DWXbefOk_wr+V9%`
zh^l...@!m3exf=uzxw`_e(Cj|A_1RFv}xQ&sS(uc+jZ|ncrZf0^m<>WPR#PNU5xwP
z#;;MInp=aujedZ>k...@s`jr`y95ucx)v}NQllX2jZXR+;&%_qJv4-}k4EW}#%<8*
zC1Zq6(+`a!l!snz...@5q&oT=&4JTsZ%6xXIuJNR8A=6a6wNF(q6...@+r*y-sm#y
zMU;=KQBMRe)BTY0hxAT*HSiS5{|G!yZDa(G03Hke86X>j&%#TC!GE9+(RlESuu|K)
z82lzZt=do1L&1w^vy^r8MDV+`o}LN*7s`(Ze?cFozYq37|1SqmgXVj|un#...@+pg
z;6tFlAyi|GQe#LMkQ~}zJWoR*ro1z>2b!D?v2}ByW+O?3kOi6tL)-A7h;?|H{v@>1
zc!542x=1h3OQ3v#zKV7Wv2HE&n...@ze@{uch3muo#8xg7s#...@$5#u852_k>7)hPDo
zgwaL^!yAlh1>dUpKTlKPGxV~ed6{OzZ!^A44}>p5=S$(!v{g...@hm}li{?n)sy&<
zak`vgdndk2e2=~...@s8;fGNEB>wi`4x10d-ck6...@a9lx$01vw@%9%V^(eUNT-*
z9x#...@mjc<|Bq-JYb#%_FbyHq{=@)$$UNt?9*l+RV$r)5tE5PFU4s9Wi#y!NP3ta
zrBBck;QQ~?W1KQV%LvGd=NUvFX9oW<ysHaC<pm##b0VHfIvlX|vu|9ci&^_N8kriO
zvfV<jy=c==u^^8iFC|guk_Ee;a&AgG*~|=0f1zcen5^lfwnn...@^eadotx@<_k2Q
zNjcJ$=L@|f85e1ArdY7k%nB?rGT-YF+gXf6pGJ?{t}Hq!dqT=A9nR#_<H^)qCU4Vd
z1>X^jDA*Om=-7od)O3ek(Bo)4nanxcsvss5q...@indcmgg~*k6`qg&)xVI;6v(k>
z?M0+-N;=5j+D=%KoXtzOkV&~PcBVKxo1DqoLj}(~4~C2nH`=SCju-QVOwR5dmbv*%
z)^;XshyCP|1}JwNx3kGb)pAShyn##x)J!IuDJ+$+-V}jL-j!LKPC1!^J*FJydxxFi
zw+}8U;|?vR?0MzzS-Ws{+;-jMtWA2KVpgeSff#da-gY1~KkH2~!^Z6VY++8JCmab2
zT<Husn4|*cu;|z&pRd|jhrssl...@c(PY-P=galt>F{(Lo=fIuSEqKekSr8kCCzKf
z`Lm;W`~G5*9pTT(6t4DqkK41wY|=tjipx9!gp*4e*^9b7rvpfe...@s6n=|id4Yz~
z=}9{!^J$kx^M##*3US!U*!gr;y...@ppzm;h1kq#QJOvHuP<|of$b8Pu7uXO?PcE5H
zv*(96...@1+bphl9fawyhp*_c%zclz_tflvz6}kwf)WR91FQ!R%fpRRlBX#RRg;s5
zSR2xcIi3ncFNBr#ct(bcj>Bq<%LV%=-*Jv&vCT`D!;*AUTkA&wdwUeZFe_^oQKj?M
zB4G)myuOcP4pn49gURAdE~9+x&Dv-QqWlN;dX27y=iNdwpR$1;OS*;8ea-?...@qq
zP~Ow%&ZLvRkaX;_p=Ip#dS=&n<VWobCCV}^(!p63?qdt+fhmwul...@md+osk|zsX
zbQaKFF7*yBO(YTWpdOR6vIIwfg8^bG&(Vgt_Er!aR9c~<oHYSKg#epK;+`I<OU>hI
z$8#d<H<_BVIs2YWW{Y~(h9y!5jrxc(t36Wk)}fZ*)2TUQj$Q)fF!u&16!%wWkqs%2
z4no%PsmY6A*Z`7Zwg5A9L(Q~qmKNtBe`&JY*><ksw=s#V!fpbel3+dhxIHhiSeSAm
z>FVWvIGM>R$Wd8<Hhyzxem<K?sn8zgK*EW8=NL)+-G`GB$jGAe=sBo!*o#v%r{FgF
z9rPge)7...@l(J#px0ye0!w!d<NTf>8_>e...@a;2pi6@;0j>k!+Ku*uG>BFw8GXmW
z$HiN~pRd>j=u=...@l6uoxd{ky>x$M3HdXSXu&QU9MZHzhT^N+)H8lve^`*`R*qbL
z_x0SG2KxmJNGV|@r!tc~m|V+Dc2Ea!iL0=4P>Mxs*I7`z#NB0AZaWOqd}o(qD<-sU
zq->l8JtSOGK2db*AvnfSp~2IAJx<peYosd2vYjc#iCvvilJd|orzU_yPa+5{MUzt&
z>rmnf$}...@xe|bI^6K-aQV3ta6}Sdn^-{|tCor)2M)I}<S6ht#h}v36A&m7y(PR0
z<};w>d38}r2rgkcCYcXA_Uc&o>H48Xr5qnK08Fp?qoGl7*3p_&>*oRV^g1k`l1Vtj
z0cKvwlhEy6$Y<Blz_8Me9m9Qpw2tUBEH2<>KQM$HPQk52hsM$4Fx0Xt^(+q!e{IZz
zo_(ZoIv|bB<vRM51;}7&xZ!O&f-y7b`8Wq2%QNQ8<TX58DVP0GT8kFAE<#`C%=~)r
zr`Kw(!V>nS5yUOT>0xhl$tTyE)k>KO<U0<og+j_n=y...@gj<?#*(U7EF-HP&ZEUq
z0qd?q;m?Lo;g-#+&bjl7+Z^-g8iBRUFAsZl1oGnYBbf6hXX8$$?YS210%)`FGTTr=
zz4;u...@x(l}oSI77$ET<su%DN^n8Yg$90naEe%j-z*sEh)hGhT&@Yvoq^<7sFT`10J&
zcCVFIuD_1vQnEDKcty;COBz<_#K3#ky<WV*C)F1E{73gxrXNmjydJK$zBFF1+=7x}
zVb$>}(q7;cU8cv`>+Lq|dtnqrm*a%m#fqod...@ko63b$-<w%Z-&D(Su4DZkGB$9w
zn1H7wJj{OP>_iO=<qXGbBLfU;i*;Vb);~xz?0cdE*fg{+<wxFiJ3JjV+(DoJ^scPi
z-Pn;!JhDo<3mWEAqSt2w9s&b...@davxft%uuy7qx$)J!Ruo~A9#ir;xG%q}M9P!H
zZqQ_UQE0Qy1CxQHeT;w1=zzO;f78jx$MVrbkJH#Bl;&yTojQpm#uMh#?gXu&0*uE{
z...@2pd5#wnprqoolk7d%jrq)@#-+(Erx?cFZy2GWL}GpZxaVTXgY?VS7n&3<*ev#
zs9XZLsd(9V?2qF|Umf}F?f3lr)DI7Ti6S3;vesa!u5...@osb?b)Bke>#8Woy+DM5
z...@z)ME)65cbwp`...@vc33plob#be2edazi83m$1u0q!wh#oxige+=p;a;SQLLh8u
zF{=Wx&R7qRsIEhaib2(}9_Gj6!5~r<WTIG)x...@r}k_pky`krdu8k^1^*fq)niL$Rvr
z*q#XLPN*ovP-{yBq^&H!wl2IOb~hM;Yn&n$Z4jr?2y;~jwc-T*0^kodA|`Bwa-an%
zZd`d(YYWJX2p7=D3dWLk5zVs&!((<Brhy;Ww<`NH7zAbEz92kMXR`EI4*1ypEH73x
znWv&PVuHuTmQ<nkwf5wxr...@f1gwmo`$;CmgRR>?FYMBWlSV?0_wssm=0LH2Rg=*
z...@d4ssgypplzmdsz=+xi{&D*`(wq}k{F6az#ji#6jAP2Q)nQ3iGSr&Ryz_gyo)b1
zBI?F#6JH`v1?o>z...@!x)z...@+x{kxh53i4{$478{v&w*aTxD5{EB08WODk_pCA5&
z...@zdewe)g3oKl6?EJ-!k6;l0x%GKJ6W`GRZVZotB4EWS%|r-#zHOrGz<_yhg)_>#3s
zH+?eW;%38|EELljX$|)g...@1dk|jha...@gzl)mqNnJB=$NJL%d!tam;=!|Gld;Zi?;
z%5BT*tf~CpB2|LV4olpk*y_qm-SXLXZ#v80ZQvJc+sHbgT5lHr7aj=kujP1pmhn3Y
zd`CmesLCGwZ3n*GT`mKnso`_ifHU1P7Zs0=;`amg<T;`d4r1?l6%q4&Kga<a3E(Fb
zbP}hlvnUVYS4xvOK_0`WhqI`U(qXju`DExvzh*twKKgFC^gUELo<q?9S3WBpR$Cix
zcCl^nCtQBEQV8V~ut`8&Yz(}I>a(lfpbv+hf=...@%e=r|5tNbRy6%MU+TYI*#7##
zpc_lyse4adzvz`nnsa~...@id?mcu&pf1t$WBV$-Ja_9r&(`?oEIvUUg$2Bo!mqGv
z9irm*^r...@pyk73~fi5wqjiw&EvB>TQ`dnB0eDz>vKTyKjy*Na;EleUp<*#SI#bF
zk3M_xnUg<jt+lft58k...@8n|H{WIO$J;D4vwq_^45c6a<zMZ)eKD7{a1qM)WBc0&
zdi&cgJD-w#7rejy#MI%Q-R%}%2&R)+nYZ`1FWGMU?YGrN_u&@E&dp?(EHKEs``e37
z...@w`t6b9u$@y...@^ru2nz_>^6E)29=xPHx?$F~(H%cX&pWvwzBO(pf...@3=e1qwc
z^0<}T-#)(NT`#FGCzA8??R}a<!71`(<$3uA^%-ne;~?9$Q$^g#FL`}H99&*QPdh!~
zwesvmz^v`ql9rp}t`?pwaefg...@h(7woK+<#K;}(jCn&$op)k-702=QhZUrzx{kN
z>)M`Cib>x!wDDE%tI&O4->MTpu&>XbA#`}Q-+bP%2M*}p4t$!^?3>U3I}iLXAgw%l
delta 4268
zcmai2...@x6@Rn0_u;j...@$a~`v+vgxf1idk_8ap#c$sto;3&pptuyu{$yce...@9c
z;wFXmQV1bb!A&}-0!K}uYN=2uF(fuhevqs23tj$4sa~c8...@nfiun&(V#Z{&E8$?
zgg?5MdGp?H-h1=r&70YE_U_ob;n0C~_kXf>Jr$p~NMYqHK-2&LoqV2J0b-%`p-Ez)
zQ;1...@+tms_wxxs!emkuk#hz;tLouKqhbe91>`4}6GcR$k`P;2yLKW1$U<un-(t;W
zec*g(miReaT($!oCZINPmNi+2<otN9K~u;}lQB3qh!|uvpe%u?5*...@azgsd>D<}
zubd~tj7wk=b25_if-`4...@7yrlogtshurxlbdz0-e5^&KO2L!W~k1mC9^EnuNhuv
z2`l>HUY}hCjC-Bif!UfI_c>GG{qUP7t=jN&pWiN*1OBu1M!=cFAP?9`fEKHiL<V%0
zxIeLi`|Tiv#N|ccuif...@owj@o<qT;u...@r<oSRbjpBA&Th`<1...@~0wdpitcik0@
zVn`4I4RD`b4bH9sX+uP1g^0?&7?C$nSdivaDRgJRO%(TY%{HYczg>&y...@2;02;9=2
zBKyvy6y=KKs`G_qbH8&6uw4h(I)E_TFoC{aLeI7t<%KZHzYxZ}Mf^$Uos#=_>Y}t(
z6s#*Qw$3B3^LhC~#L4HKUQaPxXS~G4>JesbHVkT1I{N|hN_Xio6sTr2>&?wrD4rYp
zVo`}Kir+pTi}Rc`DDF#s;vhDJ6%*&~mC<4vt??Ld;jObdS+YhIujExWPM2mc21)pM
z...@uc<jJMWqF<N;mZ<(36>`(JaaHHUhVit6h2b^>b...@zoklxynppohwy@5QIzApF6j+
z4T!rd3m3<^x6E#oE6}-)8tso}-owv5gzm$hih#yjfuo^{...@6ulxp)p%7#tVWt+T
z&<bK0**4i#bFbY_l-G<JNAs2{qQ;%w#$nZXjol%QWPv9dovurDI;BoEwBlgm&v1iR
z+X-oq1S!m6I-5#Xu3w{;*B1Qg+~i&eU3qEYdiXOum<12r01;ja...@tofm4ajvie5
zMB7ZGJwC+Sc>8P)HOq{8&Uw9kne^svQM>an%!<(bA?6_-1bDfepbooNm*?&1+Ic%~
z!zLc%b;)YysI=s6j^&2o2oUTHyH{F;kkcz9<K-dyN-)kjsjePGO%}?iP=ql(#...@l=
znMT+&B1`VV<z}uj6%q9...@i^e{qp-=#0mc2k)HCp{kh?<xzWybiT!ht&wLJMHfb-
zVcrSa$vb%m;{8KxsZ^ShO7cWw7DTlSoDnyT#F9^N26&t&ie4L>yGxqRK5)Fj&f|jD
zY+Tlwdk%oOfJ|g;C5Usfpm&u7<TVV9O9C^T&!molkyrxvc*gp9oi6p*...@6gua3g
zl00cbxesFp^YZllfG2AK^h=;9haj(BAlE@&vp|jwKycoG)5292DcWyA1`^0nf+u)<
zW?~hcKY!kN`+}NGxy<;?L^pT$DPKIN%WnPe`SWfzO9|{UjqZ4Yck%9G_7?NS3zj|z
zF9|=OtHJPYOxex5c^5Y2)cjTR|F6|Nu6L_&Gg=qljPixEAY9G)_`)h8W{>794_sKg
znW|i<>P6L_O*l7|6ikrwqtuq^0b^i...@mfi8txm_dksk!e-m7eea<hLsd&2`$psk
z#r9Ag`i1yT$X&p{e+hmXvh;K268r_!k<|YHeak{eC^)LY8+{WR?rnGC1|_yuJXaaJ
zY5*_K=5G+qq2Ge<g67ejGck2*7J3xG8~$Ou`I|S7(xZ6a;#Ew|Yc~&}NN5i7AtoAv
zZBZO8_Y*~arXVI-iw18l&qO*H&im+Qd<d1(g*bp?i0%xzold=r9`o...@vevmog4e
zX>^ZE9|!f(Z-D#gRhKSTI+Y;(Re28d45*LZQ{<@6BW)`@(smfsM<+awc...@m55h9
z%7N=$dKajVo>is6%izV8i(2xlATS!o4NbH^G=QZRR<n*c4Nx^Tg4R<9xcc1...@dg
zDq}4!-$DJ<!n$e59UY?ESuglQY$<J|i...@bczkq1ii_ixe+(r...@thc&{=Dx>y;_
z(@t8gY;oOfhuJmC2wdEtWH3CXOoDzBsuOfjalp?iz0f?S?4~`K@@{&7K347p{aiUf
zk5IK|5O}L+21wE);Ag4l2t7tyJ<{h6Pd#*YLT8BfdLBbD$R*{6=(Oh;Jxw2ZdO<(K
z...@ybt9ujpsm3dbvw3%d@VU&Ab=z65^2<<s...@l6?~uuscsq`m5cdxxmfujsd&PD1}D
z>YLDc0XnbIX?4(D<l7i6Ab(T;NiXN!=wxC)gwlIZdW_DgpMbCSj(|p8+UC+8P^tMj
z^0CJ2V06S=MP=?fx=>`>SQi<T0S(g>Q`i%v?!~TN^!y!?+~Cq9rgXD5DU0_am|-Fk
zkLXvI4W}ojsB0`|i)W0ViI-|p;!S;}h?t?Kp-q{ry~`QPY_RP~TARr&lvnS$HhqW4
zRR2&+n=7kFbLrf!EZKM7G&Ggjxhp+6k=r90YRtf<OnQ9%#8gJCui5M$8pupfjAg{5
zHErTWbFB#1)`?GRnnZ8yh^K#iT-;N;?uwz&vD-4^yCyTV#>u2}nQJp&y?`...@t6!
z8UiI46x=}&`$N}>cSC6t0Sp$Wr#0!RvCO1gxHu8cuy2d{NTPg<CXw6>O;JuPk6bVQ
z7}-<bzn*ot...@%x|J-t1(Jh)OdP3e7yIhcp^;%g(-#zSbQ^pW6r#e3R(Gf9>(oxH
zd=v0AjiXHIrZvz?yDm$P7utmf+9?YkS;|2xN4s1phj}b`l76yiIq9`>GrVM>bqf%3
z=O~QPusCFen2a+TDwy636${Z{2%}rf)-~u8uJ;9SqnL>{s1{l=c%nx~vi=x...@nx
z)yTzISVYEM@)-iob%^qkru#e`...@bg~|Wl_EE92p;5dsucu5soeh^m...@9u%y#w
z...@py?j*<g8q2*1_DyrRoWv+W4_%kB79tQo?(hxv9vx_...@uskl6@C`oK<s$DF{0d9
z{cmme>msztd...@j=u5yv0qw(*{^N8uCDj32fV}r(I94EUW+kase%SnVl0U7n`qFB
zKXro<$dy4k5`>c8;nJiulH<xkm...@ed>Js=|t}U3K%zi&}YG(_rn(-#26ox6DUDH
z...@semi%ak<<U0=13_<%zL`a^etoO-V-T_wm6Wk)1XSOSAV^R9;Zz^iGjc%R=+bl^
zvo...@ss51oz}jjhfl^ijcoo&f_N(RR=q<{>OJ^%xOAmH%m?9XWi+C1bf?$1x?NCD
z2lpx...@#cu=ish(MSOQN_{i}A;1dvf;iBcN|yeUKwcn+(N5yq?j&BUn??sl&wTHL
z=~s^aWBAB}KYsuB_l-Z~6K}WX^?k)Z+o4QHC2t*WG?hauj(@jpHxqrVN(5tB-bXYb
feHNd0S0DdQ%=VVydtJWn=T;o&T_3T-DgXZf6xhmD
diff --git a/client/tests/kvm/tests/whql_submission.py
b/client/tests/kvm/tests/whql_submission.py
index 8dd48b8..a0ff87c 100644
--- a/client/tests/kvm/tests/whql_submission.py
+++ b/client/tests/kvm/tests/whql_submission.py
@@ -6,20 +6,24 @@ import kvm_subprocess, kvm_test_utils, kvm_utils,
rss_file_transfer
def run_whql_submission(test, params, env):
"""
WHQL submission test:
- 1) Log into the guest (the client machine) and into a DTM server machine
+ 1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server
machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
- (logs and HTML reports are placed in test.bindir)
+ (logs and HTML reports are placed in test.debugdir)
@param test: kvm test object
@param params: Dictionary with the test parameters
@param env: Dictionary with test environment.
"""
- vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
- session = kvm_test_utils.wait_for_login(vm, 0, 240)
+ # Log into all client VMs
+ vms = []
+ sessions = []
+ for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+ vms.append(kvm_test_utils.get_living_vm(env, vm_name))
+ sessions.append(kvm_test_utils.wait_for_login(vms[-1], 0, 240))
# Collect parameters
server_address = params.get("server_address")
@@ -30,47 +34,54 @@ def run_whql_submission(test, params, env):
dsso_test_binary = params.get("dsso_test_binary",
"deps/whql_submission_15.exe")
dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary)
- test_device = params.get("test_device")
- job_filter = params.get("job_filter", ".*")
+ dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
+ "deps/whql_delete_machine_15.exe")
+ dsso_delete_machine_binary = kvm_utils.get_path(test.bindir,
+ dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
- wtt_services = params.get("wtt_services")
- # Restart WTT service(s) on the client
- logging.info("Restarting WTT services on client")
- for svc in wtt_services.split():
- kvm_test_utils.stop_windows_service(session, svc)
- for svc in wtt_services.split():
- kvm_test_utils.start_windows_service(session, svc)
-
- # Run whql_pre_command
- if params.get("whql_pre_command"):
- session.cmd(params.get("whql_pre_command"),
- int(params.get("whql_pre_command_timeout", 600)))
-
- # Copy dsso_test_binary to the server
- rss_file_transfer.upload(server_address, server_file_transfer_port,
- dsso_test_binary, server_studio_path, timeout=60)
+ # Copy dsso binaries to the server
+ for filename in dsso_test_binary, dsso_delete_machine_binary:
+ rss_file_transfer.upload(server_address, server_file_transfer_port,
+ filename, server_studio_path, timeout=60)
# Open a shell session with the server
server_session = kvm_utils.remote_login("nc", server_address,
server_shell_port, "", "",
- session.prompt, session.linesep)
- server_session.set_status_test_command(session.status_test_command)
+ sessions[0].prompt,
+ sessions[0].linesep)
+ server_session.set_status_test_command(sessions[0].status_test_command)
- # Get the computer names of the server and client
+ # Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
- client_name = session.cmd_output(cmd).strip()
- session.close()
+ client_names = [session.cmd_output(cmd).strip() for session in sessions]
- # Run the automation program on the server
+ # Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
+ for client_name in client_names:
+ cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
+ server_name, client_name)
+ server_session.cmd(cmd, print_func=logging.debug)
+
+ # Reboot the client machines
+ sessions = kvm_utils.parallel((kvm_test_utils.reboot, (vm, session))
+ for vm, session in zip(vms, sessions))
+
+ # Run whql_pre_command and close the sessions
+ if params.get("whql_pre_command"):
+ for session in sessions:
+ session.cmd(params.get("whql_pre_command"),
+ int(params.get("whql_pre_command_timeout", 600)))
+ session.close()
+
+ # Run the automation program on the server
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
server_name,
- client_name,
- "%s_pool" % client_name,
- "%s_submission" % client_name,
- test_timeout)
+ "%s_pool" % client_names[0],
+ "%s_submission" % client_names[0],
+ test_timeout,
+ " ".join(client_names))
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
@@ -89,13 +100,13 @@ def run_whql_submission(test, params, env):
# Tell the automation program which device to test
find_prompt("Device to test:")
- server_session.sendline(test_device)
+ server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt("Jobs to run:")
- server_session.sendline(job_filter)
+ server_session.sendline(params.get("job_filter", ".*"))
- # Give the automation program all the device data supplied by the user
+ # Set submission DeviceData
find_prompt("DeviceData name:")
for dd in kvm_utils.get_sub_dict_names(params, "device_data"):
dd_params = kvm_utils.get_sub_dict(params, dd)
@@ -104,8 +115,7 @@ def run_whql_submission(test, params, env):
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
- # Give the automation program all the descriptor information supplied by
- # the user
+ # Set submission descriptors
find_prompt("Descriptor path:")
for desc in kvm_utils.get_sub_dict_names(params, "descriptors"):
desc_params = kvm_utils.get_sub_dict(params, desc)
@@ -113,6 +123,31 @@ def run_whql_submission(test, params, env):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
+ # Set machine dimensions for each client machine
+ for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+ vm_params = kvm_utils.get_sub_dict(params, vm_name)
+ find_prompt(r"Dimension name\b.*:")
+ for dp in kvm_utils.get_sub_dict_names(vm_params, "dimensions"):
+ dp_params = kvm_utils.get_sub_dict(vm_params, dp)
+ if dp_params.get("dim_name") and dp_params.get("dim_value"):
+ server_session.sendline(dp_params.get("dim_name"))
+ server_session.sendline(dp_params.get("dim_value"))
+ server_session.sendline()
+
+ # Set extra parameters for tests that require them (e.g. NDISTest)
+ for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+ vm_params = kvm_utils.get_sub_dict(params, vm_name)
+ find_prompt(r"Parameter name\b.*:")
+ for dp in kvm_utils.get_sub_dict_names(vm_params, "device_params"):
+ dp_params = kvm_utils.get_sub_dict(vm_params, dp)
+ if dp_params.get("dp_name") and dp_params.get("dp_regex"):
+ server_session.sendline(dp_params.get("dp_name"))
+ server_session.sendline(dp_params.get("dp_regex"))
+ # Make sure the prompt appears again (if the device isn't found
+ # the automation program will terminate)
+ find_prompt(r"Parameter name\b.*:")
+ server_session.sendline()
+
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info,
@@ -162,38 +197,63 @@ def run_whql_submission(test, params, env):
except (KeyError, OSError):
pass
- # Print result summary
- logging.info("")
- logging.info("Result summary:")
- name_length = max(len(r.get("job", "")) for r in results)
- fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
- logging.info(fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun",
- "NotApplicable"))
- logging.info(fmt % ("--", "---", "------", "----", "----", "------",
- "-------------"))
- for r in results:
- logging.info(fmt % (r.get("id"), r.get("job"), r.get("status"),
- r.get("pass"), r.get("fail"), r.get("notrun"),
- r.get("notapplicable")))
- logging.info("(see logs and HTML reports in %s)" % test.debugdir)
-
- # Kill the VM and fail if the automation program did not terminate on time
+ # Print result summary (both to the regular logs and to a file named
+ # 'summary' in test.debugdir)
+ def print_summary_line(f, line):
+ logging.info(line)
+ f.write(line + "\n")
+ if results:
+ # Make sure all results have the required keys
+ for r in results:
+ r["id"] = str(r.get("id"))
+ r["job"] = str(r.get("job"))
+ r["status"] = str(r.get("status"))
+ r["pass"] = int(r.get("pass", 0))
+ r["fail"] = int(r.get("fail", 0))
+ r["notrun"] = int(r.get("notrun", 0))
+ r["notapplicable"] = int(r.get("notapplicable", 0))
+ # Sort the results by failures and total test count in descending order
+ results = [(r["fail"],
+ r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"],
+ r) for r in results]
+ results.sort(reverse=True)
+ results = [r[-1] for r in results]
+ # Print results
+ logging.info("")
+ logging.info("Result summary:")
+ name_length = max(len(r["job"]) for r in results)
+ fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
+ f = open(os.path.join(test.debugdir, "summary"), "w")
+ print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
+ "NotRun", "NotApplicable"))
+ print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
+ "------", "-------------"))
+ for r in results:
+ print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
+ r["pass"], r["fail"], r["notrun"],
+ r["notapplicable"]))
+ f.close()
+ logging.info("(see logs and HTML reports in %s)" % test.debugdir)
+
+ # Kill the client VMs and fail if the automation program did not terminate
+ # on time
if not done:
- vm.destroy()
+ kvm_utils.parallel(vm.destroy for vm in vms)
raise error.TestFail("The automation program did not terminate "
"on time")
- # Fail if there are failed or incomplete jobs (kill the VM if there are
- # incomplete jobs)
- failed_jobs = [r.get("job") for r in results
- if r.get("status", "").lower() == "investigate"]
- running_jobs = [r.get("job") for r in results
- if r.get("status", "").lower() == "inprogress"]
+ # Fail if there are failed or incomplete jobs (kill the client VMs if there
+ # are incomplete jobs)
+ failed_jobs = [r["job"] for r in results
+ if r["status"].lower() == "investigate"]
+ running_jobs = [r["job"] for r in results
+ if r["status"].lower() == "inprogress"]
errors = []
if failed_jobs:
errors += ["Jobs failed: %s." % failed_jobs]
if running_jobs:
- vm.destroy()
+ for vm in vms:
+ vm.destroy()
errors += ["Jobs did not complete on time: %s." % running_jobs]
if errors:
raise error.TestFail(" ".join(errors))
diff --git a/client/tests/kvm/tests_base.cfg.sample
b/client/tests/kvm/tests_base.cfg.sample
index 8d533ba..e81d879 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -347,11 +347,12 @@ variants:
server_shell_port = 10022
server_file_transfer_port = 10023
server_studio_path = %programfiles%\Microsoft Driver Test
Manager\Studio
+ dsso_test_binary = deps/whql_submission_15.exe
+ dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
wtt_services = wttsvc
variants:
- client_install:
type = whql_client_install
- dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
# The username and password are required for accessing the DTM
client
# installer binary shared by the server
server_username = administrator
@@ -363,7 +364,7 @@ variants:
- submission: client_install
type = whql_submission
extra_params += " -snapshot"
- dsso_test_binary = deps/whql_submission_15.exe
+ restart_vm = yes
test_timeout = 3600
device_data = cat0 cat1 cat2 cat3 logoarch logoos whqlos
whqlqual prog desc filter virt
descriptors = desc1 desc2 desc3
--
1.7.3.3
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
