[RFC] Support firecracker - addendum
Tweaked virtio device initialization to conform to more
stricter enforcement of protocol on firecracker side
Makes OSv boot on newest firecracker.
Signed-off-by: Waldemar Kozaczuk
---
drivers/virtio-blk.cc | 5 -
drivers/virtio-net.cc | 21 ++---
drivers/virtio-net.hh | 7 +++
drivers/virtio2.cc| 5 +++--
4 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/drivers/virtio-blk.cc b/drivers/virtio-blk.cc
index 3784c453..148b036f 100644
--- a/drivers/virtio-blk.cc
+++ b/drivers/virtio-blk.cc
@@ -122,6 +122,9 @@ blk::blk(mmio_device& _dev)
setup_features();
read_config();
+// Generic init of virtqueues
+probe_virt_queues();
+
//register the single irq callback for the block
sched::thread* t = sched::thread::make([this] { this->req_done(); },
sched::thread::attr().name("virtio-blk"));
@@ -318,7 +321,7 @@ hw_driver* blk::probe(hw_device* dev)
//once we have a virtio_device class
if (auto mmio_dev = dynamic_cast(dev)) {
if (mmio_dev->get_id() == hw_device_id(0x0, VIRTIO_ID_BLOCK)) {
-debug_early("virtio-blk::probe() -> found virtio-mmio device
...\n");
+//debug_early("virtio-blk::probe() -> found virtio-mmio device
...\n");
return new blk(*mmio_dev);
}
}
diff --git a/drivers/virtio-net.cc b/drivers/virtio-net.cc
index 9ad5d48a..3ae9ab9e 100644
--- a/drivers/virtio-net.cc
+++ b/drivers/virtio-net.cc
@@ -224,12 +224,26 @@ bool net::ack_irq()
return false;
}
+void net::pre_init()
+{
+_driver_name = "virtio-net";
+virtio_i("VIRTIO NET INSTANCE");
+_id = _instance++;
+
+setup_features();
+read_config();
+
+// Generic init of virtqueues
+probe_virt_queues();
+}
+
//TODO: For now this driver is hardcoded to expect mmio_device
// but eventually we could introduce some sort of virtio_device
// interface class that pci_device and mmio_device would implement/extend
// from.
net::net(mmio_device& dev)
: virtio_mmio_driver(dev),
+ _bla(this),
_rxq(get_virt_queue(0), [this] { this->receiver(); }),
_txq(this, get_virt_queue(1))
{
@@ -237,13 +251,6 @@ net::net(mmio_device& dev)
poll_task->set_priority(sched::thread::priority_infinity);
-_driver_name = "virtio-net";
-virtio_i("VIRTIO NET INSTANCE");
-_id = _instance++;
-
-setup_features();
-read_config();
-
//TODO: Legacy vs non-legacy -> the non-legacy header includes one more
field
_hdr_size = sizeof(net_hdr_mrg_rxbuf);
diff --git a/drivers/virtio-net.hh b/drivers/virtio-net.hh
index 6edca97a..e9c9c667 100644
--- a/drivers/virtio-net.hh
+++ b/drivers/virtio-net.hh
@@ -208,6 +208,7 @@ public:
explicit net(mmio_device& dev);
virtual ~net();
+void pre_init();
virtual std::string get_name() const { return _driver_name; }
void read_config();
@@ -301,6 +302,12 @@ private:
wakeup_stats tx_wakeup_stats;
};
+struct bla {
+bla(virtio::net *_net) {
+ _net->pre_init();
+}
+} _bla;
+
/* Single Rx queue object */
struct rxq {
rxq(vring* vq, std::function poll_func)
diff --git a/drivers/virtio2.cc b/drivers/virtio2.cc
index 09410888..cb4f1253 100644
--- a/drivers/virtio2.cc
+++ b/drivers/virtio2.cc
@@ -44,10 +44,11 @@ virtio_mmio_driver::virtio_mmio_driver(mmio_device& dev)
reset_host_side();
// Acknowledge device
-add_dev_status(VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER);
+add_dev_status(VIRTIO_CONFIG_S_ACKNOWLEDGE);
+add_dev_status(VIRTIO_CONFIG_S_DRIVER);
// Generic init of virtqueues
-probe_virt_queues();
+//probe_virt_queues();
}
virtio_mmio_driver::~virtio_mmio_driver()
--
2.19.1
--
You received this message because you are subscribed to the Google Groups "OSv
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
On Sunday, January 6, 2019 at 8:50:39 AM UTC-5, Nadav Har'El wrote:
>
> Hi, that's great work, very impressive - a lot of separate things you had
> to fix here.
>
> As Pekka also noted, it will be helpful to subdivide this patch into
> smaller patches so it will become clearer to me (who is not familiar with
> firecracker or the recent advances in virtio) what part of this patch does
> what. I would be especially happy if we could get all the
> "non-legacy-virtio" changes committed first, as Pekka noted they can be
> tested on regular qemu without firecracker, and then your firecracker code
> at least doesn't need to worry about that issue.
>
> For now I'll just go through this patch quickly, and probably make some
> not-very-smart comments.
>
> On Sat, Jan 5, 2019 at 8:44 AM Waldemar Kozaczuk > wrote:
>
> This patch provides all necessary changes to OSv
> to make it boot on AWS firecracker.
>
> The curl command exmaple to create and start instance with
> single block device:
>
> ./scripts/setup-block.sh && ./scripts/create-instance.sh &&
> ./scripts/start-instance.sh
>
> where:
> curl --unix-socket /tmp/firecracker.socket -i \
> -X PUT 'http://localhost/drives/rootfs' \
> -H 'Accept: application/json' \
> -H 'Content-Type: application/json' \
> -d '{
> "drive_id": "rootfs",
> "path_on_host":
> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",
> "is_root_device": false,
> "is_read_only": false
> }'
>
>
> Wow, I think that's the first time I saw sending an HTTP request to a
> UNIX-domain socket :-) But of course, no reason not to.
>
As one can imagine it should not be difficult to take advantage of it and
build a middleware component with regular HTTP REST API that could drive
firecracker to create and start OSv nano-VMs.
>
>
> curl --unix-socket /tmp/firecracker.socket -i \
> -X PUT 'http://localhost/boot-source' \
> -H 'Accept: application/json' \
> -H 'Content-Type: application/json' \
> -d '{
> "kernel_image_path":
> "/home/wkozaczuk/projects/osv/build/release/loader-stripped.elf",
> "boot_args": "--bootchart /hello"
> }'
>
> curl --unix-socket /tmp/firecracker.socket -i \
> -X PUT 'http://localhost/actions' \
> -H 'Accept: application/json' \
> -H 'Content-Type: application/json'\
> -d '{
> "action_type": "InstanceStart"
> }'
>
> Signed-off-by: Waldemar Kozaczuk >
> ---
> Makefile | 2 +
> arch/x64/arch-setup.cc | 180 +++---
> arch/x64/boot.S | 37 -
> arch/x64/loader.ld | 2 +-
> arch/x64/power.cc| 14 +-
> arch/x64/smp.cc | 16 +-
> drivers/acpi.cc | 10 +-
> drivers/virtio-assign.cc | 3 +-
> drivers/virtio-blk.cc| 78 ++
> drivers/virtio-blk.hh| 10 +-
> drivers/virtio-mmio.cc | 128
> drivers/virtio-mmio.hh | 132
> drivers/virtio-net.cc| 86 +--
> drivers/virtio-net.hh| 10 +-
> drivers/virtio-vring.cc | 25 ++-
> drivers/virtio-vring.hh | 10 +-
> drivers/virtio.hh| 18 ++-
> drivers/virtio2.cc | 289 +++
> drivers/virtio2.hh | 115 ++
> include/osv/virtio-assign.hh | 2 +-
> loader.cc| 3 +-
> 21 files changed, 1039 insertions(+), 131 deletions(-)
> create mode 100644 drivers/virtio-mmio.cc
> create mode 100644 drivers/virtio-mmio.hh
> create mode 100644 drivers/virtio2.cc
> create mode 100644 drivers/virtio2.hh
>
> diff --git a/Makefile b/Makefile
> index 68043485..cf571632 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -819,9 +819,11 @@ drivers += $(libtsm)
> drivers += drivers/vga.o drivers/kbd.o drivers/isa-serial.o
> drivers += arch/$(arch)/pvclock-abi.o
> drivers += drivers/virtio.o
> +drivers += drivers/virtio2.o
> drivers += drivers/virtio-vring.o
> drivers += drivers/virtio-net.o
> drivers += drivers/virtio-assign.o
> +drivers += drivers/virtio-mmio.o
> drivers += drivers/vmxnet3.o
> drivers += drivers/vmxnet3-queues.o
> drivers += drivers/virtio-blk.o
> diff --git a/arch/x64/arch-setup.cc b/arch/x64/arch-setup.cc
> index 6e4833cf..58da00c5 100644
> --- a/arch/x64/arch-setup.cc
> +++ b/arch/x64/arch-setup.cc
> @@ -22,6 +22,19 @@
> #include
> #include "dmi.hh"
>
> +// Not sure if Linux zero page is always located at this place
> +// in memory or its address is passed in one of the registers
> +// -> double check
> +#define ZERO_PAGE_START 0x7000
>
>
> This indeed what they have in
> https://github.com/firecracker-microvm/firecracker/blob/master/x86_64/src/layout.rs
> I don't know where they got this. In the final patch it will be useful to
> get more comments on the purpose of all this
> stuff. The term "zero page" is particularly confusing because
Re: [RFC] Support firecracker
On Sun, Jan 6, 2019 at 5:49 PM Asias He wrote:
>
>
> On Sun, Jan 6, 2019 at 11:32 AM Waldek Kozaczuk
> wrote:
>
>>
>>
>> On Saturday, January 5, 2019 at 4:09:23 AM UTC-5, דור לאור wrote:
>>>
>>> Great stuff Waldek! Is unix socket the normal way to pass parameters for
>>> Firecracker?
>>>
>> I think so. There is also experimental vsock interface but I am not
>> familiar with it and not sure what purpose of it would be (
>> https://github.com/firecracker-microvm/firecracker/issues/650).
>>
>
> Vsock was added for easier guest host communication with zero
> configuration (I happen to the author of the vsock kernel module). In
> theory, they can use vsock for anything that a socket is useful between
> host and guest.
> Btw, there are some work to implement nfs over vsock.
>
Good point, this way they can preboot OSv and run the app with the vsock
control (although any guest can
be controlled this way too)
>
>
>>
>>> How's the boot speed vs Qemu?
>>>
>> I have posted some numbers in my other email to the group but here are
>> some more details.
>>
>> First of all my numbers come from running tests on my 5-years old MacBook
>> pro that I have been using for all my OSv development in last three years.
>> It is 4-core 2.3 GHz i7 machine. Not sure how it compares to newer models
>> but given Moore's law does not work any more it might be still pretty fast.
>>
>> Architecture:x86_64
>> CPU op-mode(s): 32-bit, 64-bit
>> Byte Order: Little Endian
>> CPU(s): 8
>> On-line CPU(s) list: 0-7
>> Thread(s) per core: 2
>> Core(s) per socket: 4
>> Socket(s): 1
>> NUMA node(s):1
>> Vendor ID: GenuineIntel
>> CPU family: 6
>> Model: 70
>> Model name: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
>> Stepping:1
>> CPU MHz: 898.037
>> CPU max MHz: 3500.
>> CPU min MHz: 800.
>> BogoMIPS:4589.68
>> Virtualization: VT-x
>> L1d cache: 32K
>> L1i cache: 32K
>> L2 cache:256K
>> L3 cache:6144K
>> L4 cache:131072K
>>
>> Additionally firecracker startup/configuration is more fine-grained than
>> QEMU which does everything in one shot (start VMM process, configure
>> resources, start guest, etc). With firecracker it is broken down like
>> follows:
>>
>>1. Start VMM process which listen on socket for API calls (I have not
>>measured it but seems very fast).
>>2. Make API call to:
>> - create instance that specified number of vCPUs, memory and
>> kernel loader file path
>> - configure block device
>> - configure networking device
>> - all these calls seem to take less than 1ms and can be executed
>> in any order it seems
>>3. Make API call to start instance that eventually starts guest
>>
>> So here are some log snippets (2,3) and OSv bootchart from running
>> native-example rofs image (no networking):
>>
>> Add block device:
>> 2019-01-05T20:21:44.*096*617431 [anonymous-instance:INFO:api_server/src/
>> http_service.rs:599] The API server received a synchronous Put request
>> on "/drives/rootfs" with body "{n "drive_id": "rootfs",n "path_on_host":
>> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
>> false,n "is_read_only": falsen }".
>> 2019-01-05T20:21:44.*096*659174 [anonymous-instance:INFO:api_server/src/
>> http_service.rs:565] The synchronous Put request on "/drives/rootfs"
>> with body "{n "drive_id": "rootfs",n "path_on_host":
>> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
>> false,n "is_read_only": falsen }" was executed successfully. Status code:
>> 204 No Content.
>>
>> Create instance:
>> 2019-01-05T20:21:44.*1085*43339 [anonymous-instance:INFO:api_server/src/
>> http_service.rs:599] The API server received a synchronous Put request
>> on "/boot-source" with body "{n "kernel_image_path":
>> "/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
>> "boot_args": "--bootchart /hello"n }".
>> 2019-01-05T20:21:44.*108*584295 [anonymous-instance:INFO:api_server/src/
>> http_service.rs:565] The synchronous Put request on "/boot-source" with
>> body "{n "kernel_image_path":
>> "/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
>> "boot_args": "--bootchart /hello"n }" was executed successfully. Status
>> code: 204 No Content.
>>
>> Start instance that starts guest and terminates the process eventually:
>> 2019-01-05T20:21:44.119820357 [anonymous-instance:INFO:api_server/src/
>> http_service.rs:599] The API server received a synchronous Put request
>> on "/actions" with body "{n "action_type": "InstanceStart"n }".
>> 2019-01-05T20:21:44.119837722 [anonymous-instance:INFO:vmm/src/
>> lib.rs:1104] VMM received instance start command
>> 2019-01-05T20:21:44.119903817 [anonymous-instance:INFO:vmm/src/
>> vstate.rs:97] Guest memory starts at 7faddec0
>>
>>
Re: [RFC] Support firecracker
On Sun, Jan 6, 2019 at 11:32 AM Waldek Kozaczuk
wrote:
>
>
> On Saturday, January 5, 2019 at 4:09:23 AM UTC-5, דור לאור wrote:
>>
>> Great stuff Waldek! Is unix socket the normal way to pass parameters for
>> Firecracker?
>>
> I think so. There is also experimental vsock interface but I am not
> familiar with it and not sure what purpose of it would be (
> https://github.com/firecracker-microvm/firecracker/issues/650).
>
Vsock was added for easier guest host communication with zero configuration
(I happen to the author of the vsock kernel module). In theory, they can
use vsock for anything that a socket is useful between host and guest.
Btw, there are some work to implement nfs over vsock.
>
>> How's the boot speed vs Qemu?
>>
> I have posted some numbers in my other email to the group but here are
> some more details.
>
> First of all my numbers come from running tests on my 5-years old MacBook
> pro that I have been using for all my OSv development in last three years.
> It is 4-core 2.3 GHz i7 machine. Not sure how it compares to newer models
> but given Moore's law does not work any more it might be still pretty fast.
>
> Architecture:x86_64
> CPU op-mode(s): 32-bit, 64-bit
> Byte Order: Little Endian
> CPU(s): 8
> On-line CPU(s) list: 0-7
> Thread(s) per core: 2
> Core(s) per socket: 4
> Socket(s): 1
> NUMA node(s):1
> Vendor ID: GenuineIntel
> CPU family: 6
> Model: 70
> Model name: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
> Stepping:1
> CPU MHz: 898.037
> CPU max MHz: 3500.
> CPU min MHz: 800.
> BogoMIPS:4589.68
> Virtualization: VT-x
> L1d cache: 32K
> L1i cache: 32K
> L2 cache:256K
> L3 cache:6144K
> L4 cache:131072K
>
> Additionally firecracker startup/configuration is more fine-grained than
> QEMU which does everything in one shot (start VMM process, configure
> resources, start guest, etc). With firecracker it is broken down like
> follows:
>
>1. Start VMM process which listen on socket for API calls (I have not
>measured it but seems very fast).
>2. Make API call to:
> - create instance that specified number of vCPUs, memory and kernel
> loader file path
> - configure block device
> - configure networking device
> - all these calls seem to take less than 1ms and can be executed in
> any order it seems
>3. Make API call to start instance that eventually starts guest
>
> So here are some log snippets (2,3) and OSv bootchart from running
> native-example rofs image (no networking):
>
> Add block device:
> 2019-01-05T20:21:44.*096*617431 [anonymous-instance:INFO:api_server/src/
> http_service.rs:599] The API server received a synchronous Put request on
> "/drives/rootfs" with body "{n "drive_id": "rootfs",n "path_on_host":
> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
> false,n "is_read_only": falsen }".
> 2019-01-05T20:21:44.*096*659174 [anonymous-instance:INFO:api_server/src/
> http_service.rs:565] The synchronous Put request on "/drives/rootfs" with
> body "{n "drive_id": "rootfs",n "path_on_host":
> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
> false,n "is_read_only": falsen }" was executed successfully. Status code:
> 204 No Content.
>
> Create instance:
> 2019-01-05T20:21:44.*1085*43339 [anonymous-instance:INFO:api_server/src/
> http_service.rs:599] The API server received a synchronous Put request on
> "/boot-source" with body "{n "kernel_image_path":
> "/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
> "boot_args": "--bootchart /hello"n }".
> 2019-01-05T20:21:44.*108*584295 [anonymous-instance:INFO:api_server/src/
> http_service.rs:565] The synchronous Put request on "/boot-source" with
> body "{n "kernel_image_path":
> "/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
> "boot_args": "--bootchart /hello"n }" was executed successfully. Status
> code: 204 No Content.
>
> Start instance that starts guest and terminates the process eventually:
> 2019-01-05T20:21:44.119820357 [anonymous-instance:INFO:api_server/src/
> http_service.rs:599] The API server received a synchronous Put request on
> "/actions" with body "{n "action_type": "InstanceStart"n }".
> 2019-01-05T20:21:44.119837722 [anonymous-instance:INFO:vmm/src/lib.rs:1104]
> VMM received instance start command
> 2019-01-05T20:21:44.119903817 [anonymous-instance:INFO:vmm/src/
> vstate.rs:97] Guest memory starts at 7faddec0
>
> 2019-01-05T20:21:44.1*24*761979 [anonymous-instance:INFO:api_server/src/
> http_service.rs:565] The synchronous Put request on "/actions" with body
> "{n "action_type": "InstanceStart"n }" was executed successfully. Status
> code: 204 No Content.
> 2019-01-05T20:21:44.*141*417423 [anonymous-instance:INFO:vmm/src/
> lib.rs:1163] Vmm
Re: [RFC] Support firecracker
On Sun, Jan 6, 2019 at 5:46 AM Waldek Kozaczuk wrote: > One more thing - our serial console is dreadfully slow ( > https://github.com/cloudius-systems/osv/issues/921). So I think even > printing bootchart info slows things by 3-5 ms. Without bootchart the 17 ms > number drops to 12-11 ms. > This is firecrackers serial port driver: https://github.com/firecracker-microvm/firecracker/blob/master/devices/src/legacy/serial.rs I can't say I understand all of this of this code, but from cursory look I think that: 1. It doesn't use any sophisticated FIFO. There is just one byte of THR (transmit hold register). 2. It *does* support output interrupts: It checks bit 2 of the IER (interrupt enable register) - IER_THR, and triggers interrupt. Since I'm guessing they didn't implement this feature by accident, it is likely that Linux is using it, i.e., instead of busy-wait loop to wait for the empty bit (as in our putchar()) it goes to sleep, and allows the hypervisor to work on the output - assuming the exit (which we already had when we wrote the byte) wasn't enough. I don't know how it works on firecracker. If you can run Linux on firecracker too, you can compare the slowness of the serial console - output 1000 characters (or whatever) in the guess, and measure how slow it is. If Linux isn't faster than OSv on this, there's not much we can probably is. If Linux is much faster than OSv, then maybe these THR interrupts are indeed the reason. -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
One more thing - our serial console is dreadfully slow
(https://github.com/cloudius-systems/osv/issues/921). So I think even
printing bootchart info slows things by 3-5 ms. Without bootchart the 17 ms
number drops to 12-11 ms.
On Saturday, January 5, 2019 at 10:32:18 PM UTC-5, Waldek Kozaczuk wrote:
>
>
>
> On Saturday, January 5, 2019 at 4:09:23 AM UTC-5, דור לאור wrote:
>>
>> Great stuff Waldek! Is unix socket the normal way to pass parameters for
>> Firecracker?
>>
> I think so. There is also experimental vsock interface but I am not
> familiar with it and not sure what purpose of it would be (
> https://github.com/firecracker-microvm/firecracker/issues/650).
>
>>
>> How's the boot speed vs Qemu?
>>
> I have posted some numbers in my other email to the group but here are
> some more details.
>
> First of all my numbers come from running tests on my 5-years old MacBook
> pro that I have been using for all my OSv development in last three years.
> It is 4-core 2.3 GHz i7 machine. Not sure how it compares to newer models
> but given Moore's law does not work any more it might be still pretty fast.
>
> Architecture:x86_64
> CPU op-mode(s): 32-bit, 64-bit
> Byte Order: Little Endian
> CPU(s): 8
> On-line CPU(s) list: 0-7
> Thread(s) per core: 2
> Core(s) per socket: 4
> Socket(s): 1
> NUMA node(s):1
> Vendor ID: GenuineIntel
> CPU family: 6
> Model: 70
> Model name: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
> Stepping:1
> CPU MHz: 898.037
> CPU max MHz: 3500.
> CPU min MHz: 800.
> BogoMIPS:4589.68
> Virtualization: VT-x
> L1d cache: 32K
> L1i cache: 32K
> L2 cache:256K
> L3 cache:6144K
> L4 cache:131072K
>
> Additionally firecracker startup/configuration is more fine-grained than
> QEMU which does everything in one shot (start VMM process, configure
> resources, start guest, etc). With firecracker it is broken down like
> follows:
>
>1. Start VMM process which listen on socket for API calls (I have not
>measured it but seems very fast).
>2. Make API call to:
> - create instance that specified number of vCPUs, memory and kernel
> loader file path
> - configure block device
> - configure networking device
> - all these calls seem to take less than 1ms and can be executed in
> any order it seems
>3. Make API call to start instance that eventually starts guest
>
> So here are some log snippets (2,3) and OSv bootchart from running
> native-example rofs image (no networking):
>
> Add block device:
> 2019-01-05T20:21:44.*096*617431 [anonymous-instance:INFO:api_server/src/
> http_service.rs:599] The API server received a synchronous Put request on
> "/drives/rootfs" with body "{n "drive_id": "rootfs",n "path_on_host":
> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
> false,n "is_read_only": falsen }".
> 2019-01-05T20:21:44.*096*659174 [anonymous-instance:INFO:api_server/src/
> http_service.rs:565] The synchronous Put request on "/drives/rootfs" with
> body "{n "drive_id": "rootfs",n "path_on_host":
> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
> false,n "is_read_only": falsen }" was executed successfully. Status code:
> 204 No Content.
>
> Create instance:
> 2019-01-05T20:21:44.*1085*43339 [anonymous-instance:INFO:api_server/src/
> http_service.rs:599] The API server received a synchronous Put request on
> "/boot-source" with body "{n "kernel_image_path":
> "/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
> "boot_args": "--bootchart /hello"n }".
> 2019-01-05T20:21:44.*108*584295 [anonymous-instance:INFO:api_server/src/
> http_service.rs:565] The synchronous Put request on "/boot-source" with
> body "{n "kernel_image_path":
> "/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
> "boot_args": "--bootchart /hello"n }" was executed successfully. Status
> code: 204 No Content.
>
> Start instance that starts guest and terminates the process eventually:
> 2019-01-05T20:21:44.119820357 [anonymous-instance:INFO:api_server/src/
> http_service.rs:599] The API server received a synchronous Put request on
> "/actions" with body "{n "action_type": "InstanceStart"n }".
> 2019-01-05T20:21:44.119837722 [anonymous-instance:INFO:vmm/src/lib.rs:1104]
> VMM received instance start command
> 2019-01-05T20:21:44.119903817 [anonymous-instance:INFO:vmm/src/
> vstate.rs:97] Guest memory starts at 7faddec0
>
> 2019-01-05T20:21:44.1*24*761979 [anonymous-instance:INFO:api_server/src/
> http_service.rs:565] The synchronous Put request on "/actions" with body
> "{n "action_type": "InstanceStart"n }" was executed successfully. Status
> code: 204 No Content.
> 2019-01-05T20:21:44.*141*417423 [anonymous-instance:INFO:vmm/src/
> lib.rs:1163]
Re: [RFC] Support firecracker
On Saturday, January 5, 2019 at 2:58:14 AM UTC-5, Pekka Enberg wrote: > > > > On Sat, Jan 5, 2019 at 9:43 AM Waldek Kozaczuk > wrote: > >> Indeed I have come across some incompatibility issues. Mostly around >> configuration but nothing very fundamental. You can see in some of my >> comments. >> > > Right, I missed the comments because the diff is somewhat large. ;-) > > I would suggest to do Firecracker support in the following (largely > independent) steps: > > - Add support for modern virtio interfaces. Linux has a split driver base > model here, which we could also follow. I.e., have separate VirtioModern > and VirtioLegacy base classes that encapsulate the differences. > > - Add support for VirtIO MMIO. Most of this is restructuring the drivers > to use transport-independent interfaces and then have VirtioPCI and > VirtioMMIO classes that implement transport-specific logic. > > - Add Intel mptable (used by Firecracker) support as an alternative to > ACPI hardware discovery. > > - Add any misc Firecraker related things that are still needed to make OSv > boot and run. > > - Pekka > I like your suggestion. The extra thing would be adjusting OSv to support Linux boot protocol which would be also handy to run it on hyperkit or bhyve. -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
On Saturday, January 5, 2019 at 4:09:23 AM UTC-5, דור לאור wrote:
>
> Great stuff Waldek! Is unix socket the normal way to pass parameters for
> Firecracker?
>
I think so. There is also experimental vsock interface but I am not
familiar with it and not sure what purpose of it would be
(https://github.com/firecracker-microvm/firecracker/issues/650).
>
> How's the boot speed vs Qemu?
>
I have posted some numbers in my other email to the group but here are some
more details.
First of all my numbers come from running tests on my 5-years old MacBook
pro that I have been using for all my OSv development in last three years.
It is 4-core 2.3 GHz i7 machine. Not sure how it compares to newer models
but given Moore's law does not work any more it might be still pretty fast.
Architecture:x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
CPU(s): 8
On-line CPU(s) list: 0-7
Thread(s) per core: 2
Core(s) per socket: 4
Socket(s): 1
NUMA node(s):1
Vendor ID: GenuineIntel
CPU family: 6
Model: 70
Model name: Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
Stepping:1
CPU MHz: 898.037
CPU max MHz: 3500.
CPU min MHz: 800.
BogoMIPS:4589.68
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache:256K
L3 cache:6144K
L4 cache:131072K
Additionally firecracker startup/configuration is more fine-grained than
QEMU which does everything in one shot (start VMM process, configure
resources, start guest, etc). With firecracker it is broken down like
follows:
1. Start VMM process which listen on socket for API calls (I have not
measured it but seems very fast).
2. Make API call to:
- create instance that specified number of vCPUs, memory and kernel
loader file path
- configure block device
- configure networking device
- all these calls seem to take less than 1ms and can be executed in
any order it seems
3. Make API call to start instance that eventually starts guest
So here are some log snippets (2,3) and OSv bootchart from running
native-example rofs image (no networking):
Add block device:
2019-01-05T20:21:44.*096*617431
[anonymous-instance:INFO:api_server/src/http_service.rs:599] The API server
received a synchronous Put request on "/drives/rootfs" with body "{n
"drive_id": "rootfs",n "path_on_host":
"/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
false,n "is_read_only": falsen }".
2019-01-05T20:21:44.*096*659174
[anonymous-instance:INFO:api_server/src/http_service.rs:565] The
synchronous Put request on "/drives/rootfs" with body "{n "drive_id":
"rootfs",n "path_on_host":
"/home/wkozaczuk/projects/osv/build/release/usr.rofs",n "is_root_device":
false,n "is_read_only": falsen }" was executed successfully. Status code:
204 No Content.
Create instance:
2019-01-05T20:21:44.*1085*43339
[anonymous-instance:INFO:api_server/src/http_service.rs:599] The API server
received a synchronous Put request on "/boot-source" with body "{n
"kernel_image_path":
"/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
"boot_args": "--bootchart /hello"n }".
2019-01-05T20:21:44.*108*584295
[anonymous-instance:INFO:api_server/src/http_service.rs:565] The
synchronous Put request on "/boot-source" with body "{n
"kernel_image_path":
"/home/wkozaczuk/projects/waldek-osv/build/release/loader-stripped.elf",n
"boot_args": "--bootchart /hello"n }" was executed successfully. Status
code: 204 No Content.
Start instance that starts guest and terminates the process eventually:
2019-01-05T20:21:44.119820357
[anonymous-instance:INFO:api_server/src/http_service.rs:599] The API server
received a synchronous Put request on "/actions" with body "{n
"action_type": "InstanceStart"n }".
2019-01-05T20:21:44.119837722 [anonymous-instance:INFO:vmm/src/lib.rs:1104]
VMM received instance start command
2019-01-05T20:21:44.119903817
[anonymous-instance:INFO:vmm/src/vstate.rs:97] Guest memory starts at
7faddec0
2019-01-05T20:21:44.1*24*761979
[anonymous-instance:INFO:api_server/src/http_service.rs:565] The
synchronous Put request on "/actions" with body "{n "action_type":
"InstanceStart"n }" was executed successfully. Status code: 204 No Content.
2019-01-05T20:21:44.*141*417423
[anonymous-instance:INFO:vmm/src/lib.rs:1163] Vmm is stopping.
As you can see above the first part (from "received a synchronous Put" to "The
synchronous Put request on "/actions"") takes 5 ms and seems to be an
overhead of preparing VMM state for the guest based on information passes
in to the calls create instance and setup block device.
The delta between last line timestamp and previous one is around 17 ms and
includes 10m of OSv boot time below.
And OSv bootchart:
OSv v0.52.0-24-g3628f560
Cmdline: --bootchart /hello
Re: [RFC] Support firecracker
Great stuff Waldek! Is unix socket the normal way to pass parameters for Firecracker? How's the boot speed vs Qemu? On Sat, Jan 5, 2019 at 12:03 AM Pekka Enberg wrote: > > > On Sat, Jan 5, 2019 at 9:58 AM Pekka Enberg wrote: > >> >> >> On Sat, Jan 5, 2019 at 9:43 AM Waldek Kozaczuk >> wrote: >> >>> Indeed I have come across some incompatibility issues. Mostly around >>> configuration but nothing very fundamental. You can see in some of my >>> comments. >>> >> >> Right, I missed the comments because the diff is somewhat large. ;-) >> >> I would suggest to do Firecracker support in the following (largely >> independent) steps: >> >> - Add support for modern virtio interfaces. Linux has a split driver base >> model here, which we could also follow. I.e., have separate VirtioModern >> and VirtioLegacy base classes that encapsulate the differences. >> > > Btw, this can be also tested with QEMU by passing the "disable-modern=off" > and "disable-legacy=on" options to the "-device" option that defines the > virtio devices. > > - Pekka > >> -- > You received this message because you are subscribed to the Google Groups > "OSv Development" group. > To unsubscribe from this group and stop receiving emails from it, send an > email to osv-dev+unsubscr...@googlegroups.com. > For more options, visit https://groups.google.com/d/optout. > -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
On Sat, Jan 5, 2019 at 9:58 AM Pekka Enberg wrote: > > > On Sat, Jan 5, 2019 at 9:43 AM Waldek Kozaczuk > wrote: > >> Indeed I have come across some incompatibility issues. Mostly around >> configuration but nothing very fundamental. You can see in some of my >> comments. >> > > Right, I missed the comments because the diff is somewhat large. ;-) > > I would suggest to do Firecracker support in the following (largely > independent) steps: > > - Add support for modern virtio interfaces. Linux has a split driver base > model here, which we could also follow. I.e., have separate VirtioModern > and VirtioLegacy base classes that encapsulate the differences. > Btw, this can be also tested with QEMU by passing the "disable-modern=off" and "disable-legacy=on" options to the "-device" option that defines the virtio devices. - Pekka > -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
On Sat, Jan 5, 2019 at 9:43 AM Waldek Kozaczuk wrote: > Indeed I have come across some incompatibility issues. Mostly around > configuration but nothing very fundamental. You can see in some of my > comments. > Right, I missed the comments because the diff is somewhat large. ;-) I would suggest to do Firecracker support in the following (largely independent) steps: - Add support for modern virtio interfaces. Linux has a split driver base model here, which we could also follow. I.e., have separate VirtioModern and VirtioLegacy base classes that encapsulate the differences. - Add support for VirtIO MMIO. Most of this is restructuring the drivers to use transport-independent interfaces and then have VirtioPCI and VirtioMMIO classes that implement transport-specific logic. - Add Intel mptable (used by Firecracker) support as an alternative to ACPI hardware discovery. - Add any misc Firecraker related things that are still needed to make OSv boot and run. - Pekka -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
Indeed I have come across some incompatibility issues. Mostly around configuration but nothing very fundamental. You can see in some of my comments. -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
Hi Waldek, On Sat, Jan 5, 2019 at 8:47 AM Waldek Kozaczuk wrote: > Please bear in mind it is a RFC polluted with hacks and many TODO comments > that raise many questions. > > Feel free to review it especially if you have ideas on how to better > restructure the code (I left some comments about it) and if you see any > obvious bugs. > Looks like the bulk of the changes are related to supporting VirtIO devices via MMIO because Firecracker does not support PCI? Btw, I think that Firecracker implements the "modern" VirtioIO interface (i.e., version 1.0 and later) but I think OSv's VirtIO drivers are still using the "legacy" interfaces. Did you hit any compatibility issues because of this? - Pekka -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
Re: [RFC] Support firecracker
Please bear in mind it is a RFC polluted with hacks and many TODO comments
that raise many questions.
Feel free to review it especially if you have ideas on how to better
restructure the code (I left some comments about it) and if you see any
obvious bugs.
Regards,
Waldek
On Saturday, January 5, 2019 at 1:44:47 AM UTC-5, Waldek Kozaczuk wrote:
>
> This patch provides all necessary changes to OSv
> to make it boot on AWS firecracker.
>
> The curl command exmaple to create and start instance with
> single block device:
>
> ./scripts/setup-block.sh && ./scripts/create-instance.sh &&
> ./scripts/start-instance.sh
>
> where:
> curl --unix-socket /tmp/firecracker.socket -i \
> -X PUT 'http://localhost/drives/rootfs' \
> -H 'Accept: application/json' \
> -H 'Content-Type: application/json' \
> -d '{
> "drive_id": "rootfs",
> "path_on_host":
> "/home/wkozaczuk/projects/osv/build/release/usr.rofs",
> "is_root_device": false,
> "is_read_only": false
> }'
>
> curl --unix-socket /tmp/firecracker.socket -i \
> -X PUT 'http://localhost/boot-source' \
> -H 'Accept: application/json' \
> -H 'Content-Type: application/json' \
> -d '{
> "kernel_image_path":
> "/home/wkozaczuk/projects/osv/build/release/loader-stripped.elf",
> "boot_args": "--bootchart /hello"
> }'
>
> curl --unix-socket /tmp/firecracker.socket -i \
> -X PUT 'http://localhost/actions' \
> -H 'Accept: application/json' \
> -H 'Content-Type: application/json'\
> -d '{
> "action_type": "InstanceStart"
> }'
>
> Signed-off-by: Waldemar Kozaczuk
> ---
> Makefile | 2 +
> arch/x64/arch-setup.cc | 180 +++---
> arch/x64/boot.S | 37 -
> arch/x64/loader.ld | 2 +-
> arch/x64/power.cc| 14 +-
> arch/x64/smp.cc | 16 +-
> drivers/acpi.cc | 10 +-
> drivers/virtio-assign.cc | 3 +-
> drivers/virtio-blk.cc| 78 ++
> drivers/virtio-blk.hh| 10 +-
> drivers/virtio-mmio.cc | 128
> drivers/virtio-mmio.hh | 132
> drivers/virtio-net.cc| 86 +--
> drivers/virtio-net.hh| 10 +-
> drivers/virtio-vring.cc | 25 ++-
> drivers/virtio-vring.hh | 10 +-
> drivers/virtio.hh| 18 ++-
> drivers/virtio2.cc | 289 +++
> drivers/virtio2.hh | 115 ++
> include/osv/virtio-assign.hh | 2 +-
> loader.cc| 3 +-
> 21 files changed, 1039 insertions(+), 131 deletions(-)
> create mode 100644 drivers/virtio-mmio.cc
> create mode 100644 drivers/virtio-mmio.hh
> create mode 100644 drivers/virtio2.cc
> create mode 100644 drivers/virtio2.hh
>
> diff --git a/Makefile b/Makefile
> index 68043485..cf571632 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -819,9 +819,11 @@ drivers += $(libtsm)
> drivers += drivers/vga.o drivers/kbd.o drivers/isa-serial.o
> drivers += arch/$(arch)/pvclock-abi.o
> drivers += drivers/virtio.o
> +drivers += drivers/virtio2.o
> drivers += drivers/virtio-vring.o
> drivers += drivers/virtio-net.o
> drivers += drivers/virtio-assign.o
> +drivers += drivers/virtio-mmio.o
> drivers += drivers/vmxnet3.o
> drivers += drivers/vmxnet3-queues.o
> drivers += drivers/virtio-blk.o
> diff --git a/arch/x64/arch-setup.cc b/arch/x64/arch-setup.cc
> index 6e4833cf..58da00c5 100644
> --- a/arch/x64/arch-setup.cc
> +++ b/arch/x64/arch-setup.cc
> @@ -22,6 +22,19 @@
> #include
> #include "dmi.hh"
>
> +// Not sure if Linux zero page is always located at this place
> +// in memory or its address is passed in one of the registers
> +// -> double check
> +#define ZERO_PAGE_START 0x7000
> +#define SETUP_HEADER_OFFSET 0x1f1 // look at bootparam.h in linux
> +#define BOOT_FLAG_OFFSET sizeof(u8) + 4 * sizeof(u16) + sizeof(u32)
> +
> +#define E820_ENTRIES_OFFSET 0x1e8 // look at bootparam.h in linux
> +#define E820_TABLE_OFFSET0x2d0 // look at bootparam.h in linux
> +
> +#define CMD_LINE_PTR_OFFSET sizeof(u8) * 5 + sizeof(u16) * 11 +
> sizeof(u32) * 7
> +#define CMD_LINE_SIZE_OFFSET CMD_LINE_PTR_OFFSET + sizeof(u8) * 2 +
> sizeof(u16) + sizeof(u32) * 3
> +
> struct multiboot_info_type {
> u32 flags;
> u32 mem_lower;
> @@ -61,12 +74,81 @@ struct e820ent {
> u32 type;
> } __attribute__((packed));
>
> +struct _e820ent {
> +u64 addr;
> +u64 size;
> +u32 type;
> +} __attribute__((packed));
> +
> osv_multiboot_info_type* osv_multiboot_info;
>
> -void parse_cmdline(multiboot_info_type& mb)
> +struct mmio_device_info {
> +u64 address;
> +u64 size;
> +unsigned int irq;
> +};
> +
> +//TODO: For now we are limiting
[RFC] Support firecracker
This patch provides all necessary changes to OSv
to make it boot on AWS firecracker.
The curl command exmaple to create and start instance with
single block device:
./scripts/setup-block.sh && ./scripts/create-instance.sh &&
./scripts/start-instance.sh
where:
curl --unix-socket /tmp/firecracker.socket -i \
-X PUT 'http://localhost/drives/rootfs' \
-H 'Accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"drive_id": "rootfs",
"path_on_host": "/home/wkozaczuk/projects/osv/build/release/usr.rofs",
"is_root_device": false,
"is_read_only": false
}'
curl --unix-socket /tmp/firecracker.socket -i \
-X PUT 'http://localhost/boot-source' \
-H 'Accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"kernel_image_path":
"/home/wkozaczuk/projects/osv/build/release/loader-stripped.elf",
"boot_args": "--bootchart /hello"
}'
curl --unix-socket /tmp/firecracker.socket -i \
-X PUT 'http://localhost/actions' \
-H 'Accept: application/json' \
-H 'Content-Type: application/json'\
-d '{
"action_type": "InstanceStart"
}'
Signed-off-by: Waldemar Kozaczuk
---
Makefile | 2 +
arch/x64/arch-setup.cc | 180 +++---
arch/x64/boot.S | 37 -
arch/x64/loader.ld | 2 +-
arch/x64/power.cc| 14 +-
arch/x64/smp.cc | 16 +-
drivers/acpi.cc | 10 +-
drivers/virtio-assign.cc | 3 +-
drivers/virtio-blk.cc| 78 ++
drivers/virtio-blk.hh| 10 +-
drivers/virtio-mmio.cc | 128
drivers/virtio-mmio.hh | 132
drivers/virtio-net.cc| 86 +--
drivers/virtio-net.hh| 10 +-
drivers/virtio-vring.cc | 25 ++-
drivers/virtio-vring.hh | 10 +-
drivers/virtio.hh| 18 ++-
drivers/virtio2.cc | 289 +++
drivers/virtio2.hh | 115 ++
include/osv/virtio-assign.hh | 2 +-
loader.cc| 3 +-
21 files changed, 1039 insertions(+), 131 deletions(-)
create mode 100644 drivers/virtio-mmio.cc
create mode 100644 drivers/virtio-mmio.hh
create mode 100644 drivers/virtio2.cc
create mode 100644 drivers/virtio2.hh
diff --git a/Makefile b/Makefile
index 68043485..cf571632 100644
--- a/Makefile
+++ b/Makefile
@@ -819,9 +819,11 @@ drivers += $(libtsm)
drivers += drivers/vga.o drivers/kbd.o drivers/isa-serial.o
drivers += arch/$(arch)/pvclock-abi.o
drivers += drivers/virtio.o
+drivers += drivers/virtio2.o
drivers += drivers/virtio-vring.o
drivers += drivers/virtio-net.o
drivers += drivers/virtio-assign.o
+drivers += drivers/virtio-mmio.o
drivers += drivers/vmxnet3.o
drivers += drivers/vmxnet3-queues.o
drivers += drivers/virtio-blk.o
diff --git a/arch/x64/arch-setup.cc b/arch/x64/arch-setup.cc
index 6e4833cf..58da00c5 100644
--- a/arch/x64/arch-setup.cc
+++ b/arch/x64/arch-setup.cc
@@ -22,6 +22,19 @@
#include
#include "dmi.hh"
+// Not sure if Linux zero page is always located at this place
+// in memory or its address is passed in one of the registers
+// -> double check
+#define ZERO_PAGE_START 0x7000
+#define SETUP_HEADER_OFFSET 0x1f1 // look at bootparam.h in linux
+#define BOOT_FLAG_OFFSET sizeof(u8) + 4 * sizeof(u16) + sizeof(u32)
+
+#define E820_ENTRIES_OFFSET 0x1e8 // look at bootparam.h in linux
+#define E820_TABLE_OFFSET0x2d0 // look at bootparam.h in linux
+
+#define CMD_LINE_PTR_OFFSET sizeof(u8) * 5 + sizeof(u16) * 11 + sizeof(u32) *
7
+#define CMD_LINE_SIZE_OFFSET CMD_LINE_PTR_OFFSET + sizeof(u8) * 2 +
sizeof(u16) + sizeof(u32) * 3
+
struct multiboot_info_type {
u32 flags;
u32 mem_lower;
@@ -61,12 +74,81 @@ struct e820ent {
u32 type;
} __attribute__((packed));
+struct _e820ent {
+u64 addr;
+u64 size;
+u32 type;
+} __attribute__((packed));
+
osv_multiboot_info_type* osv_multiboot_info;
-void parse_cmdline(multiboot_info_type& mb)
+struct mmio_device_info {
+u64 address;
+u64 size;
+unsigned int irq;
+};
+
+//TODO: For now we are limiting number of mmio devices to two
+// Ideally we should be using somewhat more dynamic structure
+struct mmio_device_info mmio_device_info_entries[2];
+int mmio_device_info_count = 0;
+
+#define VIRTIO_MMIO_DEVICE_CMDLINE_PREFIX "virtio_mmio.device="
+char* parse_mmio_device_info(char *cmdline, mmio_device_info *info) {
+// [virtio_mmio.]device=@:[:]
+char *prefix_pos = strstr(cmdline,VIRTIO_MMIO_DEVICE_CMDLINE_PREFIX);
+if (!prefix_pos)
+return nullptr;
+
+char *size_pos = prefix_pos + strlen(VIRTIO_MMIO_DEVICE_CMDLINE_PREFIX);
+if (sscanf(size_pos,"%ld", >size) != 1)
+return nullptr;
+
+char *at_pos = strstr(size_pos,"@");
+if (!at_pos)
+return nullptr;
+
+
