diff --git a/arch/x64/exceptions.cc b/arch/x64/exceptions.cc index 792691291..8b8418406 100644 --- a/arch/x64/exceptions.cc +++ b/arch/x64/exceptions.cc @@ -118,7 +118,12 @@ unsigned interrupt_descriptor_table::register_interrupt_handler( } } } - abort(); + // No free IDT vector. Vectors 0..31 are CPU exceptions and are never + // returned here, so 0 is an unambiguous "exhausted" sentinel. Returning it + // rather than aborting lets MSI-X allocation (msix_vector) fail gracefully + // and lets a device fall back to fewer queues instead of crashing the boot + // when many multiqueue devices exhaust the 224 usable vectors. + return 0; } void interrupt_descriptor_table::unregister_handler(unsigned vector) diff --git a/drivers/msi.cc b/drivers/msi.cc index 87366b957..ee9477aef 100644 --- a/drivers/msi.cc +++ b/drivers/msi.cc @@ -25,7 +25,11 @@ msix_vector::msix_vector(pci::function* dev) msix_vector::~msix_vector() { - idt.unregister_handler(_vector); + // _vector == 0 means IDT vector allocation failed (see the ctor); there is + // nothing to unregister in that case. + if (_vector) { + idt.unregister_handler(_vector); + } } pci::function* msix_vector::get_pci_function(void) @@ -198,7 +202,16 @@ std::vector interrupt_manager::request_vectors(unsigned num_vector auto num = std::min(num_vectors, num_entries); for (unsigned i = 0; i < num; ++i) { - results.push_back(new msix_vector(_dev)); + auto* vec = new msix_vector(_dev); + // A zero vector means the global IDT vector pool is exhausted. Stop + // here and return however many we did get; easy_register() treats a + // short result as failure so the caller can fall back to fewer queues + // rather than crash. + if (!vec->get_vector()) { + delete vec; + break; + } + results.push_back(vec); } return (results); diff --git a/drivers/virtio-blk.cc b/drivers/virtio-blk.cc index af3324d9a..0df1b40f8 100644 --- a/drivers/virtio-blk.cc +++ b/drivers/virtio-blk.cc @@ -137,6 +137,20 @@ blk::blk(virtio_device& virtio_dev) // base class did not set up. _num_queues = virtio_driver::_num_queues; + // Cap the number of queues we arm with an interrupt against the shared + // MSI-X vector budget. Each queue uses one vector, and the x86-64 IDT has + // only 224 usable vectors shared by every device; a high-vCPU guest with + // several multiqueue virtio devices can otherwise exhaust them at boot. + // Extra probed virtqueues beyond this cap are simply left unused - I/O is + // directed only to queues 0.._num_queues-1, all of which have a serviced + // interrupt. + unsigned granted = reserve_msix_vectors(_num_queues); + if (granted < (unsigned)_num_queues) { + virtio_i("virtio-blk: capping active queues from %d to %d (MSI-X budget)\n", + _num_queues, granted); + _num_queues = granted; + } + // Resize per-queue lock vector now that _num_queues is known. _queue_locks = std::vector(_num_queues); diff --git a/drivers/virtio-net.cc b/drivers/virtio-net.cc index 5f815ef64..175611170 100644 --- a/drivers/virtio-net.cc +++ b/drivers/virtio-net.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -136,10 +137,13 @@ static int if_transmit(struct ifnet* ifp, struct mbuf* m_head) inline int net::xmit(struct mbuf* buff) { // - // We currently have only a single TX queue. Select a proper TXq here when - // we implement a multi-queue. + // Select a Tx queue by CPU id so that transmits from different CPUs use + // independent rings and do not serialize on one queue. With a single + // queue pair (_nqp == 1) this always selects queue 0, i.e. the original + // behaviour. // - return _txq.xmit(buff); + unsigned qidx = sched::cpu::current()->id % _nqp; + return _txqs[qidx]->xmit(buff); } inline int net::txq::xmit(mbuf* buff) @@ -186,9 +190,13 @@ static void if_getinfo(struct ifnet* ifp, struct if_data* out_data) void net::fill_stats(struct if_data* out_data) const { - // We currently support only a single Tx/Rx queue so no iteration so far - fill_qstats(_rxq, out_data); - fill_qstats(_txq, out_data); + // Aggregate the per-queue statistics across all Rx/Tx queue pairs. + for (auto& rxq : _rxqs) { + fill_qstats(*rxq, out_data); + } + for (auto& txq : _txqs) { + fill_qstats(*txq, out_data); + } } void net::fill_qstats(const struct rxq& rxq, struct if_data* out_data) const @@ -203,15 +211,16 @@ void net::fill_qstats(const struct rxq& rxq, struct if_data* out_data) const void net::fill_qstats(const struct txq& txq, struct if_data* out_data) const { - assert(!out_data->ifi_oerrors && !out_data->ifi_obytes && !out_data->ifi_opackets); + // Called once per Tx queue and accumulated, so every field must add rather + // than overwrite (with multiqueue there is more than one Tx queue). out_data->ifi_opackets += txq.stats.tx_packets; out_data->ifi_obytes += txq.stats.tx_bytes; out_data->ifi_oerrors += txq.stats.tx_err + txq.stats.tx_drops; - out_data->ifi_oworker_kicks = txq.stats.tx_worker_kicks; - out_data->ifi_oworker_wakeups = txq.stats.tx_worker_wakeups; - out_data->ifi_oworker_packets = txq.stats.tx_worker_packets; - out_data->ifi_okicks = txq.stats.tx_kicks; - out_data->ifi_oqueue_is_full = txq.stats.tx_hw_queue_is_full; + out_data->ifi_oworker_kicks += txq.stats.tx_worker_kicks; + out_data->ifi_oworker_wakeups += txq.stats.tx_worker_wakeups; + out_data->ifi_oworker_packets += txq.stats.tx_worker_packets; + out_data->ifi_okicks += txq.stats.tx_kicks; + out_data->ifi_oqueue_is_full += txq.stats.tx_hw_queue_is_full; out_data->ifi_owakeup_stats = txq.stats.tx_wakeup_stats; } @@ -220,7 +229,9 @@ bool net::ack_irq() auto isr = _dev.read_and_ack_isr(); if (isr) { - _rxq.vqueue->disable_interrupts(); + // Only used on the shared single-interrupt (MMIO/legacy) path, where + // there is a single queue pair; disable the Rx queue's interrupts. + _rxqs[0]->vqueue->disable_interrupts(); return true; } else { return false; @@ -239,18 +250,12 @@ void net::init() net::net(virtio_device& dev) : virtio_driver(dev), - _pre_init(this), - _rxq(get_virt_queue(0), [this] { this->receiver(); }), - _txq(this, get_virt_queue(1)) + _pre_init(this) { _driver_name = "virtio-net"; virtio_i("VIRTIO NET INSTANCE"); _id = _instance++; - sched::thread* poll_task = _rxq.poll_task.get(); - - poll_task->set_priority(sched::thread::priority_infinity); - // Please look at the section 5.1.6.1 of virtio specification for explanation if (_dev.is_modern()) { _hdr_size = sizeof(net_hdr_mrg_rxbuf); @@ -259,6 +264,62 @@ net::net(virtio_device& dev) _hdr_size = _mergeable_bufs ? sizeof(net_hdr_mrg_rxbuf) : sizeof(net_hdr); } + // + // Decide the number of Rx/Tx queue pairs to use. With VIRTIO_NET_F_MQ the + // device advertises max_virtqueue_pairs and lays out the virtqueues as + // rx[i]=2*i, tx[i]=2*i+1, ctrl=2*nqp. probe_virt_queues() (run from init() + // via _pre_init) has already created every virtqueue the device exposes, so + // _num_queues tells us how many actually exist. We use one queue pair per + // vCPU, bounded by what the device advertises, what was probed, and the + // shared MSI-X vector budget (2 vectors per pair). When F_MQ is absent this + // collapses to a single pair - identical to the original driver. + unsigned nqp = 1; + // Activating more than one queue pair requires the control virtqueue to + // send CTRL_MQ_VQ_PAIRS_SET, so multiqueue is gated on both features. + if (get_guest_feature_bit(VIRTIO_NET_F_MQ) && + get_guest_feature_bit(VIRTIO_NET_F_CTRL_VQ) && + _config.max_virtqueue_pairs > 1) { + nqp = std::min(_config.max_virtqueue_pairs, sched::cpus.size()); + // Number of pairs the probed virtqueues can support: N pairs need + // 2*N data queues plus one control queue. + unsigned probed_pairs = _num_queues > 1 ? (_num_queues - 1) / 2 : 1; + nqp = std::min(nqp, probed_pairs); + if (nqp < 1) { + nqp = 1; + } + // Cap against the shared MSI-X vector budget (2 vectors per pair). + unsigned vec_pairs = reserve_msix_vectors(2 * nqp) / 2; + if (vec_pairs < 1) { + vec_pairs = 1; + } + nqp = std::min(nqp, vec_pairs); + } + _nqp = nqp; + // Per the virtio spec the control virtqueue always sits at index + // 2*max_virtqueue_pairs (based on the device's advertised maximum), which + // is not the same as 2*_nqp when we activate fewer pairs than advertised. + if (get_guest_feature_bit(VIRTIO_NET_F_CTRL_VQ)) { + unsigned max_pairs = _config.max_virtqueue_pairs > 1 ? + _config.max_virtqueue_pairs : 1; + int ctrl_idx = 2 * max_pairs; + if ((unsigned)ctrl_idx < _num_queues) { + _ctrl_vq_idx = ctrl_idx; + } + } + + // Build the Rx/Tx queue pairs. Each Rx queue gets its own poll thread; each + // Tx queue its own xmitter. + for (unsigned i = 0; i < _nqp; i++) { + auto* rx_vq = get_virt_queue(2 * i); + auto* tx_vq = get_virt_queue(2 * i + 1); + assert(rx_vq && tx_vq); + _rxqs.push_back(std::unique_ptr( + new rxq(rx_vq, [this, i] { this->receiver(this->_rxqs[i].get()); }))); + _txqs.push_back(std::unique_ptr>( + aligned_new(this, tx_vq))); + _rxqs[i]->poll_task->set_priority(sched::thread::priority_infinity); + } + //initialize the BSD interface _if _ifn = if_alloc(IFT_ETHER); if (_ifn == NULL) { @@ -277,7 +338,7 @@ net::net(virtio_device& dev) _ifn->if_qflush = if_qflush; _ifn->if_init = if_init; _ifn->if_getinfo = if_getinfo; - IFQ_SET_MAXLEN(&_ifn->if_snd, _txq.vqueue->size()); + IFQ_SET_MAXLEN(&_ifn->if_snd, _txqs[0]->vqueue->size()); _ifn->if_capabilities = 0; @@ -297,30 +358,63 @@ net::net(virtio_device& dev) _ifn->if_capenable = _ifn->if_capabilities | IFCAP_HWSTATS; - //Start the polling thread before attaching it to the Rx interrupt - poll_task->start(); - _txq.start(); + //Start the polling threads before attaching them to the Rx interrupts + for (unsigned i = 0; i < _nqp; i++) { + _rxqs[i]->poll_task->start(); + _txqs[i]->start(); + } ether_ifattach(_ifn, _config.mac); + // Pin each Rx poll thread to a distinct CPU (round-robin) so that with + // multiqueue the per-queue receivers run in parallel across CPUs instead of + // contending for one. With a single queue pair this pins to CPU 0, which + // matches the priority_infinity single-thread behaviour of the original + // driver closely enough and keeps its Rx work on one CPU. + for (unsigned i = 0; i < _nqp; i++) { + sched::thread::pin(_rxqs[i]->poll_task.get(), sched::cpus[i % sched::cpus.size()]); + } + interrupt_factory int_factory; #if CONF_drivers_pci - int_factory.register_msi_bindings = [this,poll_task](interrupt_manager &msi) { - msi.easy_register({ - { 0, [&] { this->_rxq.vqueue->disable_interrupts(); }, poll_task }, - { 1, [&] { this->_txq.vqueue->disable_interrupts(); }, nullptr } - }); + // One MSI-X vector per data virtqueue. setup_queue() maps virtqueue index + // i -> MSI-X entry i (1:1), so the entry numbers are the virtqueue indices + // rx[i]=2*i and tx[i]=2*i+1. Each Rx vector wakes that queue's poll thread; + // Tx vectors just disable the queue's interrupts (Tx completion is reaped by + // the xmitter/gc path, as in the original single-queue driver). + int_factory.register_msi_bindings = [this](interrupt_manager &msi) { + std::vector bindings; + bindings.reserve(2 * _nqp); + for (unsigned i = 0; i < _nqp; i++) { + auto* rx = _rxqs[i].get(); + auto* tx = _txqs[i].get(); + sched::thread* poll = rx->poll_task.get(); + bindings.push_back({ (unsigned)(2 * i), + [rx] { rx->vqueue->disable_interrupts(); }, poll }); + bindings.push_back({ (unsigned)(2 * i + 1), + [tx] { tx->vqueue->disable_interrupts(); }, nullptr }); + } + if (!msi.easy_register(bindings)) { + // Vector allocation is bounded by reserve_msix_vectors() above, so + // this should not happen; if it does, fail loudly rather than run + // with queues whose completions are never serviced. + net_e("virtio-net: failed to register %d MSI-X vectors", 2 * _nqp); + abort(); + } }; - int_factory.create_pci_interrupt = [this,poll_task](pci::device &pci_dev) { + int_factory.create_pci_interrupt = [this](pci::device &pci_dev) { return new pci_interrupt( pci_dev, [=] { return this->ack_irq(); }, - [=] { poll_task->wake_with_irq_disabled(); }); + [=] { this->_rxqs[0]->poll_task->wake_with_irq_disabled(); }); }; #endif #if CONF_drivers_mmio + // The MMIO transport has a single shared interrupt line, so multiqueue Rx + // steering is not available there; _nqp is 1 on that path. + sched::thread* poll_task = _rxqs[0]->poll_task.get(); #ifdef __aarch64__ int_factory.create_spi_edge_interrupt = [this,poll_task]() { return new spi_interrupt( @@ -340,7 +434,37 @@ net::net(virtio_device& dev) _dev.register_interrupt(int_factory); - fill_rx_ring(); + // Tell the device how many queue pairs we will actually use. Must happen + // after the queues and their interrupts are set up and before we start + // feeding the Rx rings. + // + // With VIRTIO_NET_F_RSS the device steers inbound flows across the Rx + // queues using the RSS indirection table we program here; the RSS config + // command carries the queue count (via the table and max_tx_vq), so it + // replaces the plain VQ_PAIRS_SET. Without RSS, VQ_PAIRS_SET only enables + // the queues and the host has no hash to distribute flows, so inbound + // traffic piles onto Rx queue 0 (only one receiver thread ever runs). + if (_nqp > 1 && _ctrl_vq_idx >= 0) { + if (get_guest_feature_bit(VIRTIO_NET_F_RSS)) { + if (set_rss_config()) { + _rss = true; + net_i("virtio-net: RSS enabled across %d Rx queues", _nqp); + } else { + net_w("virtio-net: RSS_CONFIG not acked; falling back to " + "VQ_PAIRS_SET (inbound flows will not be steered)"); + if (!set_vq_pairs(_nqp)) { + net_w("virtio-net: CTRL_MQ_VQ_PAIRS_SET(%d) not acked by " + "device", _nqp); + } + } + } else if (!set_vq_pairs(_nqp)) { + net_w("virtio-net: CTRL_MQ_VQ_PAIRS_SET(%d) not acked by device", _nqp); + } + } + + for (unsigned i = 0; i < _nqp; i++) { + fill_rx_ring(_rxqs[i].get()); + } // Step 8 add_dev_status(VIRTIO_CONFIG_S_DRIVER_OK); @@ -384,6 +508,39 @@ void net::read_config() _host_tso4 = get_guest_feature_bit(VIRTIO_NET_F_HOST_TSO4); _guest_ufo = get_guest_feature_bit(VIRTIO_NET_F_GUEST_UFO); + // With VIRTIO_NET_F_MQ the device advertises how many Rx/Tx queue pairs it + // supports; read it so the constructor can decide how many to use. Default + // to a single pair otherwise. + _config.max_virtqueue_pairs = 1; + if (get_guest_feature_bit(VIRTIO_NET_F_MQ)) { + virtio_conf_read(offsetof(net_config, max_virtqueue_pairs), + &_config.max_virtqueue_pairs, + sizeof(_config.max_virtqueue_pairs)); + net_i("Multiqueue: device advertises %d queue pairs", + _config.max_virtqueue_pairs); + } + + // With VIRTIO_NET_F_RSS the device exposes the RSS limits and the hash + // types it can compute; read them so init() can program a valid config. + _config.rss_max_key_size = 0; + _config.rss_max_indirection_table_length = 0; + _config.supported_hash_types = 0; + if (get_guest_feature_bit(VIRTIO_NET_F_RSS)) { + virtio_conf_read(offsetof(net_config, rss_max_key_size), + &_config.rss_max_key_size, + sizeof(_config.rss_max_key_size)); + virtio_conf_read(offsetof(net_config, rss_max_indirection_table_length), + &_config.rss_max_indirection_table_length, + sizeof(_config.rss_max_indirection_table_length)); + virtio_conf_read(offsetof(net_config, supported_hash_types), + &_config.supported_hash_types, + sizeof(_config.supported_hash_types)); + net_i("RSS: device max_key=%d, max_indir_table=%d, hash_types=0x%x", + _config.rss_max_key_size, + _config.rss_max_indirection_table_length, + _config.supported_hash_types); + } + net_i("Features: %s=%d,%s=%d", "Status", _status, "TSO_ECN", _tso_ecn); net_i("Features: %s=%d,%s=%d", "Host TSO ECN", _host_tso_ecn, "CSUM", _csum); net_i("Features: %s=%d,%s=%d", "Guest_csum", _guest_csum, "guest tso4", _guest_tso4); @@ -459,9 +616,9 @@ bool net::bad_rx_csum(struct mbuf* m, struct net_hdr* hdr) return false; } -void net::receiver() +void net::receiver(struct rxq* rxq) { - vring* vq = _rxq.vqueue; + vring* vq = rxq->vqueue; std::vector packet; u64 rx_drops = 0, rx_packets = 0, csum_ok = 0; u64 csum_err = 0, rx_bytes = 0; @@ -473,8 +630,8 @@ void net::receiver() virtio_driver::wait_for_queue(vq, &vring::used_ring_not_empty); trace_virtio_net_rx_wake(); - _rxq.stats.rx_bh_wakeups++; - _rxq.update_wakeup_stats(rx_packets); + rxq->stats.rx_bh_wakeups++; + rxq->update_wakeup_stats(rx_packets); u32 len; int nbufs; @@ -490,7 +647,7 @@ void net::receiver() vq->get_buf_finalize(); if (vq->effective_avail_ring_count() >= refill_thresh) - fill_rx_ring(); + fill_rx_ring(rxq); // Bad packet/buffer - discard and continue to the next one if (len < _hdr_size + ETHER_HDR_LEN) { @@ -553,11 +710,11 @@ void net::receiver() } // Update the stats - _rxq.stats.rx_drops += rx_drops; - _rxq.stats.rx_packets += rx_packets; - _rxq.stats.rx_csum += csum_ok; - _rxq.stats.rx_csum_err += csum_err; - _rxq.stats.rx_bytes += rx_bytes; + rxq->stats.rx_drops += rx_drops; + rxq->stats.rx_packets += rx_packets; + rxq->stats.rx_csum += csum_ok; + rxq->stats.rx_csum_err += csum_err; + rxq->stats.rx_bytes += rx_bytes; } } @@ -637,11 +794,11 @@ unsigned* net::rx_buffer_refcnt(void* frame) return reinterpret_cast(base + buf_bytes - sizeof(unsigned)); } -void net::fill_rx_ring() +void net::fill_rx_ring(struct rxq* rxq) { trace_virtio_net_fill_rx_ring(_ifn->if_index); int added = 0; - vring* vq = _rxq.vqueue; + vring* vq = rxq->vqueue; int size_in_pages = _use_large_buffers ? LARGE_BUFFER_SIZE_IN_PAGES : 1; // Reserve sizeof(unsigned) at the tail of every buffer for that buffer's @@ -929,6 +1086,168 @@ void net::txq::gc() vqueue->get_buf_gc(); } +// Send VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET on the control virtqueue to tell the +// device how many Rx/Tx queue pairs the guest will use. The control queue is +// used only here at init and has no dedicated interrupt thread, so we submit +// the request and poll the used ring synchronously for the device's ACK. +bool net::set_vq_pairs(u16 pairs) +{ + vring* vq = get_virt_queue(_ctrl_vq_idx); + if (!vq) { + return false; + } + + struct { + net_ctrl_hdr hdr; + net_ctrl_mq mq; + net_ctrl_ack ack; + } cmd; + cmd.hdr.class_t = VIRTIO_NET_CTRL_MQ; + cmd.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; + cmd.mq.virtqueue_pairs = pairs; + cmd.ack = VIRTIO_NET_ERR; + + vq->init_sg(); + vq->add_out_sg(&cmd.hdr, sizeof(cmd.hdr)); + vq->add_out_sg(&cmd.mq, sizeof(cmd.mq)); + vq->add_in_sg(&cmd.ack, sizeof(cmd.ack)); + + if (!vq->add_buf(&cmd)) { + return false; + } + vq->kick(); + + // The control queue is quiescent at init; busy-wait briefly for the reply. + u32 len; + for (int spins = 0; spins < 100000; spins++) { + if (vq->used_ring_not_empty()) { + vq->get_buf_elem(&len); + vq->get_buf_finalize(); + break; + } + sched::thread::yield(); + } + + return cmd.ack == VIRTIO_NET_OK; +} + +// Program RSS via VIRTIO_NET_CTRL_MQ_RSS_CONFIG so the device hashes each +// inbound flow (by its TCP/UDP 5-tuple, using a Toeplitz key) onto one of the +// _nqp Rx queues. The indirection table is filled round-robin across the +// queues so flows spread evenly instead of piling onto queue 0. The payload +// is variable-length (indirection table and key are inline), so it is +// serialized into a byte buffer. Returns true on device ACK. +bool net::set_rss_config() +{ + vring* vq = get_virt_queue(_ctrl_vq_idx); + if (!vq) { + return false; + } + + // Hash the flow types we care about, intersected with what the device can + // compute. TCP/UDP over IPv4+IPv6 gives per-connection spreading (the PG + // workload is many TCP connections); fall back to plain IP hashing for any + // the device lacks. + u32 want = VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | + VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | + VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | VIRTIO_NET_RSS_HASH_TYPE_IPv6; + u32 hash_types = want & _config.supported_hash_types; + if (hash_types == 0) { + net_w("virtio-net: device supports no RSS hash types we want (0x%x)", + _config.supported_hash_types); + return false; + } + + // Indirection table length: a power of two, at least _nqp so every queue + // appears, capped by what the device allows. 128 is the conventional size + // (matches Linux) and is plenty to spread flows. + unsigned table_len = 128; + if (_config.rss_max_indirection_table_length && + table_len > _config.rss_max_indirection_table_length) { + table_len = _config.rss_max_indirection_table_length; + } + // Round down to a power of two and make sure it can hold every queue. + unsigned p2 = 1; + while (p2 * 2 <= table_len) p2 *= 2; + table_len = p2; + if (table_len < _nqp) { + net_w("virtio-net: RSS table (%d) smaller than queues (%d); flows will " + "not reach every queue", table_len, _nqp); + } + + // Toeplitz hash key. Length is device-dictated (rss_max_key_size); use the + // well-known symmetric-ish 40-byte Microsoft key, truncated/zero-padded to + // the device's size. A fixed key is fine: we only need a stable hash that + // spreads distinct flows, not cryptographic strength. + static const u8 toeplitz_key[40] = { + 0x6d,0x5a,0x56,0xda,0x25,0x5b,0x0e,0xc2,0x41,0x67,0x25,0x3d,0x43,0xa3,0x8f,0xb0, + 0xd0,0xca,0x2b,0xcb,0xae,0x7b,0x30,0xb4,0x77,0xcb,0x2d,0xa3,0x80,0x30,0xf2,0x0c, + 0x6a,0x42,0xb7,0x3b,0xbe,0xac,0x01,0xfa, + }; + unsigned key_len = _config.rss_max_key_size; + if (key_len == 0 || key_len > sizeof(toeplitz_key)) { + key_len = sizeof(toeplitz_key); + } + + // Serialize: hdr | indirection_table[table_len] | max_tx_vq | key_len | key. + size_t payload = sizeof(net_ctrl_rss_hdr) + + table_len * sizeof(u16) + + sizeof(u16) /* max_tx_vq */ + + sizeof(u8) /* hash_key_length */ + + key_len; + std::vector buf(payload, 0); + u8* p = buf.data(); + + auto* hdr = reinterpret_cast(p); + hdr->hash_types = hash_types; + hdr->indirection_table_mask = static_cast(table_len - 1); + hdr->unclassified_queue = 0; // flows we don't hash land on queue 0 + p += sizeof(net_ctrl_rss_hdr); + + auto* table = reinterpret_cast(p); + for (unsigned i = 0; i < table_len; i++) { + table[i] = static_cast(i % _nqp); // round-robin across all queues + } + p += table_len * sizeof(u16); + + // max_tx_vq: highest Tx virtqueue the device may read from = _nqp. + u16 max_tx_vq = static_cast(_nqp); + memcpy(p, &max_tx_vq, sizeof(max_tx_vq)); + p += sizeof(max_tx_vq); + + *p = static_cast(key_len); + p += sizeof(u8); + memcpy(p, toeplitz_key, key_len); + + net_ctrl_hdr chdr; + chdr.class_t = VIRTIO_NET_CTRL_MQ; + chdr.cmd = VIRTIO_NET_CTRL_MQ_RSS_CONFIG; + net_ctrl_ack ack = VIRTIO_NET_ERR; + + vq->init_sg(); + vq->add_out_sg(&chdr, sizeof(chdr)); + vq->add_out_sg(buf.data(), buf.size()); + vq->add_in_sg(&ack, sizeof(ack)); + + if (!vq->add_buf(buf.data())) { + return false; + } + vq->kick(); + + // The control queue is quiescent at init; busy-wait briefly for the reply. + u32 len; + for (int spins = 0; spins < 100000; spins++) { + if (vq->used_ring_not_empty()) { + vq->get_buf_elem(&len); + vq->get_buf_finalize(); + break; + } + sched::thread::yield(); + } + + return ack == VIRTIO_NET_OK; +} + u64 net::get_driver_features() { auto base = virtio_driver::get_driver_features(); @@ -942,6 +1261,19 @@ u64 net::get_driver_features() | (1 << VIRTIO_NET_F_HOST_TSO4) \ | (1 << VIRTIO_NET_F_GUEST_ECN) | (1 << VIRTIO_NET_F_GUEST_UFO) + // Multiqueue and its control channel. VIRTIO_NET_F_MQ lets the + // device steer received flows across several Rx queues; + // VIRTIO_NET_F_CTRL_VQ is required to send the + // CTRL_MQ_VQ_PAIRS_SET command that activates them. + | (1 << VIRTIO_NET_F_CTRL_VQ) + | (1 << VIRTIO_NET_F_MQ) + // Receive-Side Scaling: let the device hash each inbound flow + // (by its 5-tuple, via a Toeplitz key) onto one of the N Rx + // queues. Without this the host has no hash to distribute + // flows and inbound traffic piles onto Rx queue 0, so only one + // receiver thread ever runs even with N queues. Bit 60 is + // outside the 32-bit range, so it must be set with 1ULL. + | (1ULL << VIRTIO_NET_F_RSS) ); } diff --git a/drivers/virtio-net.hh b/drivers/virtio-net.hh index 274e226ab..8e78cbaf1 100644 --- a/drivers/virtio-net.hh +++ b/drivers/virtio-net.hh @@ -15,6 +15,7 @@ #include #include +#include #include "drivers/virtio.hh" #include "drivers/pci-device.hh" @@ -28,6 +29,14 @@ namespace virtio { class net : public virtio_driver { public: + // Forward declarations of the per-queue structs (defined in the private + // section below) so member functions can be declared with rxq*/txq*. + // Declared private to match the access of their definitions. +private: + struct rxq; + struct txq; +public: + // The feature bitmap for virtio net enum NetFeatures { VIRTIO_NET_F_CSUM = 0, /* Host handles pkts w/ partial csum */ @@ -51,6 +60,7 @@ public: VIRTIO_NET_F_GUEST_ANNOUNCE = 21, /* Guest can announce device on the network */ VIRTIO_NET_F_MQ = 22, /* Device supports Receive Flow Steering */ VIRTIO_NET_F_CTRL_MAC_ADDR = 23, /* Set MAC address */ + VIRTIO_NET_F_RSS = 60, /* Device supports RSS (Toeplitz) flow steering */ }; enum { @@ -103,9 +113,18 @@ public: VIRTIO_NET_CTRL_MQ = 4, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET = 0, + VIRTIO_NET_CTRL_MQ_RSS_CONFIG = 1, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN = 1, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX = 0x8000, + /* RSS hash types (virtio spec 5.1.6.5.6.4). */ + VIRTIO_NET_RSS_HASH_TYPE_IPv4 = (1 << 0), + VIRTIO_NET_RSS_HASH_TYPE_TCPv4 = (1 << 1), + VIRTIO_NET_RSS_HASH_TYPE_UDPv4 = (1 << 2), + VIRTIO_NET_RSS_HASH_TYPE_IPv6 = (1 << 3), + VIRTIO_NET_RSS_HASH_TYPE_TCPv6 = (1 << 4), + VIRTIO_NET_RSS_HASH_TYPE_UDPv6 = (1 << 5), + ETH_ALEN = 14, VIRTIO_NET_CSUM_OFFLOAD = CSUM_TCP | CSUM_UDP, }; @@ -125,6 +144,11 @@ public: * Legal values are between 1 and 0x8000 */ u16 max_virtqueue_pairs; + /* The following three fields are only valid (and only present in the + * device config space) when VIRTIO_NET_F_RSS is negotiated. */ + u8 rss_max_key_size; + u16 rss_max_indirection_table_length; + u32 supported_hash_types; } __attribute__((packed)); /* This is the first element of the scatter-gather list. If you don't @@ -209,6 +233,23 @@ public: u16 virtqueue_pairs; }; + /* + * RSS configuration (VIRTIO_NET_CTRL_MQ_RSS_CONFIG). The command payload + * is variable-length (the indirection table and hash key are inline), so + * this fixed prefix is followed on the wire by: + * le16 indirection_table[indirection_table_mask + 1]; + * le16 max_tx_vq; + * u8 hash_key_length; + * u8 hash_key_data[hash_key_length]; + * We therefore serialize the whole thing into a byte buffer in + * set_rss_config() rather than using a single struct. + */ + struct net_ctrl_rss_hdr { + u32 hash_types; + u16 indirection_table_mask; + u16 unclassified_queue; + } __attribute__((packed)); + explicit net(virtio_device& dev); virtual ~net(); void init(); @@ -220,8 +261,12 @@ public: void wait_for_queue(vring* queue); bool bad_rx_csum(struct mbuf* m, struct net_hdr* hdr); - void receiver(); - void fill_rx_ring(); + // receiver()/fill_rx_ring() operate on a specific Rx queue so that with + // multiqueue (VIRTIO_NET_F_MQ) each Rx queue is drained by its own poll + // thread on its own CPU, instead of all inbound traffic funnelling through + // one queue and one thread. + void receiver(struct rxq* rxq); + void fill_rx_ring(struct rxq* rxq); mbuf* packet_to_mbuf(const std::vector& iovec); unsigned* rx_buffer_refcnt(void* frame); static void free_rx_buffer(void* buffer, void* refcnt); @@ -313,7 +358,8 @@ private: } } _pre_init; - /* Single Rx queue object */ + /* An Rx queue object. With VIRTIO_NET_F_MQ there is one per queue pair, + * each drained by its own poll thread. */ struct rxq { rxq(vring* vq, std::function poll_func) : vqueue(vq), poll_task(sched::thread::make(poll_func, sched::thread::attr(). @@ -497,9 +543,29 @@ private: } } - /* We currently support only a single Rx+Tx queue */ - struct rxq _rxq; - struct txq _txq; + /* Rx+Tx queue pairs. Without VIRTIO_NET_F_MQ there is exactly one pair, + * preserving the original single-queue behaviour. txq is over-aligned + * (its percpu xmitter is cacheline aligned), so it is allocated with + * aligned_new and freed with the matching deleter. */ + std::vector> _rxqs; + std::vector>> _txqs; + /* Number of active queue pairs (1 when multiqueue is not negotiated). */ + unsigned _nqp = 1; + /* Control virtqueue index (2*_nqp) when VIRTIO_NET_F_CTRL_VQ is present. */ + int _ctrl_vq_idx = -1; + /* True when VIRTIO_NET_F_RSS was negotiated and RSS was programmed so the + * device hashes each inbound flow onto one of the _nqp Rx queues. */ + bool _rss = false; + + /* Tell the device how many queue pairs the guest will use, per the + * VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET command. Returns true on device ACK. */ + bool set_vq_pairs(u16 pairs); + + /* Program the device's RSS indirection table (round-robin across the _nqp + * Rx queues), Toeplitz hash key and hash types via + * VIRTIO_NET_CTRL_MQ_RSS_CONFIG so inbound flows are hashed across all Rx + * queues instead of piling onto queue 0. Returns true on device ACK. */ + bool set_rss_config(); //maintains the virtio instance number for multiple drives static int _instance; diff --git a/drivers/virtio.cc b/drivers/virtio.cc index c58b39148..88f938b90 100644 --- a/drivers/virtio.cc +++ b/drivers/virtio.cc @@ -6,6 +6,8 @@ */ #include +#include +#include #include "drivers/virtio.hh" #include "virtio-vring.hh" @@ -19,6 +21,38 @@ namespace virtio { int virtio_driver::_disk_idx = 0; +// Process-wide MSI-X vector budget shared by all virtio devices. The x86-64 +// IDT exposes 224 usable vectors (32..255); GSI/legacy IRQs, the APIC timer +// and a few IPIs also consume some, so we hand out at most a conservative +// fraction to per-queue MSI-X and leave headroom. Each multiqueue device caps +// its interrupt-bearing queue count against what remains here so that a +// high-vCPU guest with several such devices still boots. This is a soft cap: +// it only limits how many queues a device arms with a distinct interrupt, not +// how many virtqueues exist. +static std::atomic msix_vector_budget{192}; + +unsigned virtio_driver::reserve_msix_vectors(unsigned want) +{ + if (want == 0) { + return 0; + } + int remaining = msix_vector_budget.load(std::memory_order_relaxed); + while (true) { + if (remaining <= 0) { + // Pool empty: still allow a single vector so the device is usable + // (it will run single-queue / shared-interrupt). + return 1; + } + unsigned grant = std::min(want, (unsigned)remaining); + if (msix_vector_budget.compare_exchange_weak( + remaining, remaining - (int)grant, + std::memory_order_relaxed)) { + return grant; + } + // remaining reloaded by compare_exchange_weak; retry. + } +} + virtio_driver::virtio_driver(virtio_device& dev) : hw_driver() , _dev(dev) @@ -57,13 +91,13 @@ void virtio_driver::setup_features() //notify the host about the features in used according //to the virtio spec for (int i = 0; i < 64; i++) - if (subset & (1 << i)) + if (subset & (1ULL << i)) virtio_d("%s: found feature intersec of bit %d\n", __FUNCTION__, i); - if (subset & (1 << VIRTIO_RING_F_INDIRECT_DESC)) + if (subset & (1ULL << VIRTIO_RING_F_INDIRECT_DESC)) set_indirect_buf_cap(true); - if (subset & (1 << VIRTIO_RING_F_EVENT_IDX)) + if (subset & (1ULL << VIRTIO_RING_F_EVENT_IDX)) set_event_idx_cap(true); set_guest_features(subset); @@ -87,7 +121,7 @@ void virtio_driver::dump_config() virtio_d(" virtio features: "); for (int i = 0; i < 64; i++) { - virtio_d(" %d ", 0 != (device_features & (1 << i))); + virtio_d(" %d ", 0 != (device_features & (1ULL << i))); } #endif } @@ -191,7 +225,7 @@ void virtio_driver::set_guest_features(u64 features) bool virtio_driver::get_guest_feature_bit(int bit) { - return (_enabled_features & (1 << bit)) != 0; + return (_enabled_features & (1ULL << bit)) != 0; } u8 virtio_driver::get_dev_status() diff --git a/drivers/virtio.hh b/drivers/virtio.hh index 3e52348c7..0e49e44c5 100644 --- a/drivers/virtio.hh +++ b/drivers/virtio.hh @@ -104,6 +104,16 @@ public: size_t get_vring_alignment() { return _dev.get_vring_alignment();} + // Reserve up to `want` MSI-X vectors from a process-wide budget and return + // how many are actually available. The x86-64 IDT has only 224 usable + // vectors (32..255) shared by every device, so a high-vCPU guest with + // several multiqueue virtio devices can exhaust them and crash at boot. + // Drivers that scale their queue count with the vCPU count (for example + // virtio-blk multiqueue) call this to cap the number of queues they arm + // with an interrupt, so the total stays within the budget and every device + // still gets at least one vector. Returns 0 only if the pool is empty. + static unsigned reserve_msix_vectors(unsigned want); + protected: // Actual drivers should implement this on top of the basic ring features virtual u64 get_driver_features() { return 1 << VIRTIO_RING_F_INDIRECT_DESC | 1 << VIRTIO_RING_F_EVENT_IDX; }